1 /*
    2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "asm/assembler.hpp"
   26 #include "asm/assembler.inline.hpp"
   27 #include "code/compiledIC.hpp"
   28 #include "compiler/compiler_globals.hpp"
   29 #include "compiler/disassembler.hpp"
   30 #include "crc32c.h"
   31 #include "gc/shared/barrierSet.hpp"
   32 #include "gc/shared/barrierSetAssembler.hpp"
   33 #include "gc/shared/collectedHeap.inline.hpp"
   34 #include "gc/shared/tlab_globals.hpp"
   35 #include "interpreter/bytecodeHistogram.hpp"
   36 #include "interpreter/interpreter.hpp"
   37 #include "interpreter/interpreterRuntime.hpp"
   38 #include "jvm.h"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedKlass.inline.hpp"
   43 #include "oops/compressedOops.inline.hpp"
   44 #include "oops/klass.inline.hpp"
   45 #include "prims/methodHandles.hpp"
   46 #include "runtime/continuation.hpp"
   47 #include "runtime/interfaceSupport.inline.hpp"
   48 #include "runtime/javaThread.hpp"
   49 #include "runtime/jniHandles.hpp"
   50 #include "runtime/objectMonitor.hpp"
   51 #include "runtime/os.hpp"
   52 #include "runtime/safepoint.hpp"
   53 #include "runtime/safepointMechanism.hpp"
   54 #include "runtime/sharedRuntime.hpp"
   55 #include "runtime/stubRoutines.hpp"
   56 #include "utilities/checkedCast.hpp"
   57 #include "utilities/macros.hpp"
   58 
   59 #ifdef PRODUCT
   60 #define BLOCK_COMMENT(str) /* nothing */
   61 #define STOP(error) stop(error)
   62 #else
   63 #define BLOCK_COMMENT(str) block_comment(str)
   64 #define STOP(error) block_comment(error); stop(error)
   65 #endif
   66 
   67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   68 
   69 #ifdef ASSERT
   70 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   71 #endif
   72 
   73 static const Assembler::Condition reverse[] = {
   74     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   75     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   76     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   77     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   78     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   79     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   80     Assembler::above          /* belowEqual    = 0x6 */ ,
   81     Assembler::belowEqual     /* above         = 0x7 */ ,
   82     Assembler::positive       /* negative      = 0x8 */ ,
   83     Assembler::negative       /* positive      = 0x9 */ ,
   84     Assembler::noParity       /* parity        = 0xa */ ,
   85     Assembler::parity         /* noParity      = 0xb */ ,
   86     Assembler::greaterEqual   /* less          = 0xc */ ,
   87     Assembler::less           /* greaterEqual  = 0xd */ ,
   88     Assembler::greater        /* lessEqual     = 0xe */ ,
   89     Assembler::lessEqual      /* greater       = 0xf, */
   90 
   91 };
   92 
   93 
   94 // Implementation of MacroAssembler
   95 
   96 // First all the versions that have distinct versions depending on 32/64 bit
   97 // Unless the difference is trivial (1 line or so).
   98 
   99 #ifndef _LP64
  100 
  101 // 32bit versions
  102 
  103 Address MacroAssembler::as_Address(AddressLiteral adr) {
  104   return Address(adr.target(), adr.rspec());
  105 }
  106 
  107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  108   assert(rscratch == noreg, "");
  109   return Address::make_array(adr);
  110 }
  111 
  112 void MacroAssembler::call_VM_leaf_base(address entry_point,
  113                                        int number_of_arguments) {
  114   call(RuntimeAddress(entry_point));
  115   increment(rsp, number_of_arguments * wordSize);
  116 }
  117 
  118 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  119   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  120 }
  121 
  122 
  123 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  124   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  125 }
  126 
  127 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  128   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  129 }
  130 
  131 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  132   assert(rscratch == noreg, "redundant");
  133   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  134 }
  135 
  136 void MacroAssembler::extend_sign(Register hi, Register lo) {
  137   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  138   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  139     cdql();
  140   } else {
  141     movl(hi, lo);
  142     sarl(hi, 31);
  143   }
  144 }
  145 
  146 void MacroAssembler::jC2(Register tmp, Label& L) {
  147   // set parity bit if FPU flag C2 is set (via rax)
  148   save_rax(tmp);
  149   fwait(); fnstsw_ax();
  150   sahf();
  151   restore_rax(tmp);
  152   // branch
  153   jcc(Assembler::parity, L);
  154 }
  155 
  156 void MacroAssembler::jnC2(Register tmp, Label& L) {
  157   // set parity bit if FPU flag C2 is set (via rax)
  158   save_rax(tmp);
  159   fwait(); fnstsw_ax();
  160   sahf();
  161   restore_rax(tmp);
  162   // branch
  163   jcc(Assembler::noParity, L);
  164 }
  165 
  166 // 32bit can do a case table jump in one instruction but we no longer allow the base
  167 // to be installed in the Address class
  168 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  169   assert(rscratch == noreg, "not needed");
  170   jmp(as_Address(entry, noreg));
  171 }
  172 
  173 // Note: y_lo will be destroyed
  174 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  175   // Long compare for Java (semantics as described in JVM spec.)
  176   Label high, low, done;
  177 
  178   cmpl(x_hi, y_hi);
  179   jcc(Assembler::less, low);
  180   jcc(Assembler::greater, high);
  181   // x_hi is the return register
  182   xorl(x_hi, x_hi);
  183   cmpl(x_lo, y_lo);
  184   jcc(Assembler::below, low);
  185   jcc(Assembler::equal, done);
  186 
  187   bind(high);
  188   xorl(x_hi, x_hi);
  189   increment(x_hi);
  190   jmp(done);
  191 
  192   bind(low);
  193   xorl(x_hi, x_hi);
  194   decrementl(x_hi);
  195 
  196   bind(done);
  197 }
  198 
  199 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  200   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  201 }
  202 
  203 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  204   assert(rscratch == noreg, "not needed");
  205 
  206   // leal(dst, as_Address(adr));
  207   // see note in movl as to why we must use a move
  208   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  209 }
  210 
  211 void MacroAssembler::leave() {
  212   mov(rsp, rbp);
  213   pop(rbp);
  214 }
  215 
  216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  217   // Multiplication of two Java long values stored on the stack
  218   // as illustrated below. Result is in rdx:rax.
  219   //
  220   // rsp ---> [  ??  ] \               \
  221   //            ....    | y_rsp_offset  |
  222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  223   //          [ y_hi ]                  | (in bytes)
  224   //            ....                    |
  225   //          [ x_lo ]                 /
  226   //          [ x_hi ]
  227   //            ....
  228   //
  229   // Basic idea: lo(result) = lo(x_lo * y_lo)
  230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  233   Label quick;
  234   // load x_hi, y_hi and check if quick
  235   // multiplication is possible
  236   movl(rbx, x_hi);
  237   movl(rcx, y_hi);
  238   movl(rax, rbx);
  239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  241   // do full multiplication
  242   // 1st step
  243   mull(y_lo);                                    // x_hi * y_lo
  244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  245   // 2nd step
  246   movl(rax, x_lo);
  247   mull(rcx);                                     // x_lo * y_hi
  248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  249   // 3rd step
  250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  251   movl(rax, x_lo);
  252   mull(y_lo);                                    // x_lo * y_lo
  253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  254 }
  255 
  256 void MacroAssembler::lneg(Register hi, Register lo) {
  257   negl(lo);
  258   adcl(hi, 0);
  259   negl(hi);
  260 }
  261 
  262 void MacroAssembler::lshl(Register hi, Register lo) {
  263   // Java shift left long support (semantics as described in JVM spec., p.305)
  264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  265   // shift value is in rcx !
  266   assert(hi != rcx, "must not use rcx");
  267   assert(lo != rcx, "must not use rcx");
  268   const Register s = rcx;                        // shift count
  269   const int      n = BitsPerWord;
  270   Label L;
  271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  272   cmpl(s, n);                                    // if (s < n)
  273   jcc(Assembler::less, L);                       // else (s >= n)
  274   movl(hi, lo);                                  // x := x << n
  275   xorl(lo, lo);
  276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  277   bind(L);                                       // s (mod n) < n
  278   shldl(hi, lo);                                 // x := x << s
  279   shll(lo);
  280 }
  281 
  282 
  283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  286   assert(hi != rcx, "must not use rcx");
  287   assert(lo != rcx, "must not use rcx");
  288   const Register s = rcx;                        // shift count
  289   const int      n = BitsPerWord;
  290   Label L;
  291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  292   cmpl(s, n);                                    // if (s < n)
  293   jcc(Assembler::less, L);                       // else (s >= n)
  294   movl(lo, hi);                                  // x := x >> n
  295   if (sign_extension) sarl(hi, 31);
  296   else                xorl(hi, hi);
  297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  298   bind(L);                                       // s (mod n) < n
  299   shrdl(lo, hi);                                 // x := x >> s
  300   if (sign_extension) sarl(hi);
  301   else                shrl(hi);
  302 }
  303 
  304 void MacroAssembler::movoop(Register dst, jobject obj) {
  305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  306 }
  307 
  308 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  309   assert(rscratch == noreg, "redundant");
  310   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  311 }
  312 
  313 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  314   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  315 }
  316 
  317 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  318   assert(rscratch == noreg, "redundant");
  319   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  320 }
  321 
  322 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  323   if (src.is_lval()) {
  324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  325   } else {
  326     movl(dst, as_Address(src));
  327   }
  328 }
  329 
  330 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  331   assert(rscratch == noreg, "redundant");
  332   movl(as_Address(dst, noreg), src);
  333 }
  334 
  335 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  336   movl(dst, as_Address(src, noreg));
  337 }
  338 
  339 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  340   assert(rscratch == noreg, "redundant");
  341   movl(dst, src);
  342 }
  343 
  344 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  345   assert(rscratch == noreg, "redundant");
  346   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  347 }
  348 
  349 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  350   assert(rscratch == noreg, "redundant");
  351   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  352 }
  353 
  354 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  355   assert(rscratch == noreg, "redundant");
  356   if (src.is_lval()) {
  357     push_literal32((int32_t)src.target(), src.rspec());
  358   } else {
  359     pushl(as_Address(src));
  360   }
  361 }
  362 
  363 static void pass_arg0(MacroAssembler* masm, Register arg) {
  364   masm->push(arg);
  365 }
  366 
  367 static void pass_arg1(MacroAssembler* masm, Register arg) {
  368   masm->push(arg);
  369 }
  370 
  371 static void pass_arg2(MacroAssembler* masm, Register arg) {
  372   masm->push(arg);
  373 }
  374 
  375 static void pass_arg3(MacroAssembler* masm, Register arg) {
  376   masm->push(arg);
  377 }
  378 
  379 #ifndef PRODUCT
  380 extern "C" void findpc(intptr_t x);
  381 #endif
  382 
  383 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  384   // In order to get locks to work, we need to fake a in_VM state
  385   JavaThread* thread = JavaThread::current();
  386   JavaThreadState saved_state = thread->thread_state();
  387   thread->set_thread_state(_thread_in_vm);
  388   if (ShowMessageBoxOnError) {
  389     JavaThread* thread = JavaThread::current();
  390     JavaThreadState saved_state = thread->thread_state();
  391     thread->set_thread_state(_thread_in_vm);
  392     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  393       ttyLocker ttyl;
  394       BytecodeCounter::print();
  395     }
  396     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  397     // This is the value of eip which points to where verify_oop will return.
  398     if (os::message_box(msg, "Execution stopped, print registers?")) {
  399       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  400       BREAKPOINT;
  401     }
  402   }
  403   fatal("DEBUG MESSAGE: %s", msg);
  404 }
  405 
  406 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  407   ttyLocker ttyl;
  408   DebuggingContext debugging{};
  409   tty->print_cr("eip = 0x%08x", eip);
  410 #ifndef PRODUCT
  411   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  412     tty->cr();
  413     findpc(eip);
  414     tty->cr();
  415   }
  416 #endif
  417 #define PRINT_REG(rax) \
  418   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  419   PRINT_REG(rax);
  420   PRINT_REG(rbx);
  421   PRINT_REG(rcx);
  422   PRINT_REG(rdx);
  423   PRINT_REG(rdi);
  424   PRINT_REG(rsi);
  425   PRINT_REG(rbp);
  426   PRINT_REG(rsp);
  427 #undef PRINT_REG
  428   // Print some words near top of staack.
  429   int* dump_sp = (int*) rsp;
  430   for (int col1 = 0; col1 < 8; col1++) {
  431     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  432     os::print_location(tty, *dump_sp++);
  433   }
  434   for (int row = 0; row < 16; row++) {
  435     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  436     for (int col = 0; col < 8; col++) {
  437       tty->print(" 0x%08x", *dump_sp++);
  438     }
  439     tty->cr();
  440   }
  441   // Print some instructions around pc:
  442   Disassembler::decode((address)eip-64, (address)eip);
  443   tty->print_cr("--------");
  444   Disassembler::decode((address)eip, (address)eip+32);
  445 }
  446 
  447 void MacroAssembler::stop(const char* msg) {
  448   // push address of message
  449   ExternalAddress message((address)msg);
  450   pushptr(message.addr(), noreg);
  451   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  452   pusha();                                            // push registers
  453   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  454   hlt();
  455 }
  456 
  457 void MacroAssembler::warn(const char* msg) {
  458   push_CPU_state();
  459 
  460   // push address of message
  461   ExternalAddress message((address)msg);
  462   pushptr(message.addr(), noreg);
  463 
  464   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  465   addl(rsp, wordSize);       // discard argument
  466   pop_CPU_state();
  467 }
  468 
  469 void MacroAssembler::print_state() {
  470   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  471   pusha();                                            // push registers
  472 
  473   push_CPU_state();
  474   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  475   pop_CPU_state();
  476 
  477   popa();
  478   addl(rsp, wordSize);
  479 }
  480 
  481 #else // _LP64
  482 
  483 // 64 bit versions
  484 
  485 Address MacroAssembler::as_Address(AddressLiteral adr) {
  486   // amd64 always does this as a pc-rel
  487   // we can be absolute or disp based on the instruction type
  488   // jmp/call are displacements others are absolute
  489   assert(!adr.is_lval(), "must be rval");
  490   assert(reachable(adr), "must be");
  491   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  492 
  493 }
  494 
  495 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  496   AddressLiteral base = adr.base();
  497   lea(rscratch, base);
  498   Address index = adr.index();
  499   assert(index._disp == 0, "must not have disp"); // maybe it can?
  500   Address array(rscratch, index._index, index._scale, index._disp);
  501   return array;
  502 }
  503 
  504 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  505   Label L, E;
  506 
  507 #ifdef _WIN64
  508   // Windows always allocates space for it's register args
  509   assert(num_args <= 4, "only register arguments supported");
  510   subq(rsp,  frame::arg_reg_save_area_bytes);
  511 #endif
  512 
  513   // Align stack if necessary
  514   testl(rsp, 15);
  515   jcc(Assembler::zero, L);
  516 
  517   subq(rsp, 8);
  518   call(RuntimeAddress(entry_point));
  519   addq(rsp, 8);
  520   jmp(E);
  521 
  522   bind(L);
  523   call(RuntimeAddress(entry_point));
  524 
  525   bind(E);
  526 
  527 #ifdef _WIN64
  528   // restore stack pointer
  529   addq(rsp, frame::arg_reg_save_area_bytes);
  530 #endif
  531 }
  532 
  533 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  534   assert(!src2.is_lval(), "should use cmpptr");
  535   assert(rscratch != noreg || always_reachable(src2), "missing");
  536 
  537   if (reachable(src2)) {
  538     cmpq(src1, as_Address(src2));
  539   } else {
  540     lea(rscratch, src2);
  541     Assembler::cmpq(src1, Address(rscratch, 0));
  542   }
  543 }
  544 
  545 int MacroAssembler::corrected_idivq(Register reg) {
  546   // Full implementation of Java ldiv and lrem; checks for special
  547   // case as described in JVM spec., p.243 & p.271.  The function
  548   // returns the (pc) offset of the idivl instruction - may be needed
  549   // for implicit exceptions.
  550   //
  551   //         normal case                           special case
  552   //
  553   // input : rax: dividend                         min_long
  554   //         reg: divisor   (may not be eax/edx)   -1
  555   //
  556   // output: rax: quotient  (= rax idiv reg)       min_long
  557   //         rdx: remainder (= rax irem reg)       0
  558   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  559   static const int64_t min_long = 0x8000000000000000;
  560   Label normal_case, special_case;
  561 
  562   // check for special case
  563   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  564   jcc(Assembler::notEqual, normal_case);
  565   xorl(rdx, rdx); // prepare rdx for possible special case (where
  566                   // remainder = 0)
  567   cmpq(reg, -1);
  568   jcc(Assembler::equal, special_case);
  569 
  570   // handle normal case
  571   bind(normal_case);
  572   cdqq();
  573   int idivq_offset = offset();
  574   idivq(reg);
  575 
  576   // normal and special case exit
  577   bind(special_case);
  578 
  579   return idivq_offset;
  580 }
  581 
  582 void MacroAssembler::decrementq(Register reg, int value) {
  583   if (value == min_jint) { subq(reg, value); return; }
  584   if (value <  0) { incrementq(reg, -value); return; }
  585   if (value == 0) {                        ; return; }
  586   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  587   /* else */      { subq(reg, value)       ; return; }
  588 }
  589 
  590 void MacroAssembler::decrementq(Address dst, int value) {
  591   if (value == min_jint) { subq(dst, value); return; }
  592   if (value <  0) { incrementq(dst, -value); return; }
  593   if (value == 0) {                        ; return; }
  594   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  595   /* else */      { subq(dst, value)       ; return; }
  596 }
  597 
  598 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  599   assert(rscratch != noreg || always_reachable(dst), "missing");
  600 
  601   if (reachable(dst)) {
  602     incrementq(as_Address(dst));
  603   } else {
  604     lea(rscratch, dst);
  605     incrementq(Address(rscratch, 0));
  606   }
  607 }
  608 
  609 void MacroAssembler::incrementq(Register reg, int value) {
  610   if (value == min_jint) { addq(reg, value); return; }
  611   if (value <  0) { decrementq(reg, -value); return; }
  612   if (value == 0) {                        ; return; }
  613   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  614   /* else */      { addq(reg, value)       ; return; }
  615 }
  616 
  617 void MacroAssembler::incrementq(Address dst, int value) {
  618   if (value == min_jint) { addq(dst, value); return; }
  619   if (value <  0) { decrementq(dst, -value); return; }
  620   if (value == 0) {                        ; return; }
  621   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  622   /* else */      { addq(dst, value)       ; return; }
  623 }
  624 
  625 // 32bit can do a case table jump in one instruction but we no longer allow the base
  626 // to be installed in the Address class
  627 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  628   lea(rscratch, entry.base());
  629   Address dispatch = entry.index();
  630   assert(dispatch._base == noreg, "must be");
  631   dispatch._base = rscratch;
  632   jmp(dispatch);
  633 }
  634 
  635 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  636   ShouldNotReachHere(); // 64bit doesn't use two regs
  637   cmpq(x_lo, y_lo);
  638 }
  639 
  640 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  641   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  642 }
  643 
  644 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  645   lea(rscratch, adr);
  646   movptr(dst, rscratch);
  647 }
  648 
  649 void MacroAssembler::leave() {
  650   // %%% is this really better? Why not on 32bit too?
  651   emit_int8((unsigned char)0xC9); // LEAVE
  652 }
  653 
  654 void MacroAssembler::lneg(Register hi, Register lo) {
  655   ShouldNotReachHere(); // 64bit doesn't use two regs
  656   negq(lo);
  657 }
  658 
  659 void MacroAssembler::movoop(Register dst, jobject obj) {
  660   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  661 }
  662 
  663 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  664   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  665   movq(dst, rscratch);
  666 }
  667 
  668 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  669   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  670 }
  671 
  672 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  673   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  674   movq(dst, rscratch);
  675 }
  676 
  677 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  678   if (src.is_lval()) {
  679     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  680   } else {
  681     if (reachable(src)) {
  682       movq(dst, as_Address(src));
  683     } else {
  684       lea(dst, src);
  685       movq(dst, Address(dst, 0));
  686     }
  687   }
  688 }
  689 
  690 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  691   movq(as_Address(dst, rscratch), src);
  692 }
  693 
  694 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  695   movq(dst, as_Address(src, dst /*rscratch*/));
  696 }
  697 
  698 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  699 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  700   if (is_simm32(src)) {
  701     movptr(dst, checked_cast<int32_t>(src));
  702   } else {
  703     mov64(rscratch, src);
  704     movq(dst, rscratch);
  705   }
  706 }
  707 
  708 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  709   movoop(rscratch, obj);
  710   push(rscratch);
  711 }
  712 
  713 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  714   mov_metadata(rscratch, obj);
  715   push(rscratch);
  716 }
  717 
  718 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  719   lea(rscratch, src);
  720   if (src.is_lval()) {
  721     push(rscratch);
  722   } else {
  723     pushq(Address(rscratch, 0));
  724   }
  725 }
  726 
  727 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  728   reset_last_Java_frame(r15_thread, clear_fp);
  729 }
  730 
  731 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  732                                          Register last_java_fp,
  733                                          address  last_java_pc,
  734                                          Register rscratch) {
  735   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  736 }
  737 
  738 static void pass_arg0(MacroAssembler* masm, Register arg) {
  739   if (c_rarg0 != arg ) {
  740     masm->mov(c_rarg0, arg);
  741   }
  742 }
  743 
  744 static void pass_arg1(MacroAssembler* masm, Register arg) {
  745   if (c_rarg1 != arg ) {
  746     masm->mov(c_rarg1, arg);
  747   }
  748 }
  749 
  750 static void pass_arg2(MacroAssembler* masm, Register arg) {
  751   if (c_rarg2 != arg ) {
  752     masm->mov(c_rarg2, arg);
  753   }
  754 }
  755 
  756 static void pass_arg3(MacroAssembler* masm, Register arg) {
  757   if (c_rarg3 != arg ) {
  758     masm->mov(c_rarg3, arg);
  759   }
  760 }
  761 
  762 void MacroAssembler::stop(const char* msg) {
  763   if (ShowMessageBoxOnError) {
  764     address rip = pc();
  765     pusha(); // get regs on stack
  766     lea(c_rarg1, InternalAddress(rip));
  767     movq(c_rarg2, rsp); // pass pointer to regs array
  768   }
  769   lea(c_rarg0, ExternalAddress((address) msg));
  770   andq(rsp, -16); // align stack as required by ABI
  771   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  772   hlt();
  773 }
  774 
  775 void MacroAssembler::warn(const char* msg) {
  776   push(rbp);
  777   movq(rbp, rsp);
  778   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  779   push_CPU_state();   // keeps alignment at 16 bytes
  780 
  781   lea(c_rarg0, ExternalAddress((address) msg));
  782   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  783 
  784   pop_CPU_state();
  785   mov(rsp, rbp);
  786   pop(rbp);
  787 }
  788 
  789 void MacroAssembler::print_state() {
  790   address rip = pc();
  791   pusha();            // get regs on stack
  792   push(rbp);
  793   movq(rbp, rsp);
  794   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  795   push_CPU_state();   // keeps alignment at 16 bytes
  796 
  797   lea(c_rarg0, InternalAddress(rip));
  798   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  799   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  800 
  801   pop_CPU_state();
  802   mov(rsp, rbp);
  803   pop(rbp);
  804   popa();
  805 }
  806 
  807 #ifndef PRODUCT
  808 extern "C" void findpc(intptr_t x);
  809 #endif
  810 
  811 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  812   // In order to get locks to work, we need to fake a in_VM state
  813   if (ShowMessageBoxOnError) {
  814     JavaThread* thread = JavaThread::current();
  815     JavaThreadState saved_state = thread->thread_state();
  816     thread->set_thread_state(_thread_in_vm);
  817 #ifndef PRODUCT
  818     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  819       ttyLocker ttyl;
  820       BytecodeCounter::print();
  821     }
  822 #endif
  823     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  824     // XXX correct this offset for amd64
  825     // This is the value of eip which points to where verify_oop will return.
  826     if (os::message_box(msg, "Execution stopped, print registers?")) {
  827       print_state64(pc, regs);
  828       BREAKPOINT;
  829     }
  830   }
  831   fatal("DEBUG MESSAGE: %s", msg);
  832 }
  833 
  834 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  835   ttyLocker ttyl;
  836   DebuggingContext debugging{};
  837   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  838 #ifndef PRODUCT
  839   tty->cr();
  840   findpc(pc);
  841   tty->cr();
  842 #endif
  843 #define PRINT_REG(rax, value) \
  844   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  845   PRINT_REG(rax, regs[15]);
  846   PRINT_REG(rbx, regs[12]);
  847   PRINT_REG(rcx, regs[14]);
  848   PRINT_REG(rdx, regs[13]);
  849   PRINT_REG(rdi, regs[8]);
  850   PRINT_REG(rsi, regs[9]);
  851   PRINT_REG(rbp, regs[10]);
  852   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  853   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  854   PRINT_REG(r8 , regs[7]);
  855   PRINT_REG(r9 , regs[6]);
  856   PRINT_REG(r10, regs[5]);
  857   PRINT_REG(r11, regs[4]);
  858   PRINT_REG(r12, regs[3]);
  859   PRINT_REG(r13, regs[2]);
  860   PRINT_REG(r14, regs[1]);
  861   PRINT_REG(r15, regs[0]);
  862 #undef PRINT_REG
  863   // Print some words near the top of the stack.
  864   int64_t* rsp = &regs[16];
  865   int64_t* dump_sp = rsp;
  866   for (int col1 = 0; col1 < 8; col1++) {
  867     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  868     os::print_location(tty, *dump_sp++);
  869   }
  870   for (int row = 0; row < 25; row++) {
  871     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  872     for (int col = 0; col < 4; col++) {
  873       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  874     }
  875     tty->cr();
  876   }
  877   // Print some instructions around pc:
  878   Disassembler::decode((address)pc-64, (address)pc);
  879   tty->print_cr("--------");
  880   Disassembler::decode((address)pc, (address)pc+32);
  881 }
  882 
  883 // The java_calling_convention describes stack locations as ideal slots on
  884 // a frame with no abi restrictions. Since we must observe abi restrictions
  885 // (like the placement of the register window) the slots must be biased by
  886 // the following value.
  887 static int reg2offset_in(VMReg r) {
  888   // Account for saved rbp and return address
  889   // This should really be in_preserve_stack_slots
  890   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  891 }
  892 
  893 static int reg2offset_out(VMReg r) {
  894   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  895 }
  896 
  897 // A long move
  898 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  899 
  900   // The calling conventions assures us that each VMregpair is either
  901   // all really one physical register or adjacent stack slots.
  902 
  903   if (src.is_single_phys_reg() ) {
  904     if (dst.is_single_phys_reg()) {
  905       if (dst.first() != src.first()) {
  906         mov(dst.first()->as_Register(), src.first()->as_Register());
  907       }
  908     } else {
  909       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  910              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  911       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  912     }
  913   } else if (dst.is_single_phys_reg()) {
  914     assert(src.is_single_reg(),  "not a stack pair");
  915     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  916   } else {
  917     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  918     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  919     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  920   }
  921 }
  922 
  923 // A double move
  924 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  925 
  926   // The calling conventions assures us that each VMregpair is either
  927   // all really one physical register or adjacent stack slots.
  928 
  929   if (src.is_single_phys_reg() ) {
  930     if (dst.is_single_phys_reg()) {
  931       // In theory these overlap but the ordering is such that this is likely a nop
  932       if ( src.first() != dst.first()) {
  933         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  934       }
  935     } else {
  936       assert(dst.is_single_reg(), "not a stack pair");
  937       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  938     }
  939   } else if (dst.is_single_phys_reg()) {
  940     assert(src.is_single_reg(),  "not a stack pair");
  941     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  942   } else {
  943     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  944     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  945     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  946   }
  947 }
  948 
  949 
  950 // A float arg may have to do float reg int reg conversion
  951 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  952   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  953 
  954   // The calling conventions assures us that each VMregpair is either
  955   // all really one physical register or adjacent stack slots.
  956 
  957   if (src.first()->is_stack()) {
  958     if (dst.first()->is_stack()) {
  959       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  960       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  961     } else {
  962       // stack to reg
  963       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  964       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  965     }
  966   } else if (dst.first()->is_stack()) {
  967     // reg to stack
  968     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  969     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  970   } else {
  971     // reg to reg
  972     // In theory these overlap but the ordering is such that this is likely a nop
  973     if ( src.first() != dst.first()) {
  974       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  975     }
  976   }
  977 }
  978 
  979 // On 64 bit we will store integer like items to the stack as
  980 // 64 bits items (x86_32/64 abi) even though java would only store
  981 // 32bits for a parameter. On 32bit it will simply be 32 bits
  982 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  983 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  984   if (src.first()->is_stack()) {
  985     if (dst.first()->is_stack()) {
  986       // stack to stack
  987       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  988       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  989     } else {
  990       // stack to reg
  991       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  992     }
  993   } else if (dst.first()->is_stack()) {
  994     // reg to stack
  995     // Do we really have to sign extend???
  996     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
  997     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  998   } else {
  999     // Do we really have to sign extend???
 1000     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1001     if (dst.first() != src.first()) {
 1002       movq(dst.first()->as_Register(), src.first()->as_Register());
 1003     }
 1004   }
 1005 }
 1006 
 1007 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1008   if (src.first()->is_stack()) {
 1009     if (dst.first()->is_stack()) {
 1010       // stack to stack
 1011       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1012       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1013     } else {
 1014       // stack to reg
 1015       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1016     }
 1017   } else if (dst.first()->is_stack()) {
 1018     // reg to stack
 1019     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1020   } else {
 1021     if (dst.first() != src.first()) {
 1022       movq(dst.first()->as_Register(), src.first()->as_Register());
 1023     }
 1024   }
 1025 }
 1026 
 1027 // An oop arg. Must pass a handle not the oop itself
 1028 void MacroAssembler::object_move(OopMap* map,
 1029                         int oop_handle_offset,
 1030                         int framesize_in_slots,
 1031                         VMRegPair src,
 1032                         VMRegPair dst,
 1033                         bool is_receiver,
 1034                         int* receiver_offset) {
 1035 
 1036   // must pass a handle. First figure out the location we use as a handle
 1037 
 1038   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1039 
 1040   // See if oop is null if it is we need no handle
 1041 
 1042   if (src.first()->is_stack()) {
 1043 
 1044     // Oop is already on the stack as an argument
 1045     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1046     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1047     if (is_receiver) {
 1048       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1049     }
 1050 
 1051     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1052     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1053     // conditionally move a null
 1054     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1055   } else {
 1056 
 1057     // Oop is in a register we must store it to the space we reserve
 1058     // on the stack for oop_handles and pass a handle if oop is non-null
 1059 
 1060     const Register rOop = src.first()->as_Register();
 1061     int oop_slot;
 1062     if (rOop == j_rarg0)
 1063       oop_slot = 0;
 1064     else if (rOop == j_rarg1)
 1065       oop_slot = 1;
 1066     else if (rOop == j_rarg2)
 1067       oop_slot = 2;
 1068     else if (rOop == j_rarg3)
 1069       oop_slot = 3;
 1070     else if (rOop == j_rarg4)
 1071       oop_slot = 4;
 1072     else {
 1073       assert(rOop == j_rarg5, "wrong register");
 1074       oop_slot = 5;
 1075     }
 1076 
 1077     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1078     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1079 
 1080     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1081     // Store oop in handle area, may be null
 1082     movptr(Address(rsp, offset), rOop);
 1083     if (is_receiver) {
 1084       *receiver_offset = offset;
 1085     }
 1086 
 1087     cmpptr(rOop, NULL_WORD);
 1088     lea(rHandle, Address(rsp, offset));
 1089     // conditionally move a null from the handle area where it was just stored
 1090     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1091   }
 1092 
 1093   // If arg is on the stack then place it otherwise it is already in correct reg.
 1094   if (dst.first()->is_stack()) {
 1095     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1096   }
 1097 }
 1098 
 1099 #endif // _LP64
 1100 
 1101 // Now versions that are common to 32/64 bit
 1102 
 1103 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1104   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1105 }
 1106 
 1107 void MacroAssembler::addptr(Register dst, Register src) {
 1108   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1109 }
 1110 
 1111 void MacroAssembler::addptr(Address dst, Register src) {
 1112   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1113 }
 1114 
 1115 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1116   assert(rscratch != noreg || always_reachable(src), "missing");
 1117 
 1118   if (reachable(src)) {
 1119     Assembler::addsd(dst, as_Address(src));
 1120   } else {
 1121     lea(rscratch, src);
 1122     Assembler::addsd(dst, Address(rscratch, 0));
 1123   }
 1124 }
 1125 
 1126 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1127   assert(rscratch != noreg || always_reachable(src), "missing");
 1128 
 1129   if (reachable(src)) {
 1130     addss(dst, as_Address(src));
 1131   } else {
 1132     lea(rscratch, src);
 1133     addss(dst, Address(rscratch, 0));
 1134   }
 1135 }
 1136 
 1137 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1138   assert(rscratch != noreg || always_reachable(src), "missing");
 1139 
 1140   if (reachable(src)) {
 1141     Assembler::addpd(dst, as_Address(src));
 1142   } else {
 1143     lea(rscratch, src);
 1144     Assembler::addpd(dst, Address(rscratch, 0));
 1145   }
 1146 }
 1147 
 1148 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1149 // Stub code is generated once and never copied.
 1150 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1151 void MacroAssembler::align64() {
 1152   align(64, (uint)(uintptr_t)pc());
 1153 }
 1154 
 1155 void MacroAssembler::align32() {
 1156   align(32, (uint)(uintptr_t)pc());
 1157 }
 1158 
 1159 void MacroAssembler::align(uint modulus) {
 1160   // 8273459: Ensure alignment is possible with current segment alignment
 1161   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1162   align(modulus, offset());
 1163 }
 1164 
 1165 void MacroAssembler::align(uint modulus, uint target) {
 1166   if (target % modulus != 0) {
 1167     nop(modulus - (target % modulus));
 1168   }
 1169 }
 1170 
 1171 void MacroAssembler::push_f(XMMRegister r) {
 1172   subptr(rsp, wordSize);
 1173   movflt(Address(rsp, 0), r);
 1174 }
 1175 
 1176 void MacroAssembler::pop_f(XMMRegister r) {
 1177   movflt(r, Address(rsp, 0));
 1178   addptr(rsp, wordSize);
 1179 }
 1180 
 1181 void MacroAssembler::push_d(XMMRegister r) {
 1182   subptr(rsp, 2 * wordSize);
 1183   movdbl(Address(rsp, 0), r);
 1184 }
 1185 
 1186 void MacroAssembler::pop_d(XMMRegister r) {
 1187   movdbl(r, Address(rsp, 0));
 1188   addptr(rsp, 2 * Interpreter::stackElementSize);
 1189 }
 1190 
 1191 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1192   // Used in sign-masking with aligned address.
 1193   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1194   assert(rscratch != noreg || always_reachable(src), "missing");
 1195 
 1196   if (UseAVX > 2 &&
 1197       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 1198       (dst->encoding() >= 16)) {
 1199     vpand(dst, dst, src, AVX_512bit, rscratch);
 1200   } else if (reachable(src)) {
 1201     Assembler::andpd(dst, as_Address(src));
 1202   } else {
 1203     lea(rscratch, src);
 1204     Assembler::andpd(dst, Address(rscratch, 0));
 1205   }
 1206 }
 1207 
 1208 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1209   // Used in sign-masking with aligned address.
 1210   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1211   assert(rscratch != noreg || always_reachable(src), "missing");
 1212 
 1213   if (reachable(src)) {
 1214     Assembler::andps(dst, as_Address(src));
 1215   } else {
 1216     lea(rscratch, src);
 1217     Assembler::andps(dst, Address(rscratch, 0));
 1218   }
 1219 }
 1220 
 1221 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1222   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1223 }
 1224 
 1225 #ifdef _LP64
 1226 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1227   assert(rscratch != noreg || always_reachable(src), "missing");
 1228 
 1229   if (reachable(src)) {
 1230     andq(dst, as_Address(src));
 1231   } else {
 1232     lea(rscratch, src);
 1233     andq(dst, Address(rscratch, 0));
 1234   }
 1235 }
 1236 #endif
 1237 
 1238 void MacroAssembler::atomic_incl(Address counter_addr) {
 1239   lock();
 1240   incrementl(counter_addr);
 1241 }
 1242 
 1243 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1244   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1245 
 1246   if (reachable(counter_addr)) {
 1247     atomic_incl(as_Address(counter_addr));
 1248   } else {
 1249     lea(rscratch, counter_addr);
 1250     atomic_incl(Address(rscratch, 0));
 1251   }
 1252 }
 1253 
 1254 #ifdef _LP64
 1255 void MacroAssembler::atomic_incq(Address counter_addr) {
 1256   lock();
 1257   incrementq(counter_addr);
 1258 }
 1259 
 1260 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1261   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1262 
 1263   if (reachable(counter_addr)) {
 1264     atomic_incq(as_Address(counter_addr));
 1265   } else {
 1266     lea(rscratch, counter_addr);
 1267     atomic_incq(Address(rscratch, 0));
 1268   }
 1269 }
 1270 #endif
 1271 
 1272 // Writes to stack successive pages until offset reached to check for
 1273 // stack overflow + shadow pages.  This clobbers tmp.
 1274 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1275   movptr(tmp, rsp);
 1276   // Bang stack for total size given plus shadow page size.
 1277   // Bang one page at a time because large size can bang beyond yellow and
 1278   // red zones.
 1279   Label loop;
 1280   bind(loop);
 1281   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1282   subptr(tmp, (int)os::vm_page_size());
 1283   subl(size, (int)os::vm_page_size());
 1284   jcc(Assembler::greater, loop);
 1285 
 1286   // Bang down shadow pages too.
 1287   // At this point, (tmp-0) is the last address touched, so don't
 1288   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1289   // was post-decremented.)  Skip this address by starting at i=1, and
 1290   // touch a few more pages below.  N.B.  It is important to touch all
 1291   // the way down including all pages in the shadow zone.
 1292   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1293     // this could be any sized move but this is can be a debugging crumb
 1294     // so the bigger the better.
 1295     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1296   }
 1297 }
 1298 
 1299 void MacroAssembler::reserved_stack_check() {
 1300   // testing if reserved zone needs to be enabled
 1301   Label no_reserved_zone_enabling;
 1302   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1303   NOT_LP64(get_thread(rsi);)
 1304 
 1305   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1306   jcc(Assembler::below, no_reserved_zone_enabling);
 1307 
 1308   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1309   jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
 1310   should_not_reach_here();
 1311 
 1312   bind(no_reserved_zone_enabling);
 1313 }
 1314 
 1315 void MacroAssembler::c2bool(Register x) {
 1316   // implements x == 0 ? 0 : 1
 1317   // note: must only look at least-significant byte of x
 1318   //       since C-style booleans are stored in one byte
 1319   //       only! (was bug)
 1320   andl(x, 0xFF);
 1321   setb(Assembler::notZero, x);
 1322 }
 1323 
 1324 // Wouldn't need if AddressLiteral version had new name
 1325 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1326   Assembler::call(L, rtype);
 1327 }
 1328 
 1329 void MacroAssembler::call(Register entry) {
 1330   Assembler::call(entry);
 1331 }
 1332 
 1333 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1334   assert(rscratch != noreg || always_reachable(entry), "missing");
 1335 
 1336   if (reachable(entry)) {
 1337     Assembler::call_literal(entry.target(), entry.rspec());
 1338   } else {
 1339     lea(rscratch, entry);
 1340     Assembler::call(rscratch);
 1341   }
 1342 }
 1343 
 1344 void MacroAssembler::ic_call(address entry, jint method_index) {
 1345   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1346 #ifdef _LP64
 1347   // Needs full 64-bit immediate for later patching.
 1348   mov64(rax, (int64_t)Universe::non_oop_word());
 1349 #else
 1350   movptr(rax, (intptr_t)Universe::non_oop_word());
 1351 #endif
 1352   call(AddressLiteral(entry, rh));
 1353 }
 1354 
 1355 int MacroAssembler::ic_check_size() {
 1356   return
 1357       LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12);
 1358 }
 1359 
 1360 int MacroAssembler::ic_check(int end_alignment) {
 1361   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1362   Register data = rax;
 1363   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1364 
 1365   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1366   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1367   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1368   // before the inline cache check here, and not after
 1369   align(end_alignment, offset() + ic_check_size());
 1370 
 1371   int uep_offset = offset();
 1372 
 1373 #ifdef _LP64
 1374   if (UseCompactObjectHeaders) {
 1375     load_narrow_klass_compact(temp, receiver);
 1376     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1377   } else
 1378 #endif
 1379   if (UseCompressedClassPointers) {
 1380     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1381     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1382   } else {
 1383     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1384     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1385   }
 1386 
 1387   // if inline cache check fails, then jump to runtime routine
 1388   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1389   assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
 1390 
 1391   return uep_offset;
 1392 }
 1393 
 1394 void MacroAssembler::emit_static_call_stub() {
 1395   // Static stub relocation also tags the Method* in the code-stream.
 1396   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1397   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1398   jump(RuntimeAddress(pc()));
 1399 }
 1400 
 1401 // Implementation of call_VM versions
 1402 
 1403 void MacroAssembler::call_VM(Register oop_result,
 1404                              address entry_point,
 1405                              bool check_exceptions) {
 1406   Label C, E;
 1407   call(C, relocInfo::none);
 1408   jmp(E);
 1409 
 1410   bind(C);
 1411   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1412   ret(0);
 1413 
 1414   bind(E);
 1415 }
 1416 
 1417 void MacroAssembler::call_VM(Register oop_result,
 1418                              address entry_point,
 1419                              Register arg_1,
 1420                              bool check_exceptions) {
 1421   Label C, E;
 1422   call(C, relocInfo::none);
 1423   jmp(E);
 1424 
 1425   bind(C);
 1426   pass_arg1(this, arg_1);
 1427   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1428   ret(0);
 1429 
 1430   bind(E);
 1431 }
 1432 
 1433 void MacroAssembler::call_VM(Register oop_result,
 1434                              address entry_point,
 1435                              Register arg_1,
 1436                              Register arg_2,
 1437                              bool check_exceptions) {
 1438   Label C, E;
 1439   call(C, relocInfo::none);
 1440   jmp(E);
 1441 
 1442   bind(C);
 1443 
 1444   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1445 
 1446   pass_arg2(this, arg_2);
 1447   pass_arg1(this, arg_1);
 1448   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1449   ret(0);
 1450 
 1451   bind(E);
 1452 }
 1453 
 1454 void MacroAssembler::call_VM(Register oop_result,
 1455                              address entry_point,
 1456                              Register arg_1,
 1457                              Register arg_2,
 1458                              Register arg_3,
 1459                              bool check_exceptions) {
 1460   Label C, E;
 1461   call(C, relocInfo::none);
 1462   jmp(E);
 1463 
 1464   bind(C);
 1465 
 1466   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1467   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1468   pass_arg3(this, arg_3);
 1469   pass_arg2(this, arg_2);
 1470   pass_arg1(this, arg_1);
 1471   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1472   ret(0);
 1473 
 1474   bind(E);
 1475 }
 1476 
 1477 void MacroAssembler::call_VM(Register oop_result,
 1478                              Register last_java_sp,
 1479                              address entry_point,
 1480                              int number_of_arguments,
 1481                              bool check_exceptions) {
 1482   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1483   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1484 }
 1485 
 1486 void MacroAssembler::call_VM(Register oop_result,
 1487                              Register last_java_sp,
 1488                              address entry_point,
 1489                              Register arg_1,
 1490                              bool check_exceptions) {
 1491   pass_arg1(this, arg_1);
 1492   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1493 }
 1494 
 1495 void MacroAssembler::call_VM(Register oop_result,
 1496                              Register last_java_sp,
 1497                              address entry_point,
 1498                              Register arg_1,
 1499                              Register arg_2,
 1500                              bool check_exceptions) {
 1501 
 1502   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1503   pass_arg2(this, arg_2);
 1504   pass_arg1(this, arg_1);
 1505   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1506 }
 1507 
 1508 void MacroAssembler::call_VM(Register oop_result,
 1509                              Register last_java_sp,
 1510                              address entry_point,
 1511                              Register arg_1,
 1512                              Register arg_2,
 1513                              Register arg_3,
 1514                              bool check_exceptions) {
 1515   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1516   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1517   pass_arg3(this, arg_3);
 1518   pass_arg2(this, arg_2);
 1519   pass_arg1(this, arg_1);
 1520   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1521 }
 1522 
 1523 void MacroAssembler::super_call_VM(Register oop_result,
 1524                                    Register last_java_sp,
 1525                                    address entry_point,
 1526                                    int number_of_arguments,
 1527                                    bool check_exceptions) {
 1528   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1529   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1530 }
 1531 
 1532 void MacroAssembler::super_call_VM(Register oop_result,
 1533                                    Register last_java_sp,
 1534                                    address entry_point,
 1535                                    Register arg_1,
 1536                                    bool check_exceptions) {
 1537   pass_arg1(this, arg_1);
 1538   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1539 }
 1540 
 1541 void MacroAssembler::super_call_VM(Register oop_result,
 1542                                    Register last_java_sp,
 1543                                    address entry_point,
 1544                                    Register arg_1,
 1545                                    Register arg_2,
 1546                                    bool check_exceptions) {
 1547 
 1548   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1549   pass_arg2(this, arg_2);
 1550   pass_arg1(this, arg_1);
 1551   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1552 }
 1553 
 1554 void MacroAssembler::super_call_VM(Register oop_result,
 1555                                    Register last_java_sp,
 1556                                    address entry_point,
 1557                                    Register arg_1,
 1558                                    Register arg_2,
 1559                                    Register arg_3,
 1560                                    bool check_exceptions) {
 1561   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1562   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1563   pass_arg3(this, arg_3);
 1564   pass_arg2(this, arg_2);
 1565   pass_arg1(this, arg_1);
 1566   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1567 }
 1568 
 1569 void MacroAssembler::call_VM_base(Register oop_result,
 1570                                   Register java_thread,
 1571                                   Register last_java_sp,
 1572                                   address  entry_point,
 1573                                   int      number_of_arguments,
 1574                                   bool     check_exceptions) {
 1575   // determine java_thread register
 1576   if (!java_thread->is_valid()) {
 1577 #ifdef _LP64
 1578     java_thread = r15_thread;
 1579 #else
 1580     java_thread = rdi;
 1581     get_thread(java_thread);
 1582 #endif // LP64
 1583   }
 1584   // determine last_java_sp register
 1585   if (!last_java_sp->is_valid()) {
 1586     last_java_sp = rsp;
 1587   }
 1588   // debugging support
 1589   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1590   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1591 #ifdef ASSERT
 1592   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1593   // r12 is the heapbase.
 1594   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1595 #endif // ASSERT
 1596 
 1597   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1598   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1599 
 1600   // push java thread (becomes first argument of C function)
 1601 
 1602   NOT_LP64(push(java_thread); number_of_arguments++);
 1603   LP64_ONLY(mov(c_rarg0, r15_thread));
 1604 
 1605   // set last Java frame before call
 1606   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1607 
 1608   // Only interpreter should have to set fp
 1609   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1610 
 1611   // do the call, remove parameters
 1612   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1613 
 1614   // restore the thread (cannot use the pushed argument since arguments
 1615   // may be overwritten by C code generated by an optimizing compiler);
 1616   // however can use the register value directly if it is callee saved.
 1617   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1618     // rdi & rsi (also r15) are callee saved -> nothing to do
 1619 #ifdef ASSERT
 1620     guarantee(java_thread != rax, "change this code");
 1621     push(rax);
 1622     { Label L;
 1623       get_thread(rax);
 1624       cmpptr(java_thread, rax);
 1625       jcc(Assembler::equal, L);
 1626       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1627       bind(L);
 1628     }
 1629     pop(rax);
 1630 #endif
 1631   } else {
 1632     get_thread(java_thread);
 1633   }
 1634   // reset last Java frame
 1635   // Only interpreter should have to clear fp
 1636   reset_last_Java_frame(java_thread, true);
 1637 
 1638    // C++ interp handles this in the interpreter
 1639   check_and_handle_popframe(java_thread);
 1640   check_and_handle_earlyret(java_thread);
 1641 
 1642   if (check_exceptions) {
 1643     // check for pending exceptions (java_thread is set upon return)
 1644     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1645 #ifndef _LP64
 1646     jump_cc(Assembler::notEqual,
 1647             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1648 #else
 1649     // This used to conditionally jump to forward_exception however it is
 1650     // possible if we relocate that the branch will not reach. So we must jump
 1651     // around so we can always reach
 1652 
 1653     Label ok;
 1654     jcc(Assembler::equal, ok);
 1655     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1656     bind(ok);
 1657 #endif // LP64
 1658   }
 1659 
 1660   // get oop result if there is one and reset the value in the thread
 1661   if (oop_result->is_valid()) {
 1662     get_vm_result(oop_result, java_thread);
 1663   }
 1664 }
 1665 
 1666 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1667 
 1668   // Calculate the value for last_Java_sp
 1669   // somewhat subtle. call_VM does an intermediate call
 1670   // which places a return address on the stack just under the
 1671   // stack pointer as the user finished with it. This allows
 1672   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1673   // On 32bit we then have to push additional args on the stack to accomplish
 1674   // the actual requested call. On 64bit call_VM only can use register args
 1675   // so the only extra space is the return address that call_VM created.
 1676   // This hopefully explains the calculations here.
 1677 
 1678 #ifdef _LP64
 1679   // We've pushed one address, correct last_Java_sp
 1680   lea(rax, Address(rsp, wordSize));
 1681 #else
 1682   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1683 #endif // LP64
 1684 
 1685   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1686 
 1687 }
 1688 
 1689 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1690 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1691   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1692 }
 1693 
 1694 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1695   call_VM_leaf_base(entry_point, number_of_arguments);
 1696 }
 1697 
 1698 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1699   pass_arg0(this, arg_0);
 1700   call_VM_leaf(entry_point, 1);
 1701 }
 1702 
 1703 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1704 
 1705   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1706   pass_arg1(this, arg_1);
 1707   pass_arg0(this, arg_0);
 1708   call_VM_leaf(entry_point, 2);
 1709 }
 1710 
 1711 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1712   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1713   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1714   pass_arg2(this, arg_2);
 1715   pass_arg1(this, arg_1);
 1716   pass_arg0(this, arg_0);
 1717   call_VM_leaf(entry_point, 3);
 1718 }
 1719 
 1720 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1721   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1722   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1723   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1724   pass_arg3(this, arg_3);
 1725   pass_arg2(this, arg_2);
 1726   pass_arg1(this, arg_1);
 1727   pass_arg0(this, arg_0);
 1728   call_VM_leaf(entry_point, 3);
 1729 }
 1730 
 1731 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1732   pass_arg0(this, arg_0);
 1733   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1734 }
 1735 
 1736 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1737   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1738   pass_arg1(this, arg_1);
 1739   pass_arg0(this, arg_0);
 1740   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1741 }
 1742 
 1743 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1744   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1745   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1746   pass_arg2(this, arg_2);
 1747   pass_arg1(this, arg_1);
 1748   pass_arg0(this, arg_0);
 1749   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1750 }
 1751 
 1752 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1753   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1754   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1755   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1756   pass_arg3(this, arg_3);
 1757   pass_arg2(this, arg_2);
 1758   pass_arg1(this, arg_1);
 1759   pass_arg0(this, arg_0);
 1760   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1761 }
 1762 
 1763 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1764   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1765   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1766   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1767 }
 1768 
 1769 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1770   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1771   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1772 }
 1773 
 1774 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1775 }
 1776 
 1777 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1778 }
 1779 
 1780 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1781   assert(rscratch != noreg || always_reachable(src1), "missing");
 1782 
 1783   if (reachable(src1)) {
 1784     cmpl(as_Address(src1), imm);
 1785   } else {
 1786     lea(rscratch, src1);
 1787     cmpl(Address(rscratch, 0), imm);
 1788   }
 1789 }
 1790 
 1791 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1792   assert(!src2.is_lval(), "use cmpptr");
 1793   assert(rscratch != noreg || always_reachable(src2), "missing");
 1794 
 1795   if (reachable(src2)) {
 1796     cmpl(src1, as_Address(src2));
 1797   } else {
 1798     lea(rscratch, src2);
 1799     cmpl(src1, Address(rscratch, 0));
 1800   }
 1801 }
 1802 
 1803 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1804   Assembler::cmpl(src1, imm);
 1805 }
 1806 
 1807 void MacroAssembler::cmp32(Register src1, Address src2) {
 1808   Assembler::cmpl(src1, src2);
 1809 }
 1810 
 1811 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1812   ucomisd(opr1, opr2);
 1813 
 1814   Label L;
 1815   if (unordered_is_less) {
 1816     movl(dst, -1);
 1817     jcc(Assembler::parity, L);
 1818     jcc(Assembler::below , L);
 1819     movl(dst, 0);
 1820     jcc(Assembler::equal , L);
 1821     increment(dst);
 1822   } else { // unordered is greater
 1823     movl(dst, 1);
 1824     jcc(Assembler::parity, L);
 1825     jcc(Assembler::above , L);
 1826     movl(dst, 0);
 1827     jcc(Assembler::equal , L);
 1828     decrementl(dst);
 1829   }
 1830   bind(L);
 1831 }
 1832 
 1833 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1834   ucomiss(opr1, opr2);
 1835 
 1836   Label L;
 1837   if (unordered_is_less) {
 1838     movl(dst, -1);
 1839     jcc(Assembler::parity, L);
 1840     jcc(Assembler::below , L);
 1841     movl(dst, 0);
 1842     jcc(Assembler::equal , L);
 1843     increment(dst);
 1844   } else { // unordered is greater
 1845     movl(dst, 1);
 1846     jcc(Assembler::parity, L);
 1847     jcc(Assembler::above , L);
 1848     movl(dst, 0);
 1849     jcc(Assembler::equal , L);
 1850     decrementl(dst);
 1851   }
 1852   bind(L);
 1853 }
 1854 
 1855 
 1856 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1857   assert(rscratch != noreg || always_reachable(src1), "missing");
 1858 
 1859   if (reachable(src1)) {
 1860     cmpb(as_Address(src1), imm);
 1861   } else {
 1862     lea(rscratch, src1);
 1863     cmpb(Address(rscratch, 0), imm);
 1864   }
 1865 }
 1866 
 1867 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1868 #ifdef _LP64
 1869   assert(rscratch != noreg || always_reachable(src2), "missing");
 1870 
 1871   if (src2.is_lval()) {
 1872     movptr(rscratch, src2);
 1873     Assembler::cmpq(src1, rscratch);
 1874   } else if (reachable(src2)) {
 1875     cmpq(src1, as_Address(src2));
 1876   } else {
 1877     lea(rscratch, src2);
 1878     Assembler::cmpq(src1, Address(rscratch, 0));
 1879   }
 1880 #else
 1881   assert(rscratch == noreg, "not needed");
 1882   if (src2.is_lval()) {
 1883     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1884   } else {
 1885     cmpl(src1, as_Address(src2));
 1886   }
 1887 #endif // _LP64
 1888 }
 1889 
 1890 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1891   assert(src2.is_lval(), "not a mem-mem compare");
 1892 #ifdef _LP64
 1893   // moves src2's literal address
 1894   movptr(rscratch, src2);
 1895   Assembler::cmpq(src1, rscratch);
 1896 #else
 1897   assert(rscratch == noreg, "not needed");
 1898   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1899 #endif // _LP64
 1900 }
 1901 
 1902 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1903   cmpptr(src1, src2);
 1904 }
 1905 
 1906 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1907   cmpptr(src1, src2);
 1908 }
 1909 
 1910 #ifdef _LP64
 1911 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1912   movoop(rscratch, src2);
 1913   cmpptr(src1, rscratch);
 1914 }
 1915 #endif
 1916 
 1917 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1918   assert(rscratch != noreg || always_reachable(adr), "missing");
 1919 
 1920   if (reachable(adr)) {
 1921     lock();
 1922     cmpxchgptr(reg, as_Address(adr));
 1923   } else {
 1924     lea(rscratch, adr);
 1925     lock();
 1926     cmpxchgptr(reg, Address(rscratch, 0));
 1927   }
 1928 }
 1929 
 1930 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1931   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1932 }
 1933 
 1934 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1935   assert(rscratch != noreg || always_reachable(src), "missing");
 1936 
 1937   if (reachable(src)) {
 1938     Assembler::comisd(dst, as_Address(src));
 1939   } else {
 1940     lea(rscratch, src);
 1941     Assembler::comisd(dst, Address(rscratch, 0));
 1942   }
 1943 }
 1944 
 1945 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1946   assert(rscratch != noreg || always_reachable(src), "missing");
 1947 
 1948   if (reachable(src)) {
 1949     Assembler::comiss(dst, as_Address(src));
 1950   } else {
 1951     lea(rscratch, src);
 1952     Assembler::comiss(dst, Address(rscratch, 0));
 1953   }
 1954 }
 1955 
 1956 
 1957 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1958   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1959 
 1960   Condition negated_cond = negate_condition(cond);
 1961   Label L;
 1962   jcc(negated_cond, L);
 1963   pushf(); // Preserve flags
 1964   atomic_incl(counter_addr, rscratch);
 1965   popf();
 1966   bind(L);
 1967 }
 1968 
 1969 int MacroAssembler::corrected_idivl(Register reg) {
 1970   // Full implementation of Java idiv and irem; checks for
 1971   // special case as described in JVM spec., p.243 & p.271.
 1972   // The function returns the (pc) offset of the idivl
 1973   // instruction - may be needed for implicit exceptions.
 1974   //
 1975   //         normal case                           special case
 1976   //
 1977   // input : rax,: dividend                         min_int
 1978   //         reg: divisor   (may not be rax,/rdx)   -1
 1979   //
 1980   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1981   //         rdx: remainder (= rax, irem reg)       0
 1982   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1983   const int min_int = 0x80000000;
 1984   Label normal_case, special_case;
 1985 
 1986   // check for special case
 1987   cmpl(rax, min_int);
 1988   jcc(Assembler::notEqual, normal_case);
 1989   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1990   cmpl(reg, -1);
 1991   jcc(Assembler::equal, special_case);
 1992 
 1993   // handle normal case
 1994   bind(normal_case);
 1995   cdql();
 1996   int idivl_offset = offset();
 1997   idivl(reg);
 1998 
 1999   // normal and special case exit
 2000   bind(special_case);
 2001 
 2002   return idivl_offset;
 2003 }
 2004 
 2005 
 2006 
 2007 void MacroAssembler::decrementl(Register reg, int value) {
 2008   if (value == min_jint) {subl(reg, value) ; return; }
 2009   if (value <  0) { incrementl(reg, -value); return; }
 2010   if (value == 0) {                        ; return; }
 2011   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2012   /* else */      { subl(reg, value)       ; return; }
 2013 }
 2014 
 2015 void MacroAssembler::decrementl(Address dst, int value) {
 2016   if (value == min_jint) {subl(dst, value) ; return; }
 2017   if (value <  0) { incrementl(dst, -value); return; }
 2018   if (value == 0) {                        ; return; }
 2019   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2020   /* else */      { subl(dst, value)       ; return; }
 2021 }
 2022 
 2023 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2024   assert(shift_value > 0, "illegal shift value");
 2025   Label _is_positive;
 2026   testl (reg, reg);
 2027   jcc (Assembler::positive, _is_positive);
 2028   int offset = (1 << shift_value) - 1 ;
 2029 
 2030   if (offset == 1) {
 2031     incrementl(reg);
 2032   } else {
 2033     addl(reg, offset);
 2034   }
 2035 
 2036   bind (_is_positive);
 2037   sarl(reg, shift_value);
 2038 }
 2039 
 2040 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2041   assert(rscratch != noreg || always_reachable(src), "missing");
 2042 
 2043   if (reachable(src)) {
 2044     Assembler::divsd(dst, as_Address(src));
 2045   } else {
 2046     lea(rscratch, src);
 2047     Assembler::divsd(dst, Address(rscratch, 0));
 2048   }
 2049 }
 2050 
 2051 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2052   assert(rscratch != noreg || always_reachable(src), "missing");
 2053 
 2054   if (reachable(src)) {
 2055     Assembler::divss(dst, as_Address(src));
 2056   } else {
 2057     lea(rscratch, src);
 2058     Assembler::divss(dst, Address(rscratch, 0));
 2059   }
 2060 }
 2061 
 2062 void MacroAssembler::enter() {
 2063   push(rbp);
 2064   mov(rbp, rsp);
 2065 }
 2066 
 2067 void MacroAssembler::post_call_nop() {
 2068   if (!Continuations::enabled()) {
 2069     return;
 2070   }
 2071   InstructionMark im(this);
 2072   relocate(post_call_nop_Relocation::spec());
 2073   InlineSkippedInstructionsCounter skipCounter(this);
 2074   emit_int8((uint8_t)0x0f);
 2075   emit_int8((uint8_t)0x1f);
 2076   emit_int8((uint8_t)0x84);
 2077   emit_int8((uint8_t)0x00);
 2078   emit_int32(0x00);
 2079 }
 2080 
 2081 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2082 void MacroAssembler::fat_nop() {
 2083   if (UseAddressNop) {
 2084     addr_nop_5();
 2085   } else {
 2086     emit_int8((uint8_t)0x26); // es:
 2087     emit_int8((uint8_t)0x2e); // cs:
 2088     emit_int8((uint8_t)0x64); // fs:
 2089     emit_int8((uint8_t)0x65); // gs:
 2090     emit_int8((uint8_t)0x90);
 2091   }
 2092 }
 2093 
 2094 #ifndef _LP64
 2095 void MacroAssembler::fcmp(Register tmp) {
 2096   fcmp(tmp, 1, true, true);
 2097 }
 2098 
 2099 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2100   assert(!pop_right || pop_left, "usage error");
 2101   if (VM_Version::supports_cmov()) {
 2102     assert(tmp == noreg, "unneeded temp");
 2103     if (pop_left) {
 2104       fucomip(index);
 2105     } else {
 2106       fucomi(index);
 2107     }
 2108     if (pop_right) {
 2109       fpop();
 2110     }
 2111   } else {
 2112     assert(tmp != noreg, "need temp");
 2113     if (pop_left) {
 2114       if (pop_right) {
 2115         fcompp();
 2116       } else {
 2117         fcomp(index);
 2118       }
 2119     } else {
 2120       fcom(index);
 2121     }
 2122     // convert FPU condition into eflags condition via rax,
 2123     save_rax(tmp);
 2124     fwait(); fnstsw_ax();
 2125     sahf();
 2126     restore_rax(tmp);
 2127   }
 2128   // condition codes set as follows:
 2129   //
 2130   // CF (corresponds to C0) if x < y
 2131   // PF (corresponds to C2) if unordered
 2132   // ZF (corresponds to C3) if x = y
 2133 }
 2134 
 2135 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2136   fcmp2int(dst, unordered_is_less, 1, true, true);
 2137 }
 2138 
 2139 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2140   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2141   Label L;
 2142   if (unordered_is_less) {
 2143     movl(dst, -1);
 2144     jcc(Assembler::parity, L);
 2145     jcc(Assembler::below , L);
 2146     movl(dst, 0);
 2147     jcc(Assembler::equal , L);
 2148     increment(dst);
 2149   } else { // unordered is greater
 2150     movl(dst, 1);
 2151     jcc(Assembler::parity, L);
 2152     jcc(Assembler::above , L);
 2153     movl(dst, 0);
 2154     jcc(Assembler::equal , L);
 2155     decrementl(dst);
 2156   }
 2157   bind(L);
 2158 }
 2159 
 2160 void MacroAssembler::fld_d(AddressLiteral src) {
 2161   fld_d(as_Address(src));
 2162 }
 2163 
 2164 void MacroAssembler::fld_s(AddressLiteral src) {
 2165   fld_s(as_Address(src));
 2166 }
 2167 
 2168 void MacroAssembler::fldcw(AddressLiteral src) {
 2169   fldcw(as_Address(src));
 2170 }
 2171 
 2172 void MacroAssembler::fpop() {
 2173   ffree();
 2174   fincstp();
 2175 }
 2176 
 2177 void MacroAssembler::fremr(Register tmp) {
 2178   save_rax(tmp);
 2179   { Label L;
 2180     bind(L);
 2181     fprem();
 2182     fwait(); fnstsw_ax();
 2183     sahf();
 2184     jcc(Assembler::parity, L);
 2185   }
 2186   restore_rax(tmp);
 2187   // Result is in ST0.
 2188   // Note: fxch & fpop to get rid of ST1
 2189   // (otherwise FPU stack could overflow eventually)
 2190   fxch(1);
 2191   fpop();
 2192 }
 2193 
 2194 void MacroAssembler::empty_FPU_stack() {
 2195   if (VM_Version::supports_mmx()) {
 2196     emms();
 2197   } else {
 2198     for (int i = 8; i-- > 0; ) ffree(i);
 2199   }
 2200 }
 2201 #endif // !LP64
 2202 
 2203 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2204   assert(rscratch != noreg || always_reachable(src), "missing");
 2205   if (reachable(src)) {
 2206     Assembler::mulpd(dst, as_Address(src));
 2207   } else {
 2208     lea(rscratch, src);
 2209     Assembler::mulpd(dst, Address(rscratch, 0));
 2210   }
 2211 }
 2212 
 2213 void MacroAssembler::load_float(Address src) {
 2214 #ifdef _LP64
 2215   movflt(xmm0, src);
 2216 #else
 2217   if (UseSSE >= 1) {
 2218     movflt(xmm0, src);
 2219   } else {
 2220     fld_s(src);
 2221   }
 2222 #endif // LP64
 2223 }
 2224 
 2225 void MacroAssembler::store_float(Address dst) {
 2226 #ifdef _LP64
 2227   movflt(dst, xmm0);
 2228 #else
 2229   if (UseSSE >= 1) {
 2230     movflt(dst, xmm0);
 2231   } else {
 2232     fstp_s(dst);
 2233   }
 2234 #endif // LP64
 2235 }
 2236 
 2237 void MacroAssembler::load_double(Address src) {
 2238 #ifdef _LP64
 2239   movdbl(xmm0, src);
 2240 #else
 2241   if (UseSSE >= 2) {
 2242     movdbl(xmm0, src);
 2243   } else {
 2244     fld_d(src);
 2245   }
 2246 #endif // LP64
 2247 }
 2248 
 2249 void MacroAssembler::store_double(Address dst) {
 2250 #ifdef _LP64
 2251   movdbl(dst, xmm0);
 2252 #else
 2253   if (UseSSE >= 2) {
 2254     movdbl(dst, xmm0);
 2255   } else {
 2256     fstp_d(dst);
 2257   }
 2258 #endif // LP64
 2259 }
 2260 
 2261 // dst = c = a * b + c
 2262 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2263   Assembler::vfmadd231sd(c, a, b);
 2264   if (dst != c) {
 2265     movdbl(dst, c);
 2266   }
 2267 }
 2268 
 2269 // dst = c = a * b + c
 2270 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2271   Assembler::vfmadd231ss(c, a, b);
 2272   if (dst != c) {
 2273     movflt(dst, c);
 2274   }
 2275 }
 2276 
 2277 // dst = c = a * b + c
 2278 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2279   Assembler::vfmadd231pd(c, a, b, vector_len);
 2280   if (dst != c) {
 2281     vmovdqu(dst, c);
 2282   }
 2283 }
 2284 
 2285 // dst = c = a * b + c
 2286 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2287   Assembler::vfmadd231ps(c, a, b, vector_len);
 2288   if (dst != c) {
 2289     vmovdqu(dst, c);
 2290   }
 2291 }
 2292 
 2293 // dst = c = a * b + c
 2294 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2295   Assembler::vfmadd231pd(c, a, b, vector_len);
 2296   if (dst != c) {
 2297     vmovdqu(dst, c);
 2298   }
 2299 }
 2300 
 2301 // dst = c = a * b + c
 2302 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2303   Assembler::vfmadd231ps(c, a, b, vector_len);
 2304   if (dst != c) {
 2305     vmovdqu(dst, c);
 2306   }
 2307 }
 2308 
 2309 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2310   assert(rscratch != noreg || always_reachable(dst), "missing");
 2311 
 2312   if (reachable(dst)) {
 2313     incrementl(as_Address(dst));
 2314   } else {
 2315     lea(rscratch, dst);
 2316     incrementl(Address(rscratch, 0));
 2317   }
 2318 }
 2319 
 2320 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2321   incrementl(as_Address(dst, rscratch));
 2322 }
 2323 
 2324 void MacroAssembler::incrementl(Register reg, int value) {
 2325   if (value == min_jint) {addl(reg, value) ; return; }
 2326   if (value <  0) { decrementl(reg, -value); return; }
 2327   if (value == 0) {                        ; return; }
 2328   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2329   /* else */      { addl(reg, value)       ; return; }
 2330 }
 2331 
 2332 void MacroAssembler::incrementl(Address dst, int value) {
 2333   if (value == min_jint) {addl(dst, value) ; return; }
 2334   if (value <  0) { decrementl(dst, -value); return; }
 2335   if (value == 0) {                        ; return; }
 2336   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2337   /* else */      { addl(dst, value)       ; return; }
 2338 }
 2339 
 2340 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2341   assert(rscratch != noreg || always_reachable(dst), "missing");
 2342   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump");
 2343   if (reachable(dst)) {
 2344     jmp_literal(dst.target(), dst.rspec());
 2345   } else {
 2346     lea(rscratch, dst);
 2347     jmp(rscratch);
 2348   }
 2349 }
 2350 
 2351 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2352   assert(rscratch != noreg || always_reachable(dst), "missing");
 2353   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc");
 2354   if (reachable(dst)) {
 2355     InstructionMark im(this);
 2356     relocate(dst.reloc());
 2357     const int short_size = 2;
 2358     const int long_size = 6;
 2359     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2360     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2361       // 0111 tttn #8-bit disp
 2362       emit_int8(0x70 | cc);
 2363       emit_int8((offs - short_size) & 0xFF);
 2364     } else {
 2365       // 0000 1111 1000 tttn #32-bit disp
 2366       emit_int8(0x0F);
 2367       emit_int8((unsigned char)(0x80 | cc));
 2368       emit_int32(offs - long_size);
 2369     }
 2370   } else {
 2371 #ifdef ASSERT
 2372     warning("reversing conditional branch");
 2373 #endif /* ASSERT */
 2374     Label skip;
 2375     jccb(reverse[cc], skip);
 2376     lea(rscratch, dst);
 2377     Assembler::jmp(rscratch);
 2378     bind(skip);
 2379   }
 2380 }
 2381 
 2382 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2383   assert(rscratch != noreg || always_reachable(src), "missing");
 2384 
 2385   if (reachable(src)) {
 2386     Assembler::ldmxcsr(as_Address(src));
 2387   } else {
 2388     lea(rscratch, src);
 2389     Assembler::ldmxcsr(Address(rscratch, 0));
 2390   }
 2391 }
 2392 
 2393 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2394   int off;
 2395   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2396     off = offset();
 2397     movsbl(dst, src); // movsxb
 2398   } else {
 2399     off = load_unsigned_byte(dst, src);
 2400     shll(dst, 24);
 2401     sarl(dst, 24);
 2402   }
 2403   return off;
 2404 }
 2405 
 2406 // Note: load_signed_short used to be called load_signed_word.
 2407 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2408 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2409 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2410 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2411   int off;
 2412   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2413     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2414     // version but this is what 64bit has always done. This seems to imply
 2415     // that users are only using 32bits worth.
 2416     off = offset();
 2417     movswl(dst, src); // movsxw
 2418   } else {
 2419     off = load_unsigned_short(dst, src);
 2420     shll(dst, 16);
 2421     sarl(dst, 16);
 2422   }
 2423   return off;
 2424 }
 2425 
 2426 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2427   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2428   // and "3.9 Partial Register Penalties", p. 22).
 2429   int off;
 2430   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2431     off = offset();
 2432     movzbl(dst, src); // movzxb
 2433   } else {
 2434     xorl(dst, dst);
 2435     off = offset();
 2436     movb(dst, src);
 2437   }
 2438   return off;
 2439 }
 2440 
 2441 // Note: load_unsigned_short used to be called load_unsigned_word.
 2442 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2443   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2444   // and "3.9 Partial Register Penalties", p. 22).
 2445   int off;
 2446   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2447     off = offset();
 2448     movzwl(dst, src); // movzxw
 2449   } else {
 2450     xorl(dst, dst);
 2451     off = offset();
 2452     movw(dst, src);
 2453   }
 2454   return off;
 2455 }
 2456 
 2457 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2458   switch (size_in_bytes) {
 2459 #ifndef _LP64
 2460   case  8:
 2461     assert(dst2 != noreg, "second dest register required");
 2462     movl(dst,  src);
 2463     movl(dst2, src.plus_disp(BytesPerInt));
 2464     break;
 2465 #else
 2466   case  8:  movq(dst, src); break;
 2467 #endif
 2468   case  4:  movl(dst, src); break;
 2469   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2470   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2471   default:  ShouldNotReachHere();
 2472   }
 2473 }
 2474 
 2475 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2476   switch (size_in_bytes) {
 2477 #ifndef _LP64
 2478   case  8:
 2479     assert(src2 != noreg, "second source register required");
 2480     movl(dst,                        src);
 2481     movl(dst.plus_disp(BytesPerInt), src2);
 2482     break;
 2483 #else
 2484   case  8:  movq(dst, src); break;
 2485 #endif
 2486   case  4:  movl(dst, src); break;
 2487   case  2:  movw(dst, src); break;
 2488   case  1:  movb(dst, src); break;
 2489   default:  ShouldNotReachHere();
 2490   }
 2491 }
 2492 
 2493 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2494   assert(rscratch != noreg || always_reachable(dst), "missing");
 2495 
 2496   if (reachable(dst)) {
 2497     movl(as_Address(dst), src);
 2498   } else {
 2499     lea(rscratch, dst);
 2500     movl(Address(rscratch, 0), src);
 2501   }
 2502 }
 2503 
 2504 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2505   if (reachable(src)) {
 2506     movl(dst, as_Address(src));
 2507   } else {
 2508     lea(dst, src);
 2509     movl(dst, Address(dst, 0));
 2510   }
 2511 }
 2512 
 2513 // C++ bool manipulation
 2514 
 2515 void MacroAssembler::movbool(Register dst, Address src) {
 2516   if(sizeof(bool) == 1)
 2517     movb(dst, src);
 2518   else if(sizeof(bool) == 2)
 2519     movw(dst, src);
 2520   else if(sizeof(bool) == 4)
 2521     movl(dst, src);
 2522   else
 2523     // unsupported
 2524     ShouldNotReachHere();
 2525 }
 2526 
 2527 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2528   if(sizeof(bool) == 1)
 2529     movb(dst, (int) boolconst);
 2530   else if(sizeof(bool) == 2)
 2531     movw(dst, (int) boolconst);
 2532   else if(sizeof(bool) == 4)
 2533     movl(dst, (int) boolconst);
 2534   else
 2535     // unsupported
 2536     ShouldNotReachHere();
 2537 }
 2538 
 2539 void MacroAssembler::movbool(Address dst, Register src) {
 2540   if(sizeof(bool) == 1)
 2541     movb(dst, src);
 2542   else if(sizeof(bool) == 2)
 2543     movw(dst, src);
 2544   else if(sizeof(bool) == 4)
 2545     movl(dst, src);
 2546   else
 2547     // unsupported
 2548     ShouldNotReachHere();
 2549 }
 2550 
 2551 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2552   assert(rscratch != noreg || always_reachable(src), "missing");
 2553 
 2554   if (reachable(src)) {
 2555     movdl(dst, as_Address(src));
 2556   } else {
 2557     lea(rscratch, src);
 2558     movdl(dst, Address(rscratch, 0));
 2559   }
 2560 }
 2561 
 2562 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2563   assert(rscratch != noreg || always_reachable(src), "missing");
 2564 
 2565   if (reachable(src)) {
 2566     movq(dst, as_Address(src));
 2567   } else {
 2568     lea(rscratch, src);
 2569     movq(dst, Address(rscratch, 0));
 2570   }
 2571 }
 2572 
 2573 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2574   assert(rscratch != noreg || always_reachable(src), "missing");
 2575 
 2576   if (reachable(src)) {
 2577     if (UseXmmLoadAndClearUpper) {
 2578       movsd (dst, as_Address(src));
 2579     } else {
 2580       movlpd(dst, as_Address(src));
 2581     }
 2582   } else {
 2583     lea(rscratch, src);
 2584     if (UseXmmLoadAndClearUpper) {
 2585       movsd (dst, Address(rscratch, 0));
 2586     } else {
 2587       movlpd(dst, Address(rscratch, 0));
 2588     }
 2589   }
 2590 }
 2591 
 2592 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2593   assert(rscratch != noreg || always_reachable(src), "missing");
 2594 
 2595   if (reachable(src)) {
 2596     movss(dst, as_Address(src));
 2597   } else {
 2598     lea(rscratch, src);
 2599     movss(dst, Address(rscratch, 0));
 2600   }
 2601 }
 2602 
 2603 void MacroAssembler::movptr(Register dst, Register src) {
 2604   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2605 }
 2606 
 2607 void MacroAssembler::movptr(Register dst, Address src) {
 2608   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2609 }
 2610 
 2611 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2612 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2613 #ifdef _LP64
 2614   if (is_uimm32(src)) {
 2615     movl(dst, checked_cast<uint32_t>(src));
 2616   } else if (is_simm32(src)) {
 2617     movq(dst, checked_cast<int32_t>(src));
 2618   } else {
 2619     mov64(dst, src);
 2620   }
 2621 #else
 2622   movl(dst, src);
 2623 #endif
 2624 }
 2625 
 2626 void MacroAssembler::movptr(Address dst, Register src) {
 2627   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2628 }
 2629 
 2630 void MacroAssembler::movptr(Address dst, int32_t src) {
 2631   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2632 }
 2633 
 2634 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2635   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2636   Assembler::movdqu(dst, src);
 2637 }
 2638 
 2639 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2640   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2641   Assembler::movdqu(dst, src);
 2642 }
 2643 
 2644 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2645   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2646   Assembler::movdqu(dst, src);
 2647 }
 2648 
 2649 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2650   assert(rscratch != noreg || always_reachable(src), "missing");
 2651 
 2652   if (reachable(src)) {
 2653     movdqu(dst, as_Address(src));
 2654   } else {
 2655     lea(rscratch, src);
 2656     movdqu(dst, Address(rscratch, 0));
 2657   }
 2658 }
 2659 
 2660 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2661   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2662   Assembler::vmovdqu(dst, src);
 2663 }
 2664 
 2665 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2666   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2667   Assembler::vmovdqu(dst, src);
 2668 }
 2669 
 2670 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2671   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2672   Assembler::vmovdqu(dst, src);
 2673 }
 2674 
 2675 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2676   assert(rscratch != noreg || always_reachable(src), "missing");
 2677 
 2678   if (reachable(src)) {
 2679     vmovdqu(dst, as_Address(src));
 2680   }
 2681   else {
 2682     lea(rscratch, src);
 2683     vmovdqu(dst, Address(rscratch, 0));
 2684   }
 2685 }
 2686 
 2687 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2688   assert(rscratch != noreg || always_reachable(src), "missing");
 2689 
 2690   if (vector_len == AVX_512bit) {
 2691     evmovdquq(dst, src, AVX_512bit, rscratch);
 2692   } else if (vector_len == AVX_256bit) {
 2693     vmovdqu(dst, src, rscratch);
 2694   } else {
 2695     movdqu(dst, src, rscratch);
 2696   }
 2697 }
 2698 
 2699 void MacroAssembler::kmov(KRegister dst, Address src) {
 2700   if (VM_Version::supports_avx512bw()) {
 2701     kmovql(dst, src);
 2702   } else {
 2703     assert(VM_Version::supports_evex(), "");
 2704     kmovwl(dst, src);
 2705   }
 2706 }
 2707 
 2708 void MacroAssembler::kmov(Address dst, KRegister src) {
 2709   if (VM_Version::supports_avx512bw()) {
 2710     kmovql(dst, src);
 2711   } else {
 2712     assert(VM_Version::supports_evex(), "");
 2713     kmovwl(dst, src);
 2714   }
 2715 }
 2716 
 2717 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2718   if (VM_Version::supports_avx512bw()) {
 2719     kmovql(dst, src);
 2720   } else {
 2721     assert(VM_Version::supports_evex(), "");
 2722     kmovwl(dst, src);
 2723   }
 2724 }
 2725 
 2726 void MacroAssembler::kmov(Register dst, KRegister src) {
 2727   if (VM_Version::supports_avx512bw()) {
 2728     kmovql(dst, src);
 2729   } else {
 2730     assert(VM_Version::supports_evex(), "");
 2731     kmovwl(dst, src);
 2732   }
 2733 }
 2734 
 2735 void MacroAssembler::kmov(KRegister dst, Register src) {
 2736   if (VM_Version::supports_avx512bw()) {
 2737     kmovql(dst, src);
 2738   } else {
 2739     assert(VM_Version::supports_evex(), "");
 2740     kmovwl(dst, src);
 2741   }
 2742 }
 2743 
 2744 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2745   assert(rscratch != noreg || always_reachable(src), "missing");
 2746 
 2747   if (reachable(src)) {
 2748     kmovql(dst, as_Address(src));
 2749   } else {
 2750     lea(rscratch, src);
 2751     kmovql(dst, Address(rscratch, 0));
 2752   }
 2753 }
 2754 
 2755 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2756   assert(rscratch != noreg || always_reachable(src), "missing");
 2757 
 2758   if (reachable(src)) {
 2759     kmovwl(dst, as_Address(src));
 2760   } else {
 2761     lea(rscratch, src);
 2762     kmovwl(dst, Address(rscratch, 0));
 2763   }
 2764 }
 2765 
 2766 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2767                                int vector_len, Register rscratch) {
 2768   assert(rscratch != noreg || always_reachable(src), "missing");
 2769 
 2770   if (reachable(src)) {
 2771     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2772   } else {
 2773     lea(rscratch, src);
 2774     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2775   }
 2776 }
 2777 
 2778 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2779                                int vector_len, Register rscratch) {
 2780   assert(rscratch != noreg || always_reachable(src), "missing");
 2781 
 2782   if (reachable(src)) {
 2783     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2784   } else {
 2785     lea(rscratch, src);
 2786     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2787   }
 2788 }
 2789 
 2790 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2791   assert(rscratch != noreg || always_reachable(src), "missing");
 2792 
 2793   if (reachable(src)) {
 2794     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2795   } else {
 2796     lea(rscratch, src);
 2797     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2798   }
 2799 }
 2800 
 2801 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2802   assert(rscratch != noreg || always_reachable(src), "missing");
 2803 
 2804   if (reachable(src)) {
 2805     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2806   } else {
 2807     lea(rscratch, src);
 2808     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2809   }
 2810 }
 2811 
 2812 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2813   assert(rscratch != noreg || always_reachable(src), "missing");
 2814 
 2815   if (reachable(src)) {
 2816     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2817   } else {
 2818     lea(rscratch, src);
 2819     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2820   }
 2821 }
 2822 
 2823 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2824   assert(rscratch != noreg || always_reachable(src), "missing");
 2825 
 2826   if (reachable(src)) {
 2827     Assembler::movdqa(dst, as_Address(src));
 2828   } else {
 2829     lea(rscratch, src);
 2830     Assembler::movdqa(dst, Address(rscratch, 0));
 2831   }
 2832 }
 2833 
 2834 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2835   assert(rscratch != noreg || always_reachable(src), "missing");
 2836 
 2837   if (reachable(src)) {
 2838     Assembler::movsd(dst, as_Address(src));
 2839   } else {
 2840     lea(rscratch, src);
 2841     Assembler::movsd(dst, Address(rscratch, 0));
 2842   }
 2843 }
 2844 
 2845 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2846   assert(rscratch != noreg || always_reachable(src), "missing");
 2847 
 2848   if (reachable(src)) {
 2849     Assembler::movss(dst, as_Address(src));
 2850   } else {
 2851     lea(rscratch, src);
 2852     Assembler::movss(dst, Address(rscratch, 0));
 2853   }
 2854 }
 2855 
 2856 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2857   assert(rscratch != noreg || always_reachable(src), "missing");
 2858 
 2859   if (reachable(src)) {
 2860     Assembler::movddup(dst, as_Address(src));
 2861   } else {
 2862     lea(rscratch, src);
 2863     Assembler::movddup(dst, Address(rscratch, 0));
 2864   }
 2865 }
 2866 
 2867 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2868   assert(rscratch != noreg || always_reachable(src), "missing");
 2869 
 2870   if (reachable(src)) {
 2871     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2872   } else {
 2873     lea(rscratch, src);
 2874     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2875   }
 2876 }
 2877 
 2878 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2879   assert(rscratch != noreg || always_reachable(src), "missing");
 2880 
 2881   if (reachable(src)) {
 2882     Assembler::mulsd(dst, as_Address(src));
 2883   } else {
 2884     lea(rscratch, src);
 2885     Assembler::mulsd(dst, Address(rscratch, 0));
 2886   }
 2887 }
 2888 
 2889 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2890   assert(rscratch != noreg || always_reachable(src), "missing");
 2891 
 2892   if (reachable(src)) {
 2893     Assembler::mulss(dst, as_Address(src));
 2894   } else {
 2895     lea(rscratch, src);
 2896     Assembler::mulss(dst, Address(rscratch, 0));
 2897   }
 2898 }
 2899 
 2900 void MacroAssembler::null_check(Register reg, int offset) {
 2901   if (needs_explicit_null_check(offset)) {
 2902     // provoke OS null exception if reg is null by
 2903     // accessing M[reg] w/o changing any (non-CC) registers
 2904     // NOTE: cmpl is plenty here to provoke a segv
 2905     cmpptr(rax, Address(reg, 0));
 2906     // Note: should probably use testl(rax, Address(reg, 0));
 2907     //       may be shorter code (however, this version of
 2908     //       testl needs to be implemented first)
 2909   } else {
 2910     // nothing to do, (later) access of M[reg + offset]
 2911     // will provoke OS null exception if reg is null
 2912   }
 2913 }
 2914 
 2915 void MacroAssembler::os_breakpoint() {
 2916   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2917   // (e.g., MSVC can't call ps() otherwise)
 2918   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2919 }
 2920 
 2921 void MacroAssembler::unimplemented(const char* what) {
 2922   const char* buf = nullptr;
 2923   {
 2924     ResourceMark rm;
 2925     stringStream ss;
 2926     ss.print("unimplemented: %s", what);
 2927     buf = code_string(ss.as_string());
 2928   }
 2929   stop(buf);
 2930 }
 2931 
 2932 #ifdef _LP64
 2933 #define XSTATE_BV 0x200
 2934 #endif
 2935 
 2936 void MacroAssembler::pop_CPU_state() {
 2937   pop_FPU_state();
 2938   pop_IU_state();
 2939 }
 2940 
 2941 void MacroAssembler::pop_FPU_state() {
 2942 #ifndef _LP64
 2943   frstor(Address(rsp, 0));
 2944 #else
 2945   fxrstor(Address(rsp, 0));
 2946 #endif
 2947   addptr(rsp, FPUStateSizeInWords * wordSize);
 2948 }
 2949 
 2950 void MacroAssembler::pop_IU_state() {
 2951   popa();
 2952   LP64_ONLY(addq(rsp, 8));
 2953   popf();
 2954 }
 2955 
 2956 // Save Integer and Float state
 2957 // Warning: Stack must be 16 byte aligned (64bit)
 2958 void MacroAssembler::push_CPU_state() {
 2959   push_IU_state();
 2960   push_FPU_state();
 2961 }
 2962 
 2963 void MacroAssembler::push_FPU_state() {
 2964   subptr(rsp, FPUStateSizeInWords * wordSize);
 2965 #ifndef _LP64
 2966   fnsave(Address(rsp, 0));
 2967   fwait();
 2968 #else
 2969   fxsave(Address(rsp, 0));
 2970 #endif // LP64
 2971 }
 2972 
 2973 void MacroAssembler::push_IU_state() {
 2974   // Push flags first because pusha kills them
 2975   pushf();
 2976   // Make sure rsp stays 16-byte aligned
 2977   LP64_ONLY(subq(rsp, 8));
 2978   pusha();
 2979 }
 2980 
 2981 void MacroAssembler::push_cont_fastpath() {
 2982   if (!Continuations::enabled()) return;
 2983 
 2984 #ifndef _LP64
 2985   Register rthread = rax;
 2986   Register rrealsp = rbx;
 2987   push(rthread);
 2988   push(rrealsp);
 2989 
 2990   get_thread(rthread);
 2991 
 2992   // The code below wants the original RSP.
 2993   // Move it back after the pushes above.
 2994   movptr(rrealsp, rsp);
 2995   addptr(rrealsp, 2*wordSize);
 2996 #else
 2997   Register rthread = r15_thread;
 2998   Register rrealsp = rsp;
 2999 #endif
 3000 
 3001   Label done;
 3002   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3003   jccb(Assembler::belowEqual, done);
 3004   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3005   bind(done);
 3006 
 3007 #ifndef _LP64
 3008   pop(rrealsp);
 3009   pop(rthread);
 3010 #endif
 3011 }
 3012 
 3013 void MacroAssembler::pop_cont_fastpath() {
 3014   if (!Continuations::enabled()) return;
 3015 
 3016 #ifndef _LP64
 3017   Register rthread = rax;
 3018   Register rrealsp = rbx;
 3019   push(rthread);
 3020   push(rrealsp);
 3021 
 3022   get_thread(rthread);
 3023 
 3024   // The code below wants the original RSP.
 3025   // Move it back after the pushes above.
 3026   movptr(rrealsp, rsp);
 3027   addptr(rrealsp, 2*wordSize);
 3028 #else
 3029   Register rthread = r15_thread;
 3030   Register rrealsp = rsp;
 3031 #endif
 3032 
 3033   Label done;
 3034   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3035   jccb(Assembler::below, done);
 3036   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3037   bind(done);
 3038 
 3039 #ifndef _LP64
 3040   pop(rrealsp);
 3041   pop(rthread);
 3042 #endif
 3043 }
 3044 
 3045 void MacroAssembler::inc_held_monitor_count() {
 3046 #ifdef _LP64
 3047   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3048 #endif
 3049 }
 3050 
 3051 void MacroAssembler::dec_held_monitor_count() {
 3052 #ifdef _LP64
 3053   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3054 #endif
 3055 }
 3056 
 3057 #ifdef ASSERT
 3058 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3059 #ifdef _LP64
 3060   Label no_cont;
 3061   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3062   testl(cont, cont);
 3063   jcc(Assembler::zero, no_cont);
 3064   stop(name);
 3065   bind(no_cont);
 3066 #else
 3067   Unimplemented();
 3068 #endif
 3069 }
 3070 #endif
 3071 
 3072 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3073   if (!java_thread->is_valid()) {
 3074     java_thread = rdi;
 3075     get_thread(java_thread);
 3076   }
 3077   // we must set sp to zero to clear frame
 3078   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3079   // must clear fp, so that compiled frames are not confused; it is
 3080   // possible that we need it only for debugging
 3081   if (clear_fp) {
 3082     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3083   }
 3084   // Always clear the pc because it could have been set by make_walkable()
 3085   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3086   vzeroupper();
 3087 }
 3088 
 3089 void MacroAssembler::restore_rax(Register tmp) {
 3090   if (tmp == noreg) pop(rax);
 3091   else if (tmp != rax) mov(rax, tmp);
 3092 }
 3093 
 3094 void MacroAssembler::round_to(Register reg, int modulus) {
 3095   addptr(reg, modulus - 1);
 3096   andptr(reg, -modulus);
 3097 }
 3098 
 3099 void MacroAssembler::save_rax(Register tmp) {
 3100   if (tmp == noreg) push(rax);
 3101   else if (tmp != rax) mov(tmp, rax);
 3102 }
 3103 
 3104 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3105   if (at_return) {
 3106     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3107     // we may safely use rsp instead to perform the stack watermark check.
 3108     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3109     jcc(Assembler::above, slow_path);
 3110     return;
 3111   }
 3112   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3113   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3114 }
 3115 
 3116 // Calls to C land
 3117 //
 3118 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3119 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3120 // has to be reset to 0. This is required to allow proper stack traversal.
 3121 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3122                                          Register last_java_sp,
 3123                                          Register last_java_fp,
 3124                                          address  last_java_pc,
 3125                                          Register rscratch) {
 3126   vzeroupper();
 3127   // determine java_thread register
 3128   if (!java_thread->is_valid()) {
 3129     java_thread = rdi;
 3130     get_thread(java_thread);
 3131   }
 3132   // determine last_java_sp register
 3133   if (!last_java_sp->is_valid()) {
 3134     last_java_sp = rsp;
 3135   }
 3136   // last_java_fp is optional
 3137   if (last_java_fp->is_valid()) {
 3138     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3139   }
 3140   // last_java_pc is optional
 3141   if (last_java_pc != nullptr) {
 3142     Address java_pc(java_thread,
 3143                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3144     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3145   }
 3146   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3147 }
 3148 
 3149 #ifdef _LP64
 3150 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 3151                                          Register last_java_fp,
 3152                                          Label &L,
 3153                                          Register scratch) {
 3154   lea(scratch, L);
 3155   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch);
 3156   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, nullptr, scratch);
 3157 }
 3158 #endif
 3159 
 3160 void MacroAssembler::shlptr(Register dst, int imm8) {
 3161   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3162 }
 3163 
 3164 void MacroAssembler::shrptr(Register dst, int imm8) {
 3165   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3166 }
 3167 
 3168 void MacroAssembler::sign_extend_byte(Register reg) {
 3169   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3170     movsbl(reg, reg); // movsxb
 3171   } else {
 3172     shll(reg, 24);
 3173     sarl(reg, 24);
 3174   }
 3175 }
 3176 
 3177 void MacroAssembler::sign_extend_short(Register reg) {
 3178   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3179     movswl(reg, reg); // movsxw
 3180   } else {
 3181     shll(reg, 16);
 3182     sarl(reg, 16);
 3183   }
 3184 }
 3185 
 3186 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3187   if (imm32 >= 0 && is8bit(imm32)) {
 3188     testb(dst, imm32);
 3189   } else {
 3190     Assembler::testl(dst, imm32);
 3191   }
 3192 }
 3193 
 3194 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3195   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3196     testb(dst, imm32);
 3197   } else {
 3198     Assembler::testl(dst, imm32);
 3199   }
 3200 }
 3201 
 3202 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3203   assert(always_reachable(src), "Address should be reachable");
 3204   testl(dst, as_Address(src));
 3205 }
 3206 
 3207 #ifdef _LP64
 3208 
 3209 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3210   if (imm32 >= 0) {
 3211     testl(dst, imm32);
 3212   } else {
 3213     Assembler::testq(dst, imm32);
 3214   }
 3215 }
 3216 
 3217 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3218   if (imm32 >= 0) {
 3219     testl(dst, imm32);
 3220   } else {
 3221     Assembler::testq(dst, imm32);
 3222   }
 3223 }
 3224 
 3225 #endif
 3226 
 3227 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3228   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3229   Assembler::pcmpeqb(dst, src);
 3230 }
 3231 
 3232 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3233   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3234   Assembler::pcmpeqw(dst, src);
 3235 }
 3236 
 3237 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3238   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3239   Assembler::pcmpestri(dst, src, imm8);
 3240 }
 3241 
 3242 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3243   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3244   Assembler::pcmpestri(dst, src, imm8);
 3245 }
 3246 
 3247 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3248   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3249   Assembler::pmovzxbw(dst, src);
 3250 }
 3251 
 3252 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3253   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3254   Assembler::pmovzxbw(dst, src);
 3255 }
 3256 
 3257 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3258   assert((src->encoding() < 16),"XMM register should be 0-15");
 3259   Assembler::pmovmskb(dst, src);
 3260 }
 3261 
 3262 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3263   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3264   Assembler::ptest(dst, src);
 3265 }
 3266 
 3267 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3268   assert(rscratch != noreg || always_reachable(src), "missing");
 3269 
 3270   if (reachable(src)) {
 3271     Assembler::sqrtss(dst, as_Address(src));
 3272   } else {
 3273     lea(rscratch, src);
 3274     Assembler::sqrtss(dst, Address(rscratch, 0));
 3275   }
 3276 }
 3277 
 3278 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3279   assert(rscratch != noreg || always_reachable(src), "missing");
 3280 
 3281   if (reachable(src)) {
 3282     Assembler::subsd(dst, as_Address(src));
 3283   } else {
 3284     lea(rscratch, src);
 3285     Assembler::subsd(dst, Address(rscratch, 0));
 3286   }
 3287 }
 3288 
 3289 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3290   assert(rscratch != noreg || always_reachable(src), "missing");
 3291 
 3292   if (reachable(src)) {
 3293     Assembler::roundsd(dst, as_Address(src), rmode);
 3294   } else {
 3295     lea(rscratch, src);
 3296     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3297   }
 3298 }
 3299 
 3300 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3301   assert(rscratch != noreg || always_reachable(src), "missing");
 3302 
 3303   if (reachable(src)) {
 3304     Assembler::subss(dst, as_Address(src));
 3305   } else {
 3306     lea(rscratch, src);
 3307     Assembler::subss(dst, Address(rscratch, 0));
 3308   }
 3309 }
 3310 
 3311 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3312   assert(rscratch != noreg || always_reachable(src), "missing");
 3313 
 3314   if (reachable(src)) {
 3315     Assembler::ucomisd(dst, as_Address(src));
 3316   } else {
 3317     lea(rscratch, src);
 3318     Assembler::ucomisd(dst, Address(rscratch, 0));
 3319   }
 3320 }
 3321 
 3322 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3323   assert(rscratch != noreg || always_reachable(src), "missing");
 3324 
 3325   if (reachable(src)) {
 3326     Assembler::ucomiss(dst, as_Address(src));
 3327   } else {
 3328     lea(rscratch, src);
 3329     Assembler::ucomiss(dst, Address(rscratch, 0));
 3330   }
 3331 }
 3332 
 3333 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3334   assert(rscratch != noreg || always_reachable(src), "missing");
 3335 
 3336   // Used in sign-bit flipping with aligned address.
 3337   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3338 
 3339   if (UseAVX > 2 &&
 3340       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3341       (dst->encoding() >= 16)) {
 3342     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3343   } else if (reachable(src)) {
 3344     Assembler::xorpd(dst, as_Address(src));
 3345   } else {
 3346     lea(rscratch, src);
 3347     Assembler::xorpd(dst, Address(rscratch, 0));
 3348   }
 3349 }
 3350 
 3351 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3352   if (UseAVX > 2 &&
 3353       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3354       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3355     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3356   } else {
 3357     Assembler::xorpd(dst, src);
 3358   }
 3359 }
 3360 
 3361 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3362   if (UseAVX > 2 &&
 3363       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3364       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3365     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3366   } else {
 3367     Assembler::xorps(dst, src);
 3368   }
 3369 }
 3370 
 3371 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3372   assert(rscratch != noreg || always_reachable(src), "missing");
 3373 
 3374   // Used in sign-bit flipping with aligned address.
 3375   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3376 
 3377   if (UseAVX > 2 &&
 3378       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3379       (dst->encoding() >= 16)) {
 3380     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3381   } else if (reachable(src)) {
 3382     Assembler::xorps(dst, as_Address(src));
 3383   } else {
 3384     lea(rscratch, src);
 3385     Assembler::xorps(dst, Address(rscratch, 0));
 3386   }
 3387 }
 3388 
 3389 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3390   assert(rscratch != noreg || always_reachable(src), "missing");
 3391 
 3392   // Used in sign-bit flipping with aligned address.
 3393   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3394   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3395   if (reachable(src)) {
 3396     Assembler::pshufb(dst, as_Address(src));
 3397   } else {
 3398     lea(rscratch, src);
 3399     Assembler::pshufb(dst, Address(rscratch, 0));
 3400   }
 3401 }
 3402 
 3403 // AVX 3-operands instructions
 3404 
 3405 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3406   assert(rscratch != noreg || always_reachable(src), "missing");
 3407 
 3408   if (reachable(src)) {
 3409     vaddsd(dst, nds, as_Address(src));
 3410   } else {
 3411     lea(rscratch, src);
 3412     vaddsd(dst, nds, Address(rscratch, 0));
 3413   }
 3414 }
 3415 
 3416 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3417   assert(rscratch != noreg || always_reachable(src), "missing");
 3418 
 3419   if (reachable(src)) {
 3420     vaddss(dst, nds, as_Address(src));
 3421   } else {
 3422     lea(rscratch, src);
 3423     vaddss(dst, nds, Address(rscratch, 0));
 3424   }
 3425 }
 3426 
 3427 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3428   assert(UseAVX > 0, "requires some form of AVX");
 3429   assert(rscratch != noreg || always_reachable(src), "missing");
 3430 
 3431   if (reachable(src)) {
 3432     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3433   } else {
 3434     lea(rscratch, src);
 3435     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3436   }
 3437 }
 3438 
 3439 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3440   assert(UseAVX > 0, "requires some form of AVX");
 3441   assert(rscratch != noreg || always_reachable(src), "missing");
 3442 
 3443   if (reachable(src)) {
 3444     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3445   } else {
 3446     lea(rscratch, src);
 3447     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3448   }
 3449 }
 3450 
 3451 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3452   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3453   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3454 
 3455   vandps(dst, nds, negate_field, vector_len, rscratch);
 3456 }
 3457 
 3458 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3459   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3460   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3461 
 3462   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3463 }
 3464 
 3465 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3466   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3467   Assembler::vpaddb(dst, nds, src, vector_len);
 3468 }
 3469 
 3470 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3471   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3472   Assembler::vpaddb(dst, nds, src, vector_len);
 3473 }
 3474 
 3475 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3476   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3477   Assembler::vpaddw(dst, nds, src, vector_len);
 3478 }
 3479 
 3480 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3481   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3482   Assembler::vpaddw(dst, nds, src, vector_len);
 3483 }
 3484 
 3485 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3486   assert(rscratch != noreg || always_reachable(src), "missing");
 3487 
 3488   if (reachable(src)) {
 3489     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3490   } else {
 3491     lea(rscratch, src);
 3492     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3493   }
 3494 }
 3495 
 3496 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3497   assert(rscratch != noreg || always_reachable(src), "missing");
 3498 
 3499   if (reachable(src)) {
 3500     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3501   } else {
 3502     lea(rscratch, src);
 3503     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3504   }
 3505 }
 3506 
 3507 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3508   assert(rscratch != noreg || always_reachable(src), "missing");
 3509 
 3510   if (reachable(src)) {
 3511     Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
 3512   } else {
 3513     lea(rscratch, src);
 3514     Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
 3515   }
 3516 }
 3517 
 3518 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3519   assert(rscratch != noreg || always_reachable(src), "missing");
 3520 
 3521   if (reachable(src)) {
 3522     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3523   } else {
 3524     lea(rscratch, src);
 3525     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3526   }
 3527 }
 3528 
 3529 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3530   assert(rscratch != noreg || always_reachable(src), "missing");
 3531 
 3532   if (reachable(src)) {
 3533     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3534   } else {
 3535     lea(rscratch, src);
 3536     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3537   }
 3538 }
 3539 
 3540 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3541   assert(rscratch != noreg || always_reachable(src), "missing");
 3542 
 3543   if (reachable(src)) {
 3544     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3545   } else {
 3546     lea(rscratch, src);
 3547     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3548   }
 3549 }
 3550 
 3551 // Vector float blend
 3552 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3553 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3554   // WARN: Allow dst == (src1|src2), mask == scratch
 3555   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3556   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3557   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3558   if (blend_emulation && scratch_available && dst_available) {
 3559     if (compute_mask) {
 3560       vpsrad(scratch, mask, 32, vector_len);
 3561       mask = scratch;
 3562     }
 3563     if (dst == src1) {
 3564       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3565       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3566     } else {
 3567       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3568       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3569     }
 3570     vpor(dst, dst, scratch, vector_len);
 3571   } else {
 3572     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3573   }
 3574 }
 3575 
 3576 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3577 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3578   // WARN: Allow dst == (src1|src2), mask == scratch
 3579   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3580   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3581   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3582   if (blend_emulation && scratch_available && dst_available) {
 3583     if (compute_mask) {
 3584       vpxor(scratch, scratch, scratch, vector_len);
 3585       vpcmpgtq(scratch, scratch, mask, vector_len);
 3586       mask = scratch;
 3587     }
 3588     if (dst == src1) {
 3589       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3590       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3591     } else {
 3592       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3593       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3594     }
 3595     vpor(dst, dst, scratch, vector_len);
 3596   } else {
 3597     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3598   }
 3599 }
 3600 
 3601 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3602   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3603   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3604 }
 3605 
 3606 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 3607   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3608   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
 3609 }
 3610 
 3611 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3612   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3613   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3614 }
 3615 
 3616 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3617   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3618   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3619 }
 3620 
 3621 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3622   assert(rscratch != noreg || always_reachable(src), "missing");
 3623 
 3624   if (reachable(src)) {
 3625     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3626   } else {
 3627     lea(rscratch, src);
 3628     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3629   }
 3630 }
 3631 
 3632 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3633                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3634   assert(rscratch != noreg || always_reachable(src), "missing");
 3635 
 3636   if (reachable(src)) {
 3637     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3638   } else {
 3639     lea(rscratch, src);
 3640     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3641   }
 3642 }
 3643 
 3644 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3645                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3646   assert(rscratch != noreg || always_reachable(src), "missing");
 3647 
 3648   if (reachable(src)) {
 3649     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3650   } else {
 3651     lea(rscratch, src);
 3652     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3653   }
 3654 }
 3655 
 3656 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3657                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3658   assert(rscratch != noreg || always_reachable(src), "missing");
 3659 
 3660   if (reachable(src)) {
 3661     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3662   } else {
 3663     lea(rscratch, src);
 3664     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3665   }
 3666 }
 3667 
 3668 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3669                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3670   assert(rscratch != noreg || always_reachable(src), "missing");
 3671 
 3672   if (reachable(src)) {
 3673     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3674   } else {
 3675     lea(rscratch, src);
 3676     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3677   }
 3678 }
 3679 
 3680 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3681   if (width == Assembler::Q) {
 3682     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3683   } else {
 3684     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3685   }
 3686 }
 3687 
 3688 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3689   int eq_cond_enc = 0x29;
 3690   int gt_cond_enc = 0x37;
 3691   if (width != Assembler::Q) {
 3692     eq_cond_enc = 0x74 + width;
 3693     gt_cond_enc = 0x64 + width;
 3694   }
 3695   switch (cond) {
 3696   case eq:
 3697     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3698     break;
 3699   case neq:
 3700     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3701     vallones(xtmp, vector_len);
 3702     vpxor(dst, xtmp, dst, vector_len);
 3703     break;
 3704   case le:
 3705     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3706     vallones(xtmp, vector_len);
 3707     vpxor(dst, xtmp, dst, vector_len);
 3708     break;
 3709   case nlt:
 3710     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3711     vallones(xtmp, vector_len);
 3712     vpxor(dst, xtmp, dst, vector_len);
 3713     break;
 3714   case lt:
 3715     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3716     break;
 3717   case nle:
 3718     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3719     break;
 3720   default:
 3721     assert(false, "Should not reach here");
 3722   }
 3723 }
 3724 
 3725 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3726   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3727   Assembler::vpmovzxbw(dst, src, vector_len);
 3728 }
 3729 
 3730 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3731   assert((src->encoding() < 16),"XMM register should be 0-15");
 3732   Assembler::vpmovmskb(dst, src, vector_len);
 3733 }
 3734 
 3735 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3736   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3737   Assembler::vpmullw(dst, nds, src, vector_len);
 3738 }
 3739 
 3740 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3741   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3742   Assembler::vpmullw(dst, nds, src, vector_len);
 3743 }
 3744 
 3745 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3746   assert((UseAVX > 0), "AVX support is needed");
 3747   assert(rscratch != noreg || always_reachable(src), "missing");
 3748 
 3749   if (reachable(src)) {
 3750     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3751   } else {
 3752     lea(rscratch, src);
 3753     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3754   }
 3755 }
 3756 
 3757 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3758   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3759   Assembler::vpsubb(dst, nds, src, vector_len);
 3760 }
 3761 
 3762 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3763   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3764   Assembler::vpsubb(dst, nds, src, vector_len);
 3765 }
 3766 
 3767 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3768   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3769   Assembler::vpsubw(dst, nds, src, vector_len);
 3770 }
 3771 
 3772 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3773   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3774   Assembler::vpsubw(dst, nds, src, vector_len);
 3775 }
 3776 
 3777 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3778   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3779   Assembler::vpsraw(dst, nds, shift, vector_len);
 3780 }
 3781 
 3782 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3783   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3784   Assembler::vpsraw(dst, nds, shift, vector_len);
 3785 }
 3786 
 3787 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3788   assert(UseAVX > 2,"");
 3789   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3790      vector_len = 2;
 3791   }
 3792   Assembler::evpsraq(dst, nds, shift, vector_len);
 3793 }
 3794 
 3795 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3796   assert(UseAVX > 2,"");
 3797   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3798      vector_len = 2;
 3799   }
 3800   Assembler::evpsraq(dst, nds, shift, vector_len);
 3801 }
 3802 
 3803 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3804   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3805   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3806 }
 3807 
 3808 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3809   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3810   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3811 }
 3812 
 3813 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3814   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3815   Assembler::vpsllw(dst, nds, shift, vector_len);
 3816 }
 3817 
 3818 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3819   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3820   Assembler::vpsllw(dst, nds, shift, vector_len);
 3821 }
 3822 
 3823 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3824   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3825   Assembler::vptest(dst, src);
 3826 }
 3827 
 3828 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3829   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3830   Assembler::punpcklbw(dst, src);
 3831 }
 3832 
 3833 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3834   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3835   Assembler::pshufd(dst, src, mode);
 3836 }
 3837 
 3838 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3839   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3840   Assembler::pshuflw(dst, src, mode);
 3841 }
 3842 
 3843 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3844   assert(rscratch != noreg || always_reachable(src), "missing");
 3845 
 3846   if (reachable(src)) {
 3847     vandpd(dst, nds, as_Address(src), vector_len);
 3848   } else {
 3849     lea(rscratch, src);
 3850     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3851   }
 3852 }
 3853 
 3854 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3855   assert(rscratch != noreg || always_reachable(src), "missing");
 3856 
 3857   if (reachable(src)) {
 3858     vandps(dst, nds, as_Address(src), vector_len);
 3859   } else {
 3860     lea(rscratch, src);
 3861     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3862   }
 3863 }
 3864 
 3865 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3866                             bool merge, int vector_len, Register rscratch) {
 3867   assert(rscratch != noreg || always_reachable(src), "missing");
 3868 
 3869   if (reachable(src)) {
 3870     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3871   } else {
 3872     lea(rscratch, src);
 3873     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3874   }
 3875 }
 3876 
 3877 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3878   assert(rscratch != noreg || always_reachable(src), "missing");
 3879 
 3880   if (reachable(src)) {
 3881     vdivsd(dst, nds, as_Address(src));
 3882   } else {
 3883     lea(rscratch, src);
 3884     vdivsd(dst, nds, Address(rscratch, 0));
 3885   }
 3886 }
 3887 
 3888 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3889   assert(rscratch != noreg || always_reachable(src), "missing");
 3890 
 3891   if (reachable(src)) {
 3892     vdivss(dst, nds, as_Address(src));
 3893   } else {
 3894     lea(rscratch, src);
 3895     vdivss(dst, nds, Address(rscratch, 0));
 3896   }
 3897 }
 3898 
 3899 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3900   assert(rscratch != noreg || always_reachable(src), "missing");
 3901 
 3902   if (reachable(src)) {
 3903     vmulsd(dst, nds, as_Address(src));
 3904   } else {
 3905     lea(rscratch, src);
 3906     vmulsd(dst, nds, Address(rscratch, 0));
 3907   }
 3908 }
 3909 
 3910 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3911   assert(rscratch != noreg || always_reachable(src), "missing");
 3912 
 3913   if (reachable(src)) {
 3914     vmulss(dst, nds, as_Address(src));
 3915   } else {
 3916     lea(rscratch, src);
 3917     vmulss(dst, nds, Address(rscratch, 0));
 3918   }
 3919 }
 3920 
 3921 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3922   assert(rscratch != noreg || always_reachable(src), "missing");
 3923 
 3924   if (reachable(src)) {
 3925     vsubsd(dst, nds, as_Address(src));
 3926   } else {
 3927     lea(rscratch, src);
 3928     vsubsd(dst, nds, Address(rscratch, 0));
 3929   }
 3930 }
 3931 
 3932 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3933   assert(rscratch != noreg || always_reachable(src), "missing");
 3934 
 3935   if (reachable(src)) {
 3936     vsubss(dst, nds, as_Address(src));
 3937   } else {
 3938     lea(rscratch, src);
 3939     vsubss(dst, nds, Address(rscratch, 0));
 3940   }
 3941 }
 3942 
 3943 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3944   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3945   assert(rscratch != noreg || always_reachable(src), "missing");
 3946 
 3947   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3948 }
 3949 
 3950 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3951   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3952   assert(rscratch != noreg || always_reachable(src), "missing");
 3953 
 3954   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3955 }
 3956 
 3957 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3958   assert(rscratch != noreg || always_reachable(src), "missing");
 3959 
 3960   if (reachable(src)) {
 3961     vxorpd(dst, nds, as_Address(src), vector_len);
 3962   } else {
 3963     lea(rscratch, src);
 3964     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 3965   }
 3966 }
 3967 
 3968 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3969   assert(rscratch != noreg || always_reachable(src), "missing");
 3970 
 3971   if (reachable(src)) {
 3972     vxorps(dst, nds, as_Address(src), vector_len);
 3973   } else {
 3974     lea(rscratch, src);
 3975     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 3976   }
 3977 }
 3978 
 3979 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3980   assert(rscratch != noreg || always_reachable(src), "missing");
 3981 
 3982   if (UseAVX > 1 || (vector_len < 1)) {
 3983     if (reachable(src)) {
 3984       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 3985     } else {
 3986       lea(rscratch, src);
 3987       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 3988     }
 3989   } else {
 3990     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 3991   }
 3992 }
 3993 
 3994 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3995   assert(rscratch != noreg || always_reachable(src), "missing");
 3996 
 3997   if (reachable(src)) {
 3998     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 3999   } else {
 4000     lea(rscratch, src);
 4001     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 4002   }
 4003 }
 4004 
 4005 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 4006   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 4007   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 4008   // The inverted mask is sign-extended
 4009   andptr(possibly_non_local, inverted_mask);
 4010 }
 4011 
 4012 void MacroAssembler::resolve_jobject(Register value,
 4013                                      Register thread,
 4014                                      Register tmp) {
 4015   assert_different_registers(value, thread, tmp);
 4016   Label done, tagged, weak_tagged;
 4017   testptr(value, value);
 4018   jcc(Assembler::zero, done);           // Use null as-is.
 4019   testptr(value, JNIHandles::tag_mask); // Test for tag.
 4020   jcc(Assembler::notZero, tagged);
 4021 
 4022   // Resolve local handle
 4023   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4024   verify_oop(value);
 4025   jmp(done);
 4026 
 4027   bind(tagged);
 4028   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4029   jcc(Assembler::notZero, weak_tagged);
 4030 
 4031   // Resolve global handle
 4032   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4033   verify_oop(value);
 4034   jmp(done);
 4035 
 4036   bind(weak_tagged);
 4037   // Resolve jweak.
 4038   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4039                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4040   verify_oop(value);
 4041 
 4042   bind(done);
 4043 }
 4044 
 4045 void MacroAssembler::resolve_global_jobject(Register value,
 4046                                             Register thread,
 4047                                             Register tmp) {
 4048   assert_different_registers(value, thread, tmp);
 4049   Label done;
 4050 
 4051   testptr(value, value);
 4052   jcc(Assembler::zero, done);           // Use null as-is.
 4053 
 4054 #ifdef ASSERT
 4055   {
 4056     Label valid_global_tag;
 4057     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4058     jcc(Assembler::notZero, valid_global_tag);
 4059     stop("non global jobject using resolve_global_jobject");
 4060     bind(valid_global_tag);
 4061   }
 4062 #endif
 4063 
 4064   // Resolve global handle
 4065   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4066   verify_oop(value);
 4067 
 4068   bind(done);
 4069 }
 4070 
 4071 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4072   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4073 }
 4074 
 4075 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4076 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4077   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4078 }
 4079 
 4080 void MacroAssembler::subptr(Register dst, Register src) {
 4081   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4082 }
 4083 
 4084 // C++ bool manipulation
 4085 void MacroAssembler::testbool(Register dst) {
 4086   if(sizeof(bool) == 1)
 4087     testb(dst, 0xff);
 4088   else if(sizeof(bool) == 2) {
 4089     // testw implementation needed for two byte bools
 4090     ShouldNotReachHere();
 4091   } else if(sizeof(bool) == 4)
 4092     testl(dst, dst);
 4093   else
 4094     // unsupported
 4095     ShouldNotReachHere();
 4096 }
 4097 
 4098 void MacroAssembler::testptr(Register dst, Register src) {
 4099   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4100 }
 4101 
 4102 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4103 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4104                                    Register var_size_in_bytes,
 4105                                    int con_size_in_bytes,
 4106                                    Register t1,
 4107                                    Register t2,
 4108                                    Label& slow_case) {
 4109   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4110   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4111 }
 4112 
 4113 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4114   RegSet regs;
 4115 #ifdef _LP64
 4116   regs += RegSet::of(rax, rcx, rdx);
 4117 #ifndef _WINDOWS
 4118   regs += RegSet::of(rsi, rdi);
 4119 #endif
 4120   regs += RegSet::range(r8, r11);
 4121 #else
 4122   regs += RegSet::of(rax, rcx, rdx);
 4123 #endif
 4124 #ifdef _LP64
 4125   if (UseAPX) {
 4126     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
 4127   }
 4128 #endif
 4129   return regs;
 4130 }
 4131 
 4132 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4133   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4134 #if defined(_WINDOWS)
 4135   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4136   if (num_xmm_registers > 16) {
 4137      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4138   }
 4139   return result;
 4140 #else
 4141   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4142 #endif
 4143 }
 4144 
 4145 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4146 
 4147 #ifndef _LP64
 4148 static bool use_x87_registers() { return UseSSE < 2; }
 4149 #endif
 4150 static bool use_xmm_registers() { return UseSSE >= 1; }
 4151 
 4152 // C1 only ever uses the first double/float of the XMM register.
 4153 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4154 
 4155 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4156   if (UseSSE == 1) {
 4157     masm->movflt(Address(rsp, offset), reg);
 4158   } else {
 4159     masm->movdbl(Address(rsp, offset), reg);
 4160   }
 4161 }
 4162 
 4163 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4164   if (UseSSE == 1) {
 4165     masm->movflt(reg, Address(rsp, offset));
 4166   } else {
 4167     masm->movdbl(reg, Address(rsp, offset));
 4168   }
 4169 }
 4170 
 4171 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4172                                   bool save_fpu, int& gp_area_size,
 4173                                   int& fp_area_size, int& xmm_area_size) {
 4174 
 4175   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4176                          StackAlignmentInBytes);
 4177 #ifdef _LP64
 4178   fp_area_size = 0;
 4179 #else
 4180   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4181 #endif
 4182   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4183 
 4184   return gp_area_size + fp_area_size + xmm_area_size;
 4185 }
 4186 
 4187 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4188   block_comment("push_call_clobbered_registers start");
 4189   // Regular registers
 4190   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4191 
 4192   int gp_area_size;
 4193   int fp_area_size;
 4194   int xmm_area_size;
 4195   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4196                                                gp_area_size, fp_area_size, xmm_area_size);
 4197   subptr(rsp, total_save_size);
 4198 
 4199   push_set(gp_registers_to_push, 0);
 4200 
 4201 #ifndef _LP64
 4202   if (save_fpu && use_x87_registers()) {
 4203     fnsave(Address(rsp, gp_area_size));
 4204     fwait();
 4205   }
 4206 #endif
 4207   if (save_fpu && use_xmm_registers()) {
 4208     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4209   }
 4210 
 4211   block_comment("push_call_clobbered_registers end");
 4212 }
 4213 
 4214 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4215   block_comment("pop_call_clobbered_registers start");
 4216 
 4217   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4218 
 4219   int gp_area_size;
 4220   int fp_area_size;
 4221   int xmm_area_size;
 4222   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4223                                                gp_area_size, fp_area_size, xmm_area_size);
 4224 
 4225   if (restore_fpu && use_xmm_registers()) {
 4226     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4227   }
 4228 #ifndef _LP64
 4229   if (restore_fpu && use_x87_registers()) {
 4230     frstor(Address(rsp, gp_area_size));
 4231   }
 4232 #endif
 4233 
 4234   pop_set(gp_registers_to_pop, 0);
 4235 
 4236   addptr(rsp, total_save_size);
 4237 
 4238   vzeroupper();
 4239 
 4240   block_comment("pop_call_clobbered_registers end");
 4241 }
 4242 
 4243 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4244   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4245   int spill_offset = offset;
 4246 
 4247   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4248     save_xmm_register(this, spill_offset, *it);
 4249     spill_offset += xmm_save_size();
 4250   }
 4251 }
 4252 
 4253 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4254   int restore_size = set.size() * xmm_save_size();
 4255   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4256 
 4257   int restore_offset = offset + restore_size - xmm_save_size();
 4258 
 4259   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4260     restore_xmm_register(this, restore_offset, *it);
 4261     restore_offset -= xmm_save_size();
 4262   }
 4263 }
 4264 
 4265 void MacroAssembler::push_set(RegSet set, int offset) {
 4266   int spill_offset;
 4267   if (offset == -1) {
 4268     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4269     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4270     subptr(rsp, aligned_size);
 4271     spill_offset = 0;
 4272   } else {
 4273     spill_offset = offset;
 4274   }
 4275 
 4276   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4277     movptr(Address(rsp, spill_offset), *it);
 4278     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4279   }
 4280 }
 4281 
 4282 void MacroAssembler::pop_set(RegSet set, int offset) {
 4283 
 4284   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4285   int restore_size = set.size() * gp_reg_size;
 4286   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4287 
 4288   int restore_offset;
 4289   if (offset == -1) {
 4290     restore_offset = restore_size - gp_reg_size;
 4291   } else {
 4292     restore_offset = offset + restore_size - gp_reg_size;
 4293   }
 4294   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4295     movptr(*it, Address(rsp, restore_offset));
 4296     restore_offset -= gp_reg_size;
 4297   }
 4298 
 4299   if (offset == -1) {
 4300     addptr(rsp, aligned_size);
 4301   }
 4302 }
 4303 
 4304 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4305 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4306   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4307   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4308   Label done;
 4309 
 4310   testptr(length_in_bytes, length_in_bytes);
 4311   jcc(Assembler::zero, done);
 4312 
 4313   // initialize topmost word, divide index by 2, check if odd and test if zero
 4314   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4315 #ifdef ASSERT
 4316   {
 4317     Label L;
 4318     testptr(length_in_bytes, BytesPerWord - 1);
 4319     jcc(Assembler::zero, L);
 4320     stop("length must be a multiple of BytesPerWord");
 4321     bind(L);
 4322   }
 4323 #endif
 4324   Register index = length_in_bytes;
 4325   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4326   if (UseIncDec) {
 4327     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4328   } else {
 4329     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4330     shrptr(index, 1);
 4331   }
 4332 #ifndef _LP64
 4333   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4334   {
 4335     Label even;
 4336     // note: if index was a multiple of 8, then it cannot
 4337     //       be 0 now otherwise it must have been 0 before
 4338     //       => if it is even, we don't need to check for 0 again
 4339     jcc(Assembler::carryClear, even);
 4340     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4341     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4342     // index could be 0 now, must check again
 4343     jcc(Assembler::zero, done);
 4344     bind(even);
 4345   }
 4346 #endif // !_LP64
 4347   // initialize remaining object fields: index is a multiple of 2 now
 4348   {
 4349     Label loop;
 4350     bind(loop);
 4351     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4352     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4353     decrement(index);
 4354     jcc(Assembler::notZero, loop);
 4355   }
 4356 
 4357   bind(done);
 4358 }
 4359 
 4360 // Look up the method for a megamorphic invokeinterface call.
 4361 // The target method is determined by <intf_klass, itable_index>.
 4362 // The receiver klass is in recv_klass.
 4363 // On success, the result will be in method_result, and execution falls through.
 4364 // On failure, execution transfers to the given label.
 4365 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4366                                              Register intf_klass,
 4367                                              RegisterOrConstant itable_index,
 4368                                              Register method_result,
 4369                                              Register scan_temp,
 4370                                              Label& L_no_such_interface,
 4371                                              bool return_method) {
 4372   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4373   assert_different_registers(method_result, intf_klass, scan_temp);
 4374   assert(recv_klass != method_result || !return_method,
 4375          "recv_klass can be destroyed when method isn't needed");
 4376 
 4377   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4378          "caller must use same register for non-constant itable index as for method");
 4379 
 4380   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4381   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4382   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4383   int scan_step   = itableOffsetEntry::size() * wordSize;
 4384   int vte_size    = vtableEntry::size_in_bytes();
 4385   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4386   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4387 
 4388   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4389 
 4390   // Could store the aligned, prescaled offset in the klass.
 4391   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4392 
 4393   if (return_method) {
 4394     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4395     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4396     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4397   }
 4398 
 4399   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4400   //   if (scan->interface() == intf) {
 4401   //     result = (klass + scan->offset() + itable_index);
 4402   //   }
 4403   // }
 4404   Label search, found_method;
 4405 
 4406   for (int peel = 1; peel >= 0; peel--) {
 4407     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4408     cmpptr(intf_klass, method_result);
 4409 
 4410     if (peel) {
 4411       jccb(Assembler::equal, found_method);
 4412     } else {
 4413       jccb(Assembler::notEqual, search);
 4414       // (invert the test to fall through to found_method...)
 4415     }
 4416 
 4417     if (!peel)  break;
 4418 
 4419     bind(search);
 4420 
 4421     // Check that the previous entry is non-null.  A null entry means that
 4422     // the receiver class doesn't implement the interface, and wasn't the
 4423     // same as when the caller was compiled.
 4424     testptr(method_result, method_result);
 4425     jcc(Assembler::zero, L_no_such_interface);
 4426     addptr(scan_temp, scan_step);
 4427   }
 4428 
 4429   bind(found_method);
 4430 
 4431   if (return_method) {
 4432     // Got a hit.
 4433     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4434     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4435   }
 4436 }
 4437 
 4438 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4439 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4440 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4441 // The target method is determined by <holder_klass, itable_index>.
 4442 // The receiver klass is in recv_klass.
 4443 // On success, the result will be in method_result, and execution falls through.
 4444 // On failure, execution transfers to the given label.
 4445 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4446                                                   Register holder_klass,
 4447                                                   Register resolved_klass,
 4448                                                   Register method_result,
 4449                                                   Register scan_temp,
 4450                                                   Register temp_reg2,
 4451                                                   Register receiver,
 4452                                                   int itable_index,
 4453                                                   Label& L_no_such_interface) {
 4454   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4455   Register temp_itbl_klass = method_result;
 4456   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4457 
 4458   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4459   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4460   int scan_step = itableOffsetEntry::size() * wordSize;
 4461   int vte_size = vtableEntry::size_in_bytes();
 4462   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4463   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4464   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4465   assert(vte_size == wordSize, "adjust times_vte_scale");
 4466 
 4467   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4468 
 4469   // temp_itbl_klass = recv_klass.itable[0]
 4470   // scan_temp = &recv_klass.itable[0] + step
 4471   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4472   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4473   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4474   xorptr(temp_reg, temp_reg);
 4475 
 4476   // Initial checks:
 4477   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4478   //   - if (itable[0] == 0), no such interface
 4479   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4480   cmpptr(holder_klass, resolved_klass);
 4481   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4482   testptr(temp_itbl_klass, temp_itbl_klass);
 4483   jccb(Assembler::zero, L_no_such_interface);
 4484   cmpptr(holder_klass, temp_itbl_klass);
 4485   jccb(Assembler::equal, L_holder_found);
 4486 
 4487   // Loop: Look for holder_klass record in itable
 4488   //   do {
 4489   //     tmp = itable[index];
 4490   //     index += step;
 4491   //     if (tmp == holder_klass) {
 4492   //       goto L_holder_found; // Found!
 4493   //     }
 4494   //   } while (tmp != 0);
 4495   //   goto L_no_such_interface // Not found.
 4496   Label L_scan_holder;
 4497   bind(L_scan_holder);
 4498     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4499     addptr(scan_temp, scan_step);
 4500     cmpptr(holder_klass, temp_itbl_klass);
 4501     jccb(Assembler::equal, L_holder_found);
 4502     testptr(temp_itbl_klass, temp_itbl_klass);
 4503     jccb(Assembler::notZero, L_scan_holder);
 4504 
 4505   jmpb(L_no_such_interface);
 4506 
 4507   // Loop: Look for resolved_class record in itable
 4508   //   do {
 4509   //     tmp = itable[index];
 4510   //     index += step;
 4511   //     if (tmp == holder_klass) {
 4512   //        // Also check if we have met a holder klass
 4513   //        holder_tmp = itable[index-step-ioffset];
 4514   //     }
 4515   //     if (tmp == resolved_klass) {
 4516   //        goto L_resolved_found;  // Found!
 4517   //     }
 4518   //   } while (tmp != 0);
 4519   //   goto L_no_such_interface // Not found.
 4520   //
 4521   Label L_loop_scan_resolved;
 4522   bind(L_loop_scan_resolved);
 4523     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4524     addptr(scan_temp, scan_step);
 4525     bind(L_loop_scan_resolved_entry);
 4526     cmpptr(holder_klass, temp_itbl_klass);
 4527     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4528     cmpptr(resolved_klass, temp_itbl_klass);
 4529     jccb(Assembler::equal, L_resolved_found);
 4530     testptr(temp_itbl_klass, temp_itbl_klass);
 4531     jccb(Assembler::notZero, L_loop_scan_resolved);
 4532 
 4533   jmpb(L_no_such_interface);
 4534 
 4535   Label L_ready;
 4536 
 4537   // See if we already have a holder klass. If not, go and scan for it.
 4538   bind(L_resolved_found);
 4539   testptr(temp_reg, temp_reg);
 4540   jccb(Assembler::zero, L_scan_holder);
 4541   jmpb(L_ready);
 4542 
 4543   bind(L_holder_found);
 4544   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4545 
 4546   // Finally, temp_reg contains holder_klass vtable offset
 4547   bind(L_ready);
 4548   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4549   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4550     load_klass(scan_temp, receiver, noreg);
 4551     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4552   } else {
 4553     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4554   }
 4555 }
 4556 
 4557 
 4558 // virtual method calling
 4559 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4560                                            RegisterOrConstant vtable_index,
 4561                                            Register method_result) {
 4562   const ByteSize base = Klass::vtable_start_offset();
 4563   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4564   Address vtable_entry_addr(recv_klass,
 4565                             vtable_index, Address::times_ptr,
 4566                             base + vtableEntry::method_offset());
 4567   movptr(method_result, vtable_entry_addr);
 4568 }
 4569 
 4570 
 4571 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4572                            Register super_klass,
 4573                            Register temp_reg,
 4574                            Label& L_success) {
 4575   Label L_failure;
 4576   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4577   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4578   bind(L_failure);
 4579 }
 4580 
 4581 
 4582 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4583                                                    Register super_klass,
 4584                                                    Register temp_reg,
 4585                                                    Label* L_success,
 4586                                                    Label* L_failure,
 4587                                                    Label* L_slow_path,
 4588                                         RegisterOrConstant super_check_offset) {
 4589   assert_different_registers(sub_klass, super_klass, temp_reg);
 4590   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4591   if (super_check_offset.is_register()) {
 4592     assert_different_registers(sub_klass, super_klass,
 4593                                super_check_offset.as_register());
 4594   } else if (must_load_sco) {
 4595     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4596   }
 4597 
 4598   Label L_fallthrough;
 4599   int label_nulls = 0;
 4600   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4601   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4602   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4603   assert(label_nulls <= 1, "at most one null in the batch");
 4604 
 4605   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4606   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4607   Address super_check_offset_addr(super_klass, sco_offset);
 4608 
 4609   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4610   // range of a jccb.  If this routine grows larger, reconsider at
 4611   // least some of these.
 4612 #define local_jcc(assembler_cond, label)                                \
 4613   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4614   else                             jcc( assembler_cond, label) /*omit semi*/
 4615 
 4616   // Hacked jmp, which may only be used just before L_fallthrough.
 4617 #define final_jmp(label)                                                \
 4618   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4619   else                            jmp(label)                /*omit semi*/
 4620 
 4621   // If the pointers are equal, we are done (e.g., String[] elements).
 4622   // This self-check enables sharing of secondary supertype arrays among
 4623   // non-primary types such as array-of-interface.  Otherwise, each such
 4624   // type would need its own customized SSA.
 4625   // We move this check to the front of the fast path because many
 4626   // type checks are in fact trivially successful in this manner,
 4627   // so we get a nicely predicted branch right at the start of the check.
 4628   cmpptr(sub_klass, super_klass);
 4629   local_jcc(Assembler::equal, *L_success);
 4630 
 4631   // Check the supertype display:
 4632   if (must_load_sco) {
 4633     // Positive movl does right thing on LP64.
 4634     movl(temp_reg, super_check_offset_addr);
 4635     super_check_offset = RegisterOrConstant(temp_reg);
 4636   }
 4637   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4638   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4639 
 4640   // This check has worked decisively for primary supers.
 4641   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4642   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4643   // This works in the same check above because of a tricky aliasing
 4644   // between the super_cache and the primary super display elements.
 4645   // (The 'super_check_addr' can address either, as the case requires.)
 4646   // Note that the cache is updated below if it does not help us find
 4647   // what we need immediately.
 4648   // So if it was a primary super, we can just fail immediately.
 4649   // Otherwise, it's the slow path for us (no success at this point).
 4650 
 4651   if (super_check_offset.is_register()) {
 4652     local_jcc(Assembler::equal, *L_success);
 4653     cmpl(super_check_offset.as_register(), sc_offset);
 4654     if (L_failure == &L_fallthrough) {
 4655       local_jcc(Assembler::equal, *L_slow_path);
 4656     } else {
 4657       local_jcc(Assembler::notEqual, *L_failure);
 4658       final_jmp(*L_slow_path);
 4659     }
 4660   } else if (super_check_offset.as_constant() == sc_offset) {
 4661     // Need a slow path; fast failure is impossible.
 4662     if (L_slow_path == &L_fallthrough) {
 4663       local_jcc(Assembler::equal, *L_success);
 4664     } else {
 4665       local_jcc(Assembler::notEqual, *L_slow_path);
 4666       final_jmp(*L_success);
 4667     }
 4668   } else {
 4669     // No slow path; it's a fast decision.
 4670     if (L_failure == &L_fallthrough) {
 4671       local_jcc(Assembler::equal, *L_success);
 4672     } else {
 4673       local_jcc(Assembler::notEqual, *L_failure);
 4674       final_jmp(*L_success);
 4675     }
 4676   }
 4677 
 4678   bind(L_fallthrough);
 4679 
 4680 #undef local_jcc
 4681 #undef final_jmp
 4682 }
 4683 
 4684 
 4685 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
 4686                                                           Register super_klass,
 4687                                                           Register temp_reg,
 4688                                                           Register temp2_reg,
 4689                                                           Label* L_success,
 4690                                                           Label* L_failure,
 4691                                                           bool set_cond_codes) {
 4692   assert_different_registers(sub_klass, super_klass, temp_reg);
 4693   if (temp2_reg != noreg)
 4694     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4695 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4696 
 4697   Label L_fallthrough;
 4698   int label_nulls = 0;
 4699   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4700   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4701   assert(label_nulls <= 1, "at most one null in the batch");
 4702 
 4703   // a couple of useful fields in sub_klass:
 4704   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4705   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4706   Address secondary_supers_addr(sub_klass, ss_offset);
 4707   Address super_cache_addr(     sub_klass, sc_offset);
 4708 
 4709   // Do a linear scan of the secondary super-klass chain.
 4710   // This code is rarely used, so simplicity is a virtue here.
 4711   // The repne_scan instruction uses fixed registers, which we must spill.
 4712   // Don't worry too much about pre-existing connections with the input regs.
 4713 
 4714   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4715   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4716 
 4717   // Get super_klass value into rax (even if it was in rdi or rcx).
 4718   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4719   if (super_klass != rax) {
 4720     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4721     mov(rax, super_klass);
 4722   }
 4723   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4724   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4725 
 4726 #ifndef PRODUCT
 4727   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4728   ExternalAddress pst_counter_addr((address) pst_counter);
 4729   NOT_LP64(  incrementl(pst_counter_addr) );
 4730   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4731   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4732 #endif //PRODUCT
 4733 
 4734   // We will consult the secondary-super array.
 4735   movptr(rdi, secondary_supers_addr);
 4736   // Load the array length.  (Positive movl does right thing on LP64.)
 4737   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4738   // Skip to start of data.
 4739   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4740 
 4741   // Scan RCX words at [RDI] for an occurrence of RAX.
 4742   // Set NZ/Z based on last compare.
 4743   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4744   // not change flags (only scas instruction which is repeated sets flags).
 4745   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4746 
 4747     testptr(rax,rax); // Set Z = 0
 4748     repne_scan();
 4749 
 4750   // Unspill the temp. registers:
 4751   if (pushed_rdi)  pop(rdi);
 4752   if (pushed_rcx)  pop(rcx);
 4753   if (pushed_rax)  pop(rax);
 4754 
 4755   if (set_cond_codes) {
 4756     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4757     assert(!pushed_rdi, "rdi must be left non-null");
 4758     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4759   }
 4760 
 4761   if (L_failure == &L_fallthrough)
 4762         jccb(Assembler::notEqual, *L_failure);
 4763   else  jcc(Assembler::notEqual, *L_failure);
 4764 
 4765   // Success.  Cache the super we found and proceed in triumph.
 4766   movptr(super_cache_addr, super_klass);
 4767 
 4768   if (L_success != &L_fallthrough) {
 4769     jmp(*L_success);
 4770   }
 4771 
 4772 #undef IS_A_TEMP
 4773 
 4774   bind(L_fallthrough);
 4775 }
 4776 
 4777 #ifndef _LP64
 4778 
 4779 // 32-bit x86 only: always use the linear search.
 4780 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4781                                                    Register super_klass,
 4782                                                    Register temp_reg,
 4783                                                    Register temp2_reg,
 4784                                                    Label* L_success,
 4785                                                    Label* L_failure,
 4786                                                    bool set_cond_codes) {
 4787   check_klass_subtype_slow_path_linear
 4788     (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes);
 4789 }
 4790 
 4791 #else // _LP64
 4792 
 4793 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4794                                                    Register super_klass,
 4795                                                    Register temp_reg,
 4796                                                    Register temp2_reg,
 4797                                                    Label* L_success,
 4798                                                    Label* L_failure,
 4799                                                    bool set_cond_codes) {
 4800   assert(set_cond_codes == false, "must be false on 64-bit x86");
 4801   check_klass_subtype_slow_path
 4802     (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg,
 4803      L_success, L_failure);
 4804 }
 4805 
 4806 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4807                                                    Register super_klass,
 4808                                                    Register temp_reg,
 4809                                                    Register temp2_reg,
 4810                                                    Register temp3_reg,
 4811                                                    Register temp4_reg,
 4812                                                    Label* L_success,
 4813                                                    Label* L_failure) {
 4814   if (UseSecondarySupersTable) {
 4815     check_klass_subtype_slow_path_table
 4816       (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg,
 4817        L_success, L_failure);
 4818   } else {
 4819     check_klass_subtype_slow_path_linear
 4820       (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false);
 4821   }
 4822 }
 4823 
 4824 Register MacroAssembler::allocate_if_noreg(Register r,
 4825                                   RegSetIterator<Register> &available_regs,
 4826                                   RegSet &regs_to_push) {
 4827   if (!r->is_valid()) {
 4828     r = *available_regs++;
 4829     regs_to_push += r;
 4830   }
 4831   return r;
 4832 }
 4833 
 4834 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
 4835                                                          Register super_klass,
 4836                                                          Register temp_reg,
 4837                                                          Register temp2_reg,
 4838                                                          Register temp3_reg,
 4839                                                          Register result_reg,
 4840                                                          Label* L_success,
 4841                                                          Label* L_failure) {
 4842   // NB! Callers may assume that, when temp2_reg is a valid register,
 4843   // this code sets it to a nonzero value.
 4844   bool temp2_reg_was_valid = temp2_reg->is_valid();
 4845 
 4846   RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg);
 4847 
 4848   Label L_fallthrough;
 4849   int label_nulls = 0;
 4850   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4851   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4852   assert(label_nulls <= 1, "at most one null in the batch");
 4853 
 4854   BLOCK_COMMENT("check_klass_subtype_slow_path_table");
 4855 
 4856   RegSetIterator<Register> available_regs
 4857     = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin();
 4858 
 4859   RegSet pushed_regs;
 4860 
 4861   temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
 4862   temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
 4863   temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);
 4864   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
 4865   Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs);
 4866 
 4867   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg);
 4868 
 4869   {
 4870 
 4871     int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4872     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4873     subptr(rsp, aligned_size);
 4874     push_set(pushed_regs, 0);
 4875 
 4876     lookup_secondary_supers_table_var(sub_klass,
 4877                                       super_klass,
 4878                                       temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
 4879     cmpq(result_reg, 0);
 4880 
 4881     // Unspill the temp. registers:
 4882     pop_set(pushed_regs, 0);
 4883     // Increment SP but do not clobber flags.
 4884     lea(rsp, Address(rsp, aligned_size));
 4885   }
 4886 
 4887   if (temp2_reg_was_valid) {
 4888     movq(temp2_reg, 1);
 4889   }
 4890 
 4891   jcc(Assembler::notEqual, *L_failure);
 4892 
 4893   if (L_success != &L_fallthrough) {
 4894     jmp(*L_success);
 4895   }
 4896 
 4897   bind(L_fallthrough);
 4898 }
 4899 
 4900 // population_count variant for running without the POPCNT
 4901 // instruction, which was introduced with SSE4.2 in 2008.
 4902 void MacroAssembler::population_count(Register dst, Register src,
 4903                                       Register scratch1, Register scratch2) {
 4904   assert_different_registers(src, scratch1, scratch2);
 4905   if (UsePopCountInstruction) {
 4906     Assembler::popcntq(dst, src);
 4907   } else {
 4908     assert_different_registers(src, scratch1, scratch2);
 4909     assert_different_registers(dst, scratch1, scratch2);
 4910     Label loop, done;
 4911 
 4912     mov(scratch1, src);
 4913     // dst = 0;
 4914     // while(scratch1 != 0) {
 4915     //   dst++;
 4916     //   scratch1 &= (scratch1 - 1);
 4917     // }
 4918     xorl(dst, dst);
 4919     testq(scratch1, scratch1);
 4920     jccb(Assembler::equal, done);
 4921     {
 4922       bind(loop);
 4923       incq(dst);
 4924       movq(scratch2, scratch1);
 4925       decq(scratch2);
 4926       andq(scratch1, scratch2);
 4927       jccb(Assembler::notEqual, loop);
 4928     }
 4929     bind(done);
 4930   }
 4931 #ifdef ASSERT
 4932   mov64(scratch1, 0xCafeBabeDeadBeef);
 4933   movq(scratch2, scratch1);
 4934 #endif
 4935 }
 4936 
 4937 // Ensure that the inline code and the stub are using the same registers.
 4938 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
 4939 do {                                                                 \
 4940   assert(r_super_klass  == rax, "mismatch");                         \
 4941   assert(r_array_base   == rbx, "mismatch");                         \
 4942   assert(r_array_length == rcx, "mismatch");                         \
 4943   assert(r_array_index  == rdx, "mismatch");                         \
 4944   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
 4945   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
 4946   assert(result         == rdi || result      == noreg, "mismatch"); \
 4947 } while(0)
 4948 
 4949 // Versions of salq and rorq that don't need count to be in rcx
 4950 
 4951 void MacroAssembler::salq(Register dest, Register count) {
 4952   if (count == rcx) {
 4953     Assembler::salq(dest);
 4954   } else {
 4955     assert_different_registers(rcx, dest);
 4956     xchgq(rcx, count);
 4957     Assembler::salq(dest);
 4958     xchgq(rcx, count);
 4959   }
 4960 }
 4961 
 4962 void MacroAssembler::rorq(Register dest, Register count) {
 4963   if (count == rcx) {
 4964     Assembler::rorq(dest);
 4965   } else {
 4966     assert_different_registers(rcx, dest);
 4967     xchgq(rcx, count);
 4968     Assembler::rorq(dest);
 4969     xchgq(rcx, count);
 4970   }
 4971 }
 4972 
 4973 // Return true: we succeeded in generating this code
 4974 //
 4975 // At runtime, return 0 in result if r_super_klass is a superclass of
 4976 // r_sub_klass, otherwise return nonzero. Use this if you know the
 4977 // super_klass_slot of the class you're looking for. This is always
 4978 // the case for instanceof and checkcast.
 4979 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
 4980                                                          Register r_super_klass,
 4981                                                          Register temp1,
 4982                                                          Register temp2,
 4983                                                          Register temp3,
 4984                                                          Register temp4,
 4985                                                          Register result,
 4986                                                          u1 super_klass_slot) {
 4987   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 4988 
 4989   Label L_fallthrough, L_success, L_failure;
 4990 
 4991   BLOCK_COMMENT("lookup_secondary_supers_table {");
 4992 
 4993   const Register
 4994     r_array_index  = temp1,
 4995     r_array_length = temp2,
 4996     r_array_base   = temp3,
 4997     r_bitmap       = temp4;
 4998 
 4999   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 5000 
 5001   xorq(result, result); // = 0
 5002 
 5003   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5004   movq(r_array_index, r_bitmap);
 5005 
 5006   // First check the bitmap to see if super_klass might be present. If
 5007   // the bit is zero, we are certain that super_klass is not one of
 5008   // the secondary supers.
 5009   u1 bit = super_klass_slot;
 5010   {
 5011     // NB: If the count in a x86 shift instruction is 0, the flags are
 5012     // not affected, so we do a testq instead.
 5013     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
 5014     if (shift_count != 0) {
 5015       salq(r_array_index, shift_count);
 5016     } else {
 5017       testq(r_array_index, r_array_index);
 5018     }
 5019   }
 5020   // We test the MSB of r_array_index, i.e. its sign bit
 5021   jcc(Assembler::positive, L_failure);
 5022 
 5023   // Get the first array index that can contain super_klass into r_array_index.
 5024   if (bit != 0) {
 5025     population_count(r_array_index, r_array_index, temp2, temp3);
 5026   } else {
 5027     movl(r_array_index, 1);
 5028   }
 5029   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5030 
 5031   // We will consult the secondary-super array.
 5032   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5033 
 5034   // We're asserting that the first word in an Array<Klass*> is the
 5035   // length, and the second word is the first word of the data. If
 5036   // that ever changes, r_array_base will have to be adjusted here.
 5037   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5038   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5039 
 5040   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5041   jccb(Assembler::equal, L_success);
 5042 
 5043   // Is there another entry to check? Consult the bitmap.
 5044   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
 5045   jccb(Assembler::carryClear, L_failure);
 5046 
 5047   // Linear probe. Rotate the bitmap so that the next bit to test is
 5048   // in Bit 1.
 5049   if (bit != 0) {
 5050     rorq(r_bitmap, bit);
 5051   }
 5052 
 5053   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5054   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5055   // Kills: r_array_length.
 5056   // Returns: result.
 5057   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
 5058   // Result (0/1) is in rdi
 5059   jmpb(L_fallthrough);
 5060 
 5061   bind(L_failure);
 5062   incq(result); // 0 => 1
 5063 
 5064   bind(L_success);
 5065   // result = 0;
 5066 
 5067   bind(L_fallthrough);
 5068   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5069 
 5070   if (VerifySecondarySupers) {
 5071     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5072                                   temp1, temp2, temp3);
 5073   }
 5074 }
 5075 
 5076 // At runtime, return 0 in result if r_super_klass is a superclass of
 5077 // r_sub_klass, otherwise return nonzero. Use this version of
 5078 // lookup_secondary_supers_table() if you don't know ahead of time
 5079 // which superclass will be searched for. Used by interpreter and
 5080 // runtime stubs. It is larger and has somewhat greater latency than
 5081 // the version above, which takes a constant super_klass_slot.
 5082 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
 5083                                                        Register r_super_klass,
 5084                                                        Register temp1,
 5085                                                        Register temp2,
 5086                                                        Register temp3,
 5087                                                        Register temp4,
 5088                                                        Register result) {
 5089   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 5090   assert_different_registers(r_sub_klass, r_super_klass, rcx);
 5091   RegSet temps = RegSet::of(temp1, temp2, temp3, temp4);
 5092 
 5093   Label L_fallthrough, L_success, L_failure;
 5094 
 5095   BLOCK_COMMENT("lookup_secondary_supers_table {");
 5096 
 5097   RegSetIterator<Register> available_regs = (temps - rcx).begin();
 5098 
 5099   // FIXME. Once we are sure that all paths reaching this point really
 5100   // do pass rcx as one of our temps we can get rid of the following
 5101   // workaround.
 5102   assert(temps.contains(rcx), "fix this code");
 5103 
 5104   // We prefer to have our shift count in rcx. If rcx is one of our
 5105   // temps, use it for slot. If not, pick any of our temps.
 5106   Register slot;
 5107   if (!temps.contains(rcx)) {
 5108     slot = *available_regs++;
 5109   } else {
 5110     slot = rcx;
 5111   }
 5112 
 5113   const Register r_array_index = *available_regs++;
 5114   const Register r_bitmap      = *available_regs++;
 5115 
 5116   // The logic above guarantees this property, but we state it here.
 5117   assert_different_registers(r_array_index, r_bitmap, rcx);
 5118 
 5119   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5120   movq(r_array_index, r_bitmap);
 5121 
 5122   // First check the bitmap to see if super_klass might be present. If
 5123   // the bit is zero, we are certain that super_klass is not one of
 5124   // the secondary supers.
 5125   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5126   xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
 5127   salq(r_array_index, slot);
 5128 
 5129   testq(r_array_index, r_array_index);
 5130   // We test the MSB of r_array_index, i.e. its sign bit
 5131   jcc(Assembler::positive, L_failure);
 5132 
 5133   const Register r_array_base = *available_regs++;
 5134 
 5135   // Get the first array index that can contain super_klass into r_array_index.
 5136   // Note: Clobbers r_array_base and slot.
 5137   population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot);
 5138 
 5139   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5140 
 5141   // We will consult the secondary-super array.
 5142   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5143 
 5144   // We're asserting that the first word in an Array<Klass*> is the
 5145   // length, and the second word is the first word of the data. If
 5146   // that ever changes, r_array_base will have to be adjusted here.
 5147   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5148   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5149 
 5150   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5151   jccb(Assembler::equal, L_success);
 5152 
 5153   // Restore slot to its true value
 5154   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5155 
 5156   // Linear probe. Rotate the bitmap so that the next bit to test is
 5157   // in Bit 1.
 5158   rorq(r_bitmap, slot);
 5159 
 5160   // Is there another entry to check? Consult the bitmap.
 5161   btq(r_bitmap, 1);
 5162   jccb(Assembler::carryClear, L_failure);
 5163 
 5164   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5165   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5166   // Kills: r_array_length.
 5167   // Returns: result.
 5168   lookup_secondary_supers_table_slow_path(r_super_klass,
 5169                                           r_array_base,
 5170                                           r_array_index,
 5171                                           r_bitmap,
 5172                                           /*temp1*/result,
 5173                                           /*temp2*/slot,
 5174                                           &L_success,
 5175                                           nullptr);
 5176 
 5177   bind(L_failure);
 5178   movq(result, 1);
 5179   jmpb(L_fallthrough);
 5180 
 5181   bind(L_success);
 5182   xorq(result, result); // = 0
 5183 
 5184   bind(L_fallthrough);
 5185   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5186 
 5187   if (VerifySecondarySupers) {
 5188     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5189                                   temp1, temp2, temp3);
 5190   }
 5191 }
 5192 
 5193 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
 5194                                  Label* L_success, Label* L_failure) {
 5195   Label L_loop, L_fallthrough;
 5196   {
 5197     int label_nulls = 0;
 5198     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
 5199     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
 5200     assert(label_nulls <= 1, "at most one null in the batch");
 5201   }
 5202   bind(L_loop);
 5203   cmpq(value, Address(addr, count, Address::times_8));
 5204   jcc(Assembler::equal, *L_success);
 5205   addl(count, 1);
 5206   cmpl(count, limit);
 5207   jcc(Assembler::less, L_loop);
 5208 
 5209   if (&L_fallthrough != L_failure) {
 5210     jmp(*L_failure);
 5211   }
 5212   bind(L_fallthrough);
 5213 }
 5214 
 5215 // Called by code generated by check_klass_subtype_slow_path
 5216 // above. This is called when there is a collision in the hashed
 5217 // lookup in the secondary supers array.
 5218 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
 5219                                                              Register r_array_base,
 5220                                                              Register r_array_index,
 5221                                                              Register r_bitmap,
 5222                                                              Register temp1,
 5223                                                              Register temp2,
 5224                                                              Label* L_success,
 5225                                                              Label* L_failure) {
 5226   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
 5227 
 5228   const Register
 5229     r_array_length = temp1,
 5230     r_sub_klass    = noreg,
 5231     result         = noreg;
 5232 
 5233   Label L_fallthrough;
 5234   int label_nulls = 0;
 5235   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 5236   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 5237   assert(label_nulls <= 1, "at most one null in the batch");
 5238 
 5239   // Load the array length.
 5240   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5241   // And adjust the array base to point to the data.
 5242   // NB! Effectively increments current slot index by 1.
 5243   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
 5244   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5245 
 5246   // Linear probe
 5247   Label L_huge;
 5248 
 5249   // The bitmap is full to bursting.
 5250   // Implicit invariant: BITMAP_FULL implies (length > 0)
 5251   cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
 5252   jcc(Assembler::greater, L_huge);
 5253 
 5254   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
 5255   // current slot (at secondary_supers[r_array_index]) has not yet
 5256   // been inspected, and r_array_index may be out of bounds if we
 5257   // wrapped around the end of the array.
 5258 
 5259   { // This is conventional linear probing, but instead of terminating
 5260     // when a null entry is found in the table, we maintain a bitmap
 5261     // in which a 0 indicates missing entries.
 5262     // The check above guarantees there are 0s in the bitmap, so the loop
 5263     // eventually terminates.
 5264 
 5265     xorl(temp2, temp2); // = 0;
 5266 
 5267     Label L_again;
 5268     bind(L_again);
 5269 
 5270     // Check for array wraparound.
 5271     cmpl(r_array_index, r_array_length);
 5272     cmovl(Assembler::greaterEqual, r_array_index, temp2);
 5273 
 5274     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5275     jcc(Assembler::equal, *L_success);
 5276 
 5277     // If the next bit in bitmap is zero, we're done.
 5278     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
 5279     jcc(Assembler::carryClear, *L_failure);
 5280 
 5281     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
 5282     addl(r_array_index, 1);
 5283 
 5284     jmp(L_again);
 5285   }
 5286 
 5287   { // Degenerate case: more than 64 secondary supers.
 5288     // FIXME: We could do something smarter here, maybe a vectorized
 5289     // comparison or a binary search, but is that worth any added
 5290     // complexity?
 5291     bind(L_huge);
 5292     xorl(r_array_index, r_array_index); // = 0
 5293     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
 5294                 L_success,
 5295                 (&L_fallthrough != L_failure ? L_failure : nullptr));
 5296 
 5297     bind(L_fallthrough);
 5298   }
 5299 }
 5300 
 5301 struct VerifyHelperArguments {
 5302   Klass* _super;
 5303   Klass* _sub;
 5304   intptr_t _linear_result;
 5305   intptr_t _table_result;
 5306 };
 5307 
 5308 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
 5309   Klass::on_secondary_supers_verification_failure(args->_super,
 5310                                                   args->_sub,
 5311                                                   args->_linear_result,
 5312                                                   args->_table_result,
 5313                                                   msg);
 5314 }
 5315 
 5316 // Make sure that the hashed lookup and a linear scan agree.
 5317 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
 5318                                                    Register r_super_klass,
 5319                                                    Register result,
 5320                                                    Register temp1,
 5321                                                    Register temp2,
 5322                                                    Register temp3) {
 5323   const Register
 5324       r_array_index  = temp1,
 5325       r_array_length = temp2,
 5326       r_array_base   = temp3,
 5327       r_bitmap       = noreg;
 5328 
 5329   BLOCK_COMMENT("verify_secondary_supers_table {");
 5330 
 5331   Label L_success, L_failure, L_check, L_done;
 5332 
 5333   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5334   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5335   // And adjust the array base to point to the data.
 5336   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5337 
 5338   testl(r_array_length, r_array_length); // array_length == 0?
 5339   jcc(Assembler::zero, L_failure);
 5340 
 5341   movl(r_array_index, 0);
 5342   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
 5343   // fall through to L_failure
 5344 
 5345   const Register linear_result = r_array_index; // reuse temp1
 5346 
 5347   bind(L_failure); // not present
 5348   movl(linear_result, 1);
 5349   jmp(L_check);
 5350 
 5351   bind(L_success); // present
 5352   movl(linear_result, 0);
 5353 
 5354   bind(L_check);
 5355   cmpl(linear_result, result);
 5356   jcc(Assembler::equal, L_done);
 5357 
 5358   { // To avoid calling convention issues, build a record on the stack
 5359     // and pass the pointer to that instead.
 5360     push(result);
 5361     push(linear_result);
 5362     push(r_sub_klass);
 5363     push(r_super_klass);
 5364     movptr(c_rarg1, rsp);
 5365     movptr(c_rarg0, (uintptr_t) "mismatch");
 5366     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
 5367     should_not_reach_here();
 5368   }
 5369   bind(L_done);
 5370 
 5371   BLOCK_COMMENT("} verify_secondary_supers_table");
 5372 }
 5373 
 5374 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
 5375 
 5376 #endif // LP64
 5377 
 5378 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 5379   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5380 
 5381   Label L_fallthrough;
 5382   if (L_fast_path == nullptr) {
 5383     L_fast_path = &L_fallthrough;
 5384   } else if (L_slow_path == nullptr) {
 5385     L_slow_path = &L_fallthrough;
 5386   }
 5387 
 5388   // Fast path check: class is fully initialized.
 5389   // init_state needs acquire, but x86 is TSO, and so we are already good.
 5390   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5391   jcc(Assembler::equal, *L_fast_path);
 5392 
 5393   // Fast path check: current thread is initializer thread
 5394   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 5395   if (L_slow_path == &L_fallthrough) {
 5396     jcc(Assembler::equal, *L_fast_path);
 5397     bind(*L_slow_path);
 5398   } else if (L_fast_path == &L_fallthrough) {
 5399     jcc(Assembler::notEqual, *L_slow_path);
 5400     bind(*L_fast_path);
 5401   } else {
 5402     Unimplemented();
 5403   }
 5404 }
 5405 
 5406 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5407   if (VM_Version::supports_cmov()) {
 5408     cmovl(cc, dst, src);
 5409   } else {
 5410     Label L;
 5411     jccb(negate_condition(cc), L);
 5412     movl(dst, src);
 5413     bind(L);
 5414   }
 5415 }
 5416 
 5417 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5418   if (VM_Version::supports_cmov()) {
 5419     cmovl(cc, dst, src);
 5420   } else {
 5421     Label L;
 5422     jccb(negate_condition(cc), L);
 5423     movl(dst, src);
 5424     bind(L);
 5425   }
 5426 }
 5427 
 5428 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5429   if (!VerifyOops) return;
 5430 
 5431   BLOCK_COMMENT("verify_oop {");
 5432 #ifdef _LP64
 5433   push(rscratch1);
 5434 #endif
 5435   push(rax);                          // save rax
 5436   push(reg);                          // pass register argument
 5437 
 5438   // Pass register number to verify_oop_subroutine
 5439   const char* b = nullptr;
 5440   {
 5441     ResourceMark rm;
 5442     stringStream ss;
 5443     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5444     b = code_string(ss.as_string());
 5445   }
 5446   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5447   pushptr(buffer.addr(), rscratch1);
 5448 
 5449   // call indirectly to solve generation ordering problem
 5450   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5451   call(rax);
 5452   // Caller pops the arguments (oop, message) and restores rax, r10
 5453   BLOCK_COMMENT("} verify_oop");
 5454 }
 5455 
 5456 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5457   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5458     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5459     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5460     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5461   } else if (VM_Version::supports_avx()) {
 5462     vpcmpeqd(dst, dst, dst, vector_len);
 5463   } else {
 5464     pcmpeqd(dst, dst);
 5465   }
 5466 }
 5467 
 5468 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5469                                          int extra_slot_offset) {
 5470   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5471   int stackElementSize = Interpreter::stackElementSize;
 5472   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5473 #ifdef ASSERT
 5474   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5475   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5476 #endif
 5477   Register             scale_reg    = noreg;
 5478   Address::ScaleFactor scale_factor = Address::no_scale;
 5479   if (arg_slot.is_constant()) {
 5480     offset += arg_slot.as_constant() * stackElementSize;
 5481   } else {
 5482     scale_reg    = arg_slot.as_register();
 5483     scale_factor = Address::times(stackElementSize);
 5484   }
 5485   offset += wordSize;           // return PC is on stack
 5486   return Address(rsp, scale_reg, scale_factor, offset);
 5487 }
 5488 
 5489 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5490   if (!VerifyOops) return;
 5491 
 5492 #ifdef _LP64
 5493   push(rscratch1);
 5494 #endif
 5495   push(rax); // save rax,
 5496   // addr may contain rsp so we will have to adjust it based on the push
 5497   // we just did (and on 64 bit we do two pushes)
 5498   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5499   // stores rax into addr which is backwards of what was intended.
 5500   if (addr.uses(rsp)) {
 5501     lea(rax, addr);
 5502     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5503   } else {
 5504     pushptr(addr);
 5505   }
 5506 
 5507   // Pass register number to verify_oop_subroutine
 5508   const char* b = nullptr;
 5509   {
 5510     ResourceMark rm;
 5511     stringStream ss;
 5512     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5513     b = code_string(ss.as_string());
 5514   }
 5515   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5516   pushptr(buffer.addr(), rscratch1);
 5517 
 5518   // call indirectly to solve generation ordering problem
 5519   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5520   call(rax);
 5521   // Caller pops the arguments (addr, message) and restores rax, r10.
 5522 }
 5523 
 5524 void MacroAssembler::verify_tlab() {
 5525 #ifdef ASSERT
 5526   if (UseTLAB && VerifyOops) {
 5527     Label next, ok;
 5528     Register t1 = rsi;
 5529     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5530 
 5531     push(t1);
 5532     NOT_LP64(push(thread_reg));
 5533     NOT_LP64(get_thread(thread_reg));
 5534 
 5535     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5536     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5537     jcc(Assembler::aboveEqual, next);
 5538     STOP("assert(top >= start)");
 5539     should_not_reach_here();
 5540 
 5541     bind(next);
 5542     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5543     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5544     jcc(Assembler::aboveEqual, ok);
 5545     STOP("assert(top <= end)");
 5546     should_not_reach_here();
 5547 
 5548     bind(ok);
 5549     NOT_LP64(pop(thread_reg));
 5550     pop(t1);
 5551   }
 5552 #endif
 5553 }
 5554 
 5555 class ControlWord {
 5556  public:
 5557   int32_t _value;
 5558 
 5559   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5560   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5561   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5562   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5563   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5564   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5565   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5566   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5567 
 5568   void print() const {
 5569     // rounding control
 5570     const char* rc;
 5571     switch (rounding_control()) {
 5572       case 0: rc = "round near"; break;
 5573       case 1: rc = "round down"; break;
 5574       case 2: rc = "round up  "; break;
 5575       case 3: rc = "chop      "; break;
 5576       default:
 5577         rc = nullptr; // silence compiler warnings
 5578         fatal("Unknown rounding control: %d", rounding_control());
 5579     };
 5580     // precision control
 5581     const char* pc;
 5582     switch (precision_control()) {
 5583       case 0: pc = "24 bits "; break;
 5584       case 1: pc = "reserved"; break;
 5585       case 2: pc = "53 bits "; break;
 5586       case 3: pc = "64 bits "; break;
 5587       default:
 5588         pc = nullptr; // silence compiler warnings
 5589         fatal("Unknown precision control: %d", precision_control());
 5590     };
 5591     // flags
 5592     char f[9];
 5593     f[0] = ' ';
 5594     f[1] = ' ';
 5595     f[2] = (precision   ()) ? 'P' : 'p';
 5596     f[3] = (underflow   ()) ? 'U' : 'u';
 5597     f[4] = (overflow    ()) ? 'O' : 'o';
 5598     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5599     f[6] = (denormalized()) ? 'D' : 'd';
 5600     f[7] = (invalid     ()) ? 'I' : 'i';
 5601     f[8] = '\x0';
 5602     // output
 5603     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5604   }
 5605 
 5606 };
 5607 
 5608 class StatusWord {
 5609  public:
 5610   int32_t _value;
 5611 
 5612   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5613   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5614   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5615   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5616   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5617   int  top() const                     { return  (_value >> 11) & 7      ; }
 5618   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5619   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5620   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5621   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5622   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5623   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5624   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5625   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5626 
 5627   void print() const {
 5628     // condition codes
 5629     char c[5];
 5630     c[0] = (C3()) ? '3' : '-';
 5631     c[1] = (C2()) ? '2' : '-';
 5632     c[2] = (C1()) ? '1' : '-';
 5633     c[3] = (C0()) ? '0' : '-';
 5634     c[4] = '\x0';
 5635     // flags
 5636     char f[9];
 5637     f[0] = (error_status()) ? 'E' : '-';
 5638     f[1] = (stack_fault ()) ? 'S' : '-';
 5639     f[2] = (precision   ()) ? 'P' : '-';
 5640     f[3] = (underflow   ()) ? 'U' : '-';
 5641     f[4] = (overflow    ()) ? 'O' : '-';
 5642     f[5] = (zero_divide ()) ? 'Z' : '-';
 5643     f[6] = (denormalized()) ? 'D' : '-';
 5644     f[7] = (invalid     ()) ? 'I' : '-';
 5645     f[8] = '\x0';
 5646     // output
 5647     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5648   }
 5649 
 5650 };
 5651 
 5652 class TagWord {
 5653  public:
 5654   int32_t _value;
 5655 
 5656   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5657 
 5658   void print() const {
 5659     printf("%04x", _value & 0xFFFF);
 5660   }
 5661 
 5662 };
 5663 
 5664 class FPU_Register {
 5665  public:
 5666   int32_t _m0;
 5667   int32_t _m1;
 5668   int16_t _ex;
 5669 
 5670   bool is_indefinite() const           {
 5671     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5672   }
 5673 
 5674   void print() const {
 5675     char  sign = (_ex < 0) ? '-' : '+';
 5676     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5677     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5678   };
 5679 
 5680 };
 5681 
 5682 class FPU_State {
 5683  public:
 5684   enum {
 5685     register_size       = 10,
 5686     number_of_registers =  8,
 5687     register_mask       =  7
 5688   };
 5689 
 5690   ControlWord  _control_word;
 5691   StatusWord   _status_word;
 5692   TagWord      _tag_word;
 5693   int32_t      _error_offset;
 5694   int32_t      _error_selector;
 5695   int32_t      _data_offset;
 5696   int32_t      _data_selector;
 5697   int8_t       _register[register_size * number_of_registers];
 5698 
 5699   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5700   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5701 
 5702   const char* tag_as_string(int tag) const {
 5703     switch (tag) {
 5704       case 0: return "valid";
 5705       case 1: return "zero";
 5706       case 2: return "special";
 5707       case 3: return "empty";
 5708     }
 5709     ShouldNotReachHere();
 5710     return nullptr;
 5711   }
 5712 
 5713   void print() const {
 5714     // print computation registers
 5715     { int t = _status_word.top();
 5716       for (int i = 0; i < number_of_registers; i++) {
 5717         int j = (i - t) & register_mask;
 5718         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5719         st(j)->print();
 5720         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5721       }
 5722     }
 5723     printf("\n");
 5724     // print control registers
 5725     printf("ctrl = "); _control_word.print(); printf("\n");
 5726     printf("stat = "); _status_word .print(); printf("\n");
 5727     printf("tags = "); _tag_word    .print(); printf("\n");
 5728   }
 5729 
 5730 };
 5731 
 5732 class Flag_Register {
 5733  public:
 5734   int32_t _value;
 5735 
 5736   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5737   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5738   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5739   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5740   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5741   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5742   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5743 
 5744   void print() const {
 5745     // flags
 5746     char f[8];
 5747     f[0] = (overflow       ()) ? 'O' : '-';
 5748     f[1] = (direction      ()) ? 'D' : '-';
 5749     f[2] = (sign           ()) ? 'S' : '-';
 5750     f[3] = (zero           ()) ? 'Z' : '-';
 5751     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5752     f[5] = (parity         ()) ? 'P' : '-';
 5753     f[6] = (carry          ()) ? 'C' : '-';
 5754     f[7] = '\x0';
 5755     // output
 5756     printf("%08x  flags = %s", _value, f);
 5757   }
 5758 
 5759 };
 5760 
 5761 class IU_Register {
 5762  public:
 5763   int32_t _value;
 5764 
 5765   void print() const {
 5766     printf("%08x  %11d", _value, _value);
 5767   }
 5768 
 5769 };
 5770 
 5771 class IU_State {
 5772  public:
 5773   Flag_Register _eflags;
 5774   IU_Register   _rdi;
 5775   IU_Register   _rsi;
 5776   IU_Register   _rbp;
 5777   IU_Register   _rsp;
 5778   IU_Register   _rbx;
 5779   IU_Register   _rdx;
 5780   IU_Register   _rcx;
 5781   IU_Register   _rax;
 5782 
 5783   void print() const {
 5784     // computation registers
 5785     printf("rax,  = "); _rax.print(); printf("\n");
 5786     printf("rbx,  = "); _rbx.print(); printf("\n");
 5787     printf("rcx  = "); _rcx.print(); printf("\n");
 5788     printf("rdx  = "); _rdx.print(); printf("\n");
 5789     printf("rdi  = "); _rdi.print(); printf("\n");
 5790     printf("rsi  = "); _rsi.print(); printf("\n");
 5791     printf("rbp,  = "); _rbp.print(); printf("\n");
 5792     printf("rsp  = "); _rsp.print(); printf("\n");
 5793     printf("\n");
 5794     // control registers
 5795     printf("flgs = "); _eflags.print(); printf("\n");
 5796   }
 5797 };
 5798 
 5799 
 5800 class CPU_State {
 5801  public:
 5802   FPU_State _fpu_state;
 5803   IU_State  _iu_state;
 5804 
 5805   void print() const {
 5806     printf("--------------------------------------------------\n");
 5807     _iu_state .print();
 5808     printf("\n");
 5809     _fpu_state.print();
 5810     printf("--------------------------------------------------\n");
 5811   }
 5812 
 5813 };
 5814 
 5815 
 5816 static void _print_CPU_state(CPU_State* state) {
 5817   state->print();
 5818 };
 5819 
 5820 
 5821 void MacroAssembler::print_CPU_state() {
 5822   push_CPU_state();
 5823   push(rsp);                // pass CPU state
 5824   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5825   addptr(rsp, wordSize);       // discard argument
 5826   pop_CPU_state();
 5827 }
 5828 
 5829 
 5830 #ifndef _LP64
 5831 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5832   static int counter = 0;
 5833   FPU_State* fs = &state->_fpu_state;
 5834   counter++;
 5835   // For leaf calls, only verify that the top few elements remain empty.
 5836   // We only need 1 empty at the top for C2 code.
 5837   if( stack_depth < 0 ) {
 5838     if( fs->tag_for_st(7) != 3 ) {
 5839       printf("FPR7 not empty\n");
 5840       state->print();
 5841       assert(false, "error");
 5842       return false;
 5843     }
 5844     return true;                // All other stack states do not matter
 5845   }
 5846 
 5847   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5848          "bad FPU control word");
 5849 
 5850   // compute stack depth
 5851   int i = 0;
 5852   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5853   int d = i;
 5854   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5855   // verify findings
 5856   if (i != FPU_State::number_of_registers) {
 5857     // stack not contiguous
 5858     printf("%s: stack not contiguous at ST%d\n", s, i);
 5859     state->print();
 5860     assert(false, "error");
 5861     return false;
 5862   }
 5863   // check if computed stack depth corresponds to expected stack depth
 5864   if (stack_depth < 0) {
 5865     // expected stack depth is -stack_depth or less
 5866     if (d > -stack_depth) {
 5867       // too many elements on the stack
 5868       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5869       state->print();
 5870       assert(false, "error");
 5871       return false;
 5872     }
 5873   } else {
 5874     // expected stack depth is stack_depth
 5875     if (d != stack_depth) {
 5876       // wrong stack depth
 5877       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5878       state->print();
 5879       assert(false, "error");
 5880       return false;
 5881     }
 5882   }
 5883   // everything is cool
 5884   return true;
 5885 }
 5886 
 5887 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5888   if (!VerifyFPU) return;
 5889   push_CPU_state();
 5890   push(rsp);                // pass CPU state
 5891   ExternalAddress msg((address) s);
 5892   // pass message string s
 5893   pushptr(msg.addr(), noreg);
 5894   push(stack_depth);        // pass stack depth
 5895   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5896   addptr(rsp, 3 * wordSize);   // discard arguments
 5897   // check for error
 5898   { Label L;
 5899     testl(rax, rax);
 5900     jcc(Assembler::notZero, L);
 5901     int3();                  // break if error condition
 5902     bind(L);
 5903   }
 5904   pop_CPU_state();
 5905 }
 5906 #endif // _LP64
 5907 
 5908 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5909   // Either restore the MXCSR register after returning from the JNI Call
 5910   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5911   if (VM_Version::supports_sse()) {
 5912     if (RestoreMXCSROnJNICalls) {
 5913       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5914     } else if (CheckJNICalls) {
 5915       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5916     }
 5917   }
 5918   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5919   vzeroupper();
 5920 
 5921 #ifndef _LP64
 5922   // Either restore the x87 floating pointer control word after returning
 5923   // from the JNI call or verify that it wasn't changed.
 5924   if (CheckJNICalls) {
 5925     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5926   }
 5927 #endif // _LP64
 5928 }
 5929 
 5930 // ((OopHandle)result).resolve();
 5931 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5932   assert_different_registers(result, tmp);
 5933 
 5934   // Only 64 bit platforms support GCs that require a tmp register
 5935   // Only IN_HEAP loads require a thread_tmp register
 5936   // OopHandle::resolve is an indirection like jobject.
 5937   access_load_at(T_OBJECT, IN_NATIVE,
 5938                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5939 }
 5940 
 5941 // ((WeakHandle)result).resolve();
 5942 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5943   assert_different_registers(rresult, rtmp);
 5944   Label resolved;
 5945 
 5946   // A null weak handle resolves to null.
 5947   cmpptr(rresult, 0);
 5948   jcc(Assembler::equal, resolved);
 5949 
 5950   // Only 64 bit platforms support GCs that require a tmp register
 5951   // Only IN_HEAP loads require a thread_tmp register
 5952   // WeakHandle::resolve is an indirection like jweak.
 5953   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5954                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5955   bind(resolved);
 5956 }
 5957 
 5958 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5959   // get mirror
 5960   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5961   load_method_holder(mirror, method);
 5962   movptr(mirror, Address(mirror, mirror_offset));
 5963   resolve_oop_handle(mirror, tmp);
 5964 }
 5965 
 5966 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5967   load_method_holder(rresult, rmethod);
 5968   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5969 }
 5970 
 5971 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5972   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5973   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5974   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5975 }
 5976 
 5977 #ifdef _LP64
 5978 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
 5979   assert(UseCompactObjectHeaders, "expect compact object headers");
 5980   movl(dst, Address(src, oopDesc::mark_offset_in_bytes()));
 5981   shrl(dst, markWord::klass_shift);
 5982 }
 5983 #endif
 5984 
 5985 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5986   assert_different_registers(src, tmp);
 5987   assert_different_registers(dst, tmp);
 5988 #ifdef _LP64
 5989   if (UseCompactObjectHeaders) {
 5990     load_narrow_klass_compact(dst, src);
 5991     decode_klass_not_null(dst, tmp);
 5992   } else if (UseCompressedClassPointers) {
 5993     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5994     decode_klass_not_null(dst, tmp);
 5995   } else
 5996 #endif
 5997   {
 5998     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5999   }
 6000 }
 6001 
 6002 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 6003   assert(!UseCompactObjectHeaders, "not with compact headers");
 6004   assert_different_registers(src, tmp);
 6005   assert_different_registers(dst, tmp);
 6006 #ifdef _LP64
 6007   if (UseCompressedClassPointers) {
 6008     encode_klass_not_null(src, tmp);
 6009     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6010   } else
 6011 #endif
 6012     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6013 }
 6014 
 6015 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
 6016 #ifdef _LP64
 6017   if (UseCompactObjectHeaders) {
 6018     assert(tmp != noreg, "need tmp");
 6019     assert_different_registers(klass, obj, tmp);
 6020     load_narrow_klass_compact(tmp, obj);
 6021     cmpl(klass, tmp);
 6022   } else if (UseCompressedClassPointers) {
 6023     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6024   } else
 6025 #endif
 6026   {
 6027     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6028   }
 6029 }
 6030 
 6031 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
 6032 #ifdef _LP64
 6033   if (UseCompactObjectHeaders) {
 6034     assert(tmp2 != noreg, "need tmp2");
 6035     assert_different_registers(obj1, obj2, tmp1, tmp2);
 6036     load_narrow_klass_compact(tmp1, obj1);
 6037     load_narrow_klass_compact(tmp2, obj2);
 6038     cmpl(tmp1, tmp2);
 6039   } else if (UseCompressedClassPointers) {
 6040     movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6041     cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6042   } else
 6043 #endif
 6044   {
 6045     movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6046     cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6047   }
 6048 }
 6049 
 6050 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 6051                                     Register tmp1, Register thread_tmp) {
 6052   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6053   decorators = AccessInternal::decorator_fixup(decorators, type);
 6054   bool as_raw = (decorators & AS_RAW) != 0;
 6055   if (as_raw) {
 6056     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6057   } else {
 6058     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6059   }
 6060 }
 6061 
 6062 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 6063                                      Register tmp1, Register tmp2, Register tmp3) {
 6064   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6065   decorators = AccessInternal::decorator_fixup(decorators, type);
 6066   bool as_raw = (decorators & AS_RAW) != 0;
 6067   if (as_raw) {
 6068     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6069   } else {
 6070     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6071   }
 6072 }
 6073 
 6074 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 6075                                    Register thread_tmp, DecoratorSet decorators) {
 6076   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 6077 }
 6078 
 6079 // Doesn't do verification, generates fixed size code
 6080 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 6081                                             Register thread_tmp, DecoratorSet decorators) {
 6082   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 6083 }
 6084 
 6085 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 6086                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 6087   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 6088 }
 6089 
 6090 // Used for storing nulls.
 6091 void MacroAssembler::store_heap_oop_null(Address dst) {
 6092   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 6093 }
 6094 
 6095 #ifdef _LP64
 6096 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 6097   assert(!UseCompactObjectHeaders, "Don't use with compact headers");
 6098   if (UseCompressedClassPointers) {
 6099     // Store to klass gap in destination
 6100     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 6101   }
 6102 }
 6103 
 6104 #ifdef ASSERT
 6105 void MacroAssembler::verify_heapbase(const char* msg) {
 6106   assert (UseCompressedOops, "should be compressed");
 6107   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6108   if (CheckCompressedOops) {
 6109     Label ok;
 6110     ExternalAddress src2(CompressedOops::base_addr());
 6111     const bool is_src2_reachable = reachable(src2);
 6112     if (!is_src2_reachable) {
 6113       push(rscratch1);  // cmpptr trashes rscratch1
 6114     }
 6115     cmpptr(r12_heapbase, src2, rscratch1);
 6116     jcc(Assembler::equal, ok);
 6117     STOP(msg);
 6118     bind(ok);
 6119     if (!is_src2_reachable) {
 6120       pop(rscratch1);
 6121     }
 6122   }
 6123 }
 6124 #endif
 6125 
 6126 // Algorithm must match oop.inline.hpp encode_heap_oop.
 6127 void MacroAssembler::encode_heap_oop(Register r) {
 6128 #ifdef ASSERT
 6129   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 6130 #endif
 6131   verify_oop_msg(r, "broken oop in encode_heap_oop");
 6132   if (CompressedOops::base() == nullptr) {
 6133     if (CompressedOops::shift() != 0) {
 6134       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6135       shrq(r, LogMinObjAlignmentInBytes);
 6136     }
 6137     return;
 6138   }
 6139   testq(r, r);
 6140   cmovq(Assembler::equal, r, r12_heapbase);
 6141   subq(r, r12_heapbase);
 6142   shrq(r, LogMinObjAlignmentInBytes);
 6143 }
 6144 
 6145 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 6146 #ifdef ASSERT
 6147   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 6148   if (CheckCompressedOops) {
 6149     Label ok;
 6150     testq(r, r);
 6151     jcc(Assembler::notEqual, ok);
 6152     STOP("null oop passed to encode_heap_oop_not_null");
 6153     bind(ok);
 6154   }
 6155 #endif
 6156   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 6157   if (CompressedOops::base() != nullptr) {
 6158     subq(r, r12_heapbase);
 6159   }
 6160   if (CompressedOops::shift() != 0) {
 6161     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6162     shrq(r, LogMinObjAlignmentInBytes);
 6163   }
 6164 }
 6165 
 6166 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 6167 #ifdef ASSERT
 6168   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 6169   if (CheckCompressedOops) {
 6170     Label ok;
 6171     testq(src, src);
 6172     jcc(Assembler::notEqual, ok);
 6173     STOP("null oop passed to encode_heap_oop_not_null2");
 6174     bind(ok);
 6175   }
 6176 #endif
 6177   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 6178   if (dst != src) {
 6179     movq(dst, src);
 6180   }
 6181   if (CompressedOops::base() != nullptr) {
 6182     subq(dst, r12_heapbase);
 6183   }
 6184   if (CompressedOops::shift() != 0) {
 6185     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6186     shrq(dst, LogMinObjAlignmentInBytes);
 6187   }
 6188 }
 6189 
 6190 void  MacroAssembler::decode_heap_oop(Register r) {
 6191 #ifdef ASSERT
 6192   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 6193 #endif
 6194   if (CompressedOops::base() == nullptr) {
 6195     if (CompressedOops::shift() != 0) {
 6196       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6197       shlq(r, LogMinObjAlignmentInBytes);
 6198     }
 6199   } else {
 6200     Label done;
 6201     shlq(r, LogMinObjAlignmentInBytes);
 6202     jccb(Assembler::equal, done);
 6203     addq(r, r12_heapbase);
 6204     bind(done);
 6205   }
 6206   verify_oop_msg(r, "broken oop in decode_heap_oop");
 6207 }
 6208 
 6209 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 6210   // Note: it will change flags
 6211   assert (UseCompressedOops, "should only be used for compressed headers");
 6212   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6213   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6214   // vtableStubs also counts instructions in pd_code_size_limit.
 6215   // Also do not verify_oop as this is called by verify_oop.
 6216   if (CompressedOops::shift() != 0) {
 6217     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6218     shlq(r, LogMinObjAlignmentInBytes);
 6219     if (CompressedOops::base() != nullptr) {
 6220       addq(r, r12_heapbase);
 6221     }
 6222   } else {
 6223     assert (CompressedOops::base() == nullptr, "sanity");
 6224   }
 6225 }
 6226 
 6227 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 6228   // Note: it will change flags
 6229   assert (UseCompressedOops, "should only be used for compressed headers");
 6230   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6231   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6232   // vtableStubs also counts instructions in pd_code_size_limit.
 6233   // Also do not verify_oop as this is called by verify_oop.
 6234   if (CompressedOops::shift() != 0) {
 6235     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6236     if (LogMinObjAlignmentInBytes == Address::times_8) {
 6237       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 6238     } else {
 6239       if (dst != src) {
 6240         movq(dst, src);
 6241       }
 6242       shlq(dst, LogMinObjAlignmentInBytes);
 6243       if (CompressedOops::base() != nullptr) {
 6244         addq(dst, r12_heapbase);
 6245       }
 6246     }
 6247   } else {
 6248     assert (CompressedOops::base() == nullptr, "sanity");
 6249     if (dst != src) {
 6250       movq(dst, src);
 6251     }
 6252   }
 6253 }
 6254 
 6255 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 6256   assert_different_registers(r, tmp);
 6257   if (CompressedKlassPointers::base() != nullptr) {
 6258     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6259     subq(r, tmp);
 6260   }
 6261   if (CompressedKlassPointers::shift() != 0) {
 6262     shrq(r, CompressedKlassPointers::shift());
 6263   }
 6264 }
 6265 
 6266 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 6267   assert_different_registers(src, dst);
 6268   if (CompressedKlassPointers::base() != nullptr) {
 6269     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 6270     addq(dst, src);
 6271   } else {
 6272     movptr(dst, src);
 6273   }
 6274   if (CompressedKlassPointers::shift() != 0) {
 6275     shrq(dst, CompressedKlassPointers::shift());
 6276   }
 6277 }
 6278 
 6279 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 6280   assert_different_registers(r, tmp);
 6281   // Note: it will change flags
 6282   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 6283   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6284   // vtableStubs also counts instructions in pd_code_size_limit.
 6285   // Also do not verify_oop as this is called by verify_oop.
 6286   if (CompressedKlassPointers::shift() != 0) {
 6287     shlq(r, CompressedKlassPointers::shift());
 6288   }
 6289   if (CompressedKlassPointers::base() != nullptr) {
 6290     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6291     addq(r, tmp);
 6292   }
 6293 }
 6294 
 6295 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 6296   assert_different_registers(src, dst);
 6297   // Note: it will change flags
 6298   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6299   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6300   // vtableStubs also counts instructions in pd_code_size_limit.
 6301   // Also do not verify_oop as this is called by verify_oop.
 6302 
 6303   if (CompressedKlassPointers::base() == nullptr &&
 6304       CompressedKlassPointers::shift() == 0) {
 6305     // The best case scenario is that there is no base or shift. Then it is already
 6306     // a pointer that needs nothing but a register rename.
 6307     movl(dst, src);
 6308   } else {
 6309     if (CompressedKlassPointers::shift() <= Address::times_8) {
 6310       if (CompressedKlassPointers::base() != nullptr) {
 6311         mov64(dst, (int64_t)CompressedKlassPointers::base());
 6312       } else {
 6313         xorq(dst, dst);
 6314       }
 6315       if (CompressedKlassPointers::shift() != 0) {
 6316         assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
 6317         leaq(dst, Address(dst, src, Address::times_8, 0));
 6318       } else {
 6319         addq(dst, src);
 6320       }
 6321     } else {
 6322       if (CompressedKlassPointers::base() != nullptr) {
 6323         const uint64_t base_right_shifted =
 6324             (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
 6325         mov64(dst, base_right_shifted);
 6326       } else {
 6327         xorq(dst, dst);
 6328       }
 6329       addq(dst, src);
 6330       shlq(dst, CompressedKlassPointers::shift());
 6331     }
 6332   }
 6333 }
 6334 
 6335 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 6336   assert (UseCompressedOops, "should only be used for compressed headers");
 6337   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6338   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6339   int oop_index = oop_recorder()->find_index(obj);
 6340   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6341   mov_narrow_oop(dst, oop_index, rspec);
 6342 }
 6343 
 6344 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 6345   assert (UseCompressedOops, "should only be used for compressed headers");
 6346   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6347   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6348   int oop_index = oop_recorder()->find_index(obj);
 6349   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6350   mov_narrow_oop(dst, oop_index, rspec);
 6351 }
 6352 
 6353 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 6354   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6355   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6356   int klass_index = oop_recorder()->find_index(k);
 6357   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6358   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6359 }
 6360 
 6361 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 6362   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6363   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6364   int klass_index = oop_recorder()->find_index(k);
 6365   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6366   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6367 }
 6368 
 6369 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6370   assert (UseCompressedOops, "should only be used for compressed headers");
 6371   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6372   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6373   int oop_index = oop_recorder()->find_index(obj);
 6374   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6375   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6376 }
 6377 
 6378 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6379   assert (UseCompressedOops, "should only be used for compressed headers");
 6380   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6381   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6382   int oop_index = oop_recorder()->find_index(obj);
 6383   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6384   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6385 }
 6386 
 6387 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6388   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6389   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6390   int klass_index = oop_recorder()->find_index(k);
 6391   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6392   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6393 }
 6394 
 6395 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6396   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6397   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6398   int klass_index = oop_recorder()->find_index(k);
 6399   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6400   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6401 }
 6402 
 6403 void MacroAssembler::reinit_heapbase() {
 6404   if (UseCompressedOops) {
 6405     if (Universe::heap() != nullptr) {
 6406       if (CompressedOops::base() == nullptr) {
 6407         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6408       } else {
 6409         mov64(r12_heapbase, (int64_t)CompressedOops::base());
 6410       }
 6411     } else {
 6412       movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr()));
 6413     }
 6414   }
 6415 }
 6416 
 6417 #endif // _LP64
 6418 
 6419 #if COMPILER2_OR_JVMCI
 6420 
 6421 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6422 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6423   // cnt - number of qwords (8-byte words).
 6424   // base - start address, qword aligned.
 6425   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6426   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6427   if (use64byteVector) {
 6428     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 6429   } else if (MaxVectorSize >= 32) {
 6430     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
 6431   } else {
 6432     pxor(xtmp, xtmp);
 6433   }
 6434   jmp(L_zero_64_bytes);
 6435 
 6436   BIND(L_loop);
 6437   if (MaxVectorSize >= 32) {
 6438     fill64(base, 0, xtmp, use64byteVector);
 6439   } else {
 6440     movdqu(Address(base,  0), xtmp);
 6441     movdqu(Address(base, 16), xtmp);
 6442     movdqu(Address(base, 32), xtmp);
 6443     movdqu(Address(base, 48), xtmp);
 6444   }
 6445   addptr(base, 64);
 6446 
 6447   BIND(L_zero_64_bytes);
 6448   subptr(cnt, 8);
 6449   jccb(Assembler::greaterEqual, L_loop);
 6450 
 6451   // Copy trailing 64 bytes
 6452   if (use64byteVector) {
 6453     addptr(cnt, 8);
 6454     jccb(Assembler::equal, L_end);
 6455     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 6456     jmp(L_end);
 6457   } else {
 6458     addptr(cnt, 4);
 6459     jccb(Assembler::less, L_tail);
 6460     if (MaxVectorSize >= 32) {
 6461       vmovdqu(Address(base, 0), xtmp);
 6462     } else {
 6463       movdqu(Address(base,  0), xtmp);
 6464       movdqu(Address(base, 16), xtmp);
 6465     }
 6466   }
 6467   addptr(base, 32);
 6468   subptr(cnt, 4);
 6469 
 6470   BIND(L_tail);
 6471   addptr(cnt, 4);
 6472   jccb(Assembler::lessEqual, L_end);
 6473   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6474     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 6475   } else {
 6476     decrement(cnt);
 6477 
 6478     BIND(L_sloop);
 6479     movq(Address(base, 0), xtmp);
 6480     addptr(base, 8);
 6481     decrement(cnt);
 6482     jccb(Assembler::greaterEqual, L_sloop);
 6483   }
 6484   BIND(L_end);
 6485 }
 6486 
 6487 // Clearing constant sized memory using YMM/ZMM registers.
 6488 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6489   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6490   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6491 
 6492   int vector64_count = (cnt & (~0x7)) >> 3;
 6493   cnt = cnt & 0x7;
 6494   const int fill64_per_loop = 4;
 6495   const int max_unrolled_fill64 = 8;
 6496 
 6497   // 64 byte initialization loop.
 6498   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6499   int start64 = 0;
 6500   if (vector64_count > max_unrolled_fill64) {
 6501     Label LOOP;
 6502     Register index = rtmp;
 6503 
 6504     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6505 
 6506     movl(index, 0);
 6507     BIND(LOOP);
 6508     for (int i = 0; i < fill64_per_loop; i++) {
 6509       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6510     }
 6511     addl(index, fill64_per_loop * 64);
 6512     cmpl(index, start64 * 64);
 6513     jccb(Assembler::less, LOOP);
 6514   }
 6515   for (int i = start64; i < vector64_count; i++) {
 6516     fill64(base, i * 64, xtmp, use64byteVector);
 6517   }
 6518 
 6519   // Clear remaining 64 byte tail.
 6520   int disp = vector64_count * 64;
 6521   if (cnt) {
 6522     switch (cnt) {
 6523       case 1:
 6524         movq(Address(base, disp), xtmp);
 6525         break;
 6526       case 2:
 6527         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6528         break;
 6529       case 3:
 6530         movl(rtmp, 0x7);
 6531         kmovwl(mask, rtmp);
 6532         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6533         break;
 6534       case 4:
 6535         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6536         break;
 6537       case 5:
 6538         if (use64byteVector) {
 6539           movl(rtmp, 0x1F);
 6540           kmovwl(mask, rtmp);
 6541           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6542         } else {
 6543           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6544           movq(Address(base, disp + 32), xtmp);
 6545         }
 6546         break;
 6547       case 6:
 6548         if (use64byteVector) {
 6549           movl(rtmp, 0x3F);
 6550           kmovwl(mask, rtmp);
 6551           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6552         } else {
 6553           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6554           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6555         }
 6556         break;
 6557       case 7:
 6558         if (use64byteVector) {
 6559           movl(rtmp, 0x7F);
 6560           kmovwl(mask, rtmp);
 6561           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6562         } else {
 6563           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6564           movl(rtmp, 0x7);
 6565           kmovwl(mask, rtmp);
 6566           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6567         }
 6568         break;
 6569       default:
 6570         fatal("Unexpected length : %d\n",cnt);
 6571         break;
 6572     }
 6573   }
 6574 }
 6575 
 6576 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 6577                                bool is_large, KRegister mask) {
 6578   // cnt      - number of qwords (8-byte words).
 6579   // base     - start address, qword aligned.
 6580   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6581   assert(base==rdi, "base register must be edi for rep stos");
 6582   assert(tmp==rax,   "tmp register must be eax for rep stos");
 6583   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6584   assert(InitArrayShortSize % BytesPerLong == 0,
 6585     "InitArrayShortSize should be the multiple of BytesPerLong");
 6586 
 6587   Label DONE;
 6588   if (!is_large || !UseXMMForObjInit) {
 6589     xorptr(tmp, tmp);
 6590   }
 6591 
 6592   if (!is_large) {
 6593     Label LOOP, LONG;
 6594     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6595     jccb(Assembler::greater, LONG);
 6596 
 6597     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6598 
 6599     decrement(cnt);
 6600     jccb(Assembler::negative, DONE); // Zero length
 6601 
 6602     // Use individual pointer-sized stores for small counts:
 6603     BIND(LOOP);
 6604     movptr(Address(base, cnt, Address::times_ptr), tmp);
 6605     decrement(cnt);
 6606     jccb(Assembler::greaterEqual, LOOP);
 6607     jmpb(DONE);
 6608 
 6609     BIND(LONG);
 6610   }
 6611 
 6612   // Use longer rep-prefixed ops for non-small counts:
 6613   if (UseFastStosb) {
 6614     shlptr(cnt, 3); // convert to number of bytes
 6615     rep_stosb();
 6616   } else if (UseXMMForObjInit) {
 6617     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 6618   } else {
 6619     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6620     rep_stos();
 6621   }
 6622 
 6623   BIND(DONE);
 6624 }
 6625 
 6626 #endif //COMPILER2_OR_JVMCI
 6627 
 6628 
 6629 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6630                                    Register to, Register value, Register count,
 6631                                    Register rtmp, XMMRegister xtmp) {
 6632   ShortBranchVerifier sbv(this);
 6633   assert_different_registers(to, value, count, rtmp);
 6634   Label L_exit;
 6635   Label L_fill_2_bytes, L_fill_4_bytes;
 6636 
 6637 #if defined(COMPILER2) && defined(_LP64)
 6638   if(MaxVectorSize >=32 &&
 6639      VM_Version::supports_avx512vlbw() &&
 6640      VM_Version::supports_bmi2()) {
 6641     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6642     return;
 6643   }
 6644 #endif
 6645 
 6646   int shift = -1;
 6647   switch (t) {
 6648     case T_BYTE:
 6649       shift = 2;
 6650       break;
 6651     case T_SHORT:
 6652       shift = 1;
 6653       break;
 6654     case T_INT:
 6655       shift = 0;
 6656       break;
 6657     default: ShouldNotReachHere();
 6658   }
 6659 
 6660   if (t == T_BYTE) {
 6661     andl(value, 0xff);
 6662     movl(rtmp, value);
 6663     shll(rtmp, 8);
 6664     orl(value, rtmp);
 6665   }
 6666   if (t == T_SHORT) {
 6667     andl(value, 0xffff);
 6668   }
 6669   if (t == T_BYTE || t == T_SHORT) {
 6670     movl(rtmp, value);
 6671     shll(rtmp, 16);
 6672     orl(value, rtmp);
 6673   }
 6674 
 6675   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6676   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6677   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6678     Label L_skip_align2;
 6679     // align source address at 4 bytes address boundary
 6680     if (t == T_BYTE) {
 6681       Label L_skip_align1;
 6682       // One byte misalignment happens only for byte arrays
 6683       testptr(to, 1);
 6684       jccb(Assembler::zero, L_skip_align1);
 6685       movb(Address(to, 0), value);
 6686       increment(to);
 6687       decrement(count);
 6688       BIND(L_skip_align1);
 6689     }
 6690     // Two bytes misalignment happens only for byte and short (char) arrays
 6691     testptr(to, 2);
 6692     jccb(Assembler::zero, L_skip_align2);
 6693     movw(Address(to, 0), value);
 6694     addptr(to, 2);
 6695     subptr(count, 1<<(shift-1));
 6696     BIND(L_skip_align2);
 6697   }
 6698   if (UseSSE < 2) {
 6699     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6700     // Fill 32-byte chunks
 6701     subptr(count, 8 << shift);
 6702     jcc(Assembler::less, L_check_fill_8_bytes);
 6703     align(16);
 6704 
 6705     BIND(L_fill_32_bytes_loop);
 6706 
 6707     for (int i = 0; i < 32; i += 4) {
 6708       movl(Address(to, i), value);
 6709     }
 6710 
 6711     addptr(to, 32);
 6712     subptr(count, 8 << shift);
 6713     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6714     BIND(L_check_fill_8_bytes);
 6715     addptr(count, 8 << shift);
 6716     jccb(Assembler::zero, L_exit);
 6717     jmpb(L_fill_8_bytes);
 6718 
 6719     //
 6720     // length is too short, just fill qwords
 6721     //
 6722     BIND(L_fill_8_bytes_loop);
 6723     movl(Address(to, 0), value);
 6724     movl(Address(to, 4), value);
 6725     addptr(to, 8);
 6726     BIND(L_fill_8_bytes);
 6727     subptr(count, 1 << (shift + 1));
 6728     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6729     // fall through to fill 4 bytes
 6730   } else {
 6731     Label L_fill_32_bytes;
 6732     if (!UseUnalignedLoadStores) {
 6733       // align to 8 bytes, we know we are 4 byte aligned to start
 6734       testptr(to, 4);
 6735       jccb(Assembler::zero, L_fill_32_bytes);
 6736       movl(Address(to, 0), value);
 6737       addptr(to, 4);
 6738       subptr(count, 1<<shift);
 6739     }
 6740     BIND(L_fill_32_bytes);
 6741     {
 6742       assert( UseSSE >= 2, "supported cpu only" );
 6743       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6744       movdl(xtmp, value);
 6745       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6746         Label L_check_fill_32_bytes;
 6747         if (UseAVX > 2) {
 6748           // Fill 64-byte chunks
 6749           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6750 
 6751           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6752           cmpptr(count, VM_Version::avx3_threshold());
 6753           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6754 
 6755           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6756 
 6757           subptr(count, 16 << shift);
 6758           jccb(Assembler::less, L_check_fill_32_bytes);
 6759           align(16);
 6760 
 6761           BIND(L_fill_64_bytes_loop_avx3);
 6762           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6763           addptr(to, 64);
 6764           subptr(count, 16 << shift);
 6765           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6766           jmpb(L_check_fill_32_bytes);
 6767 
 6768           BIND(L_check_fill_64_bytes_avx2);
 6769         }
 6770         // Fill 64-byte chunks
 6771         Label L_fill_64_bytes_loop;
 6772         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6773 
 6774         subptr(count, 16 << shift);
 6775         jcc(Assembler::less, L_check_fill_32_bytes);
 6776         align(16);
 6777 
 6778         BIND(L_fill_64_bytes_loop);
 6779         vmovdqu(Address(to, 0), xtmp);
 6780         vmovdqu(Address(to, 32), xtmp);
 6781         addptr(to, 64);
 6782         subptr(count, 16 << shift);
 6783         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6784 
 6785         BIND(L_check_fill_32_bytes);
 6786         addptr(count, 8 << shift);
 6787         jccb(Assembler::less, L_check_fill_8_bytes);
 6788         vmovdqu(Address(to, 0), xtmp);
 6789         addptr(to, 32);
 6790         subptr(count, 8 << shift);
 6791 
 6792         BIND(L_check_fill_8_bytes);
 6793         // clean upper bits of YMM registers
 6794         movdl(xtmp, value);
 6795         pshufd(xtmp, xtmp, 0);
 6796       } else {
 6797         // Fill 32-byte chunks
 6798         pshufd(xtmp, xtmp, 0);
 6799 
 6800         subptr(count, 8 << shift);
 6801         jcc(Assembler::less, L_check_fill_8_bytes);
 6802         align(16);
 6803 
 6804         BIND(L_fill_32_bytes_loop);
 6805 
 6806         if (UseUnalignedLoadStores) {
 6807           movdqu(Address(to, 0), xtmp);
 6808           movdqu(Address(to, 16), xtmp);
 6809         } else {
 6810           movq(Address(to, 0), xtmp);
 6811           movq(Address(to, 8), xtmp);
 6812           movq(Address(to, 16), xtmp);
 6813           movq(Address(to, 24), xtmp);
 6814         }
 6815 
 6816         addptr(to, 32);
 6817         subptr(count, 8 << shift);
 6818         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6819 
 6820         BIND(L_check_fill_8_bytes);
 6821       }
 6822       addptr(count, 8 << shift);
 6823       jccb(Assembler::zero, L_exit);
 6824       jmpb(L_fill_8_bytes);
 6825 
 6826       //
 6827       // length is too short, just fill qwords
 6828       //
 6829       BIND(L_fill_8_bytes_loop);
 6830       movq(Address(to, 0), xtmp);
 6831       addptr(to, 8);
 6832       BIND(L_fill_8_bytes);
 6833       subptr(count, 1 << (shift + 1));
 6834       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6835     }
 6836   }
 6837   // fill trailing 4 bytes
 6838   BIND(L_fill_4_bytes);
 6839   testl(count, 1<<shift);
 6840   jccb(Assembler::zero, L_fill_2_bytes);
 6841   movl(Address(to, 0), value);
 6842   if (t == T_BYTE || t == T_SHORT) {
 6843     Label L_fill_byte;
 6844     addptr(to, 4);
 6845     BIND(L_fill_2_bytes);
 6846     // fill trailing 2 bytes
 6847     testl(count, 1<<(shift-1));
 6848     jccb(Assembler::zero, L_fill_byte);
 6849     movw(Address(to, 0), value);
 6850     if (t == T_BYTE) {
 6851       addptr(to, 2);
 6852       BIND(L_fill_byte);
 6853       // fill trailing byte
 6854       testl(count, 1);
 6855       jccb(Assembler::zero, L_exit);
 6856       movb(Address(to, 0), value);
 6857     } else {
 6858       BIND(L_fill_byte);
 6859     }
 6860   } else {
 6861     BIND(L_fill_2_bytes);
 6862   }
 6863   BIND(L_exit);
 6864 }
 6865 
 6866 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6867   switch(type) {
 6868     case T_BYTE:
 6869     case T_BOOLEAN:
 6870       evpbroadcastb(dst, src, vector_len);
 6871       break;
 6872     case T_SHORT:
 6873     case T_CHAR:
 6874       evpbroadcastw(dst, src, vector_len);
 6875       break;
 6876     case T_INT:
 6877     case T_FLOAT:
 6878       evpbroadcastd(dst, src, vector_len);
 6879       break;
 6880     case T_LONG:
 6881     case T_DOUBLE:
 6882       evpbroadcastq(dst, src, vector_len);
 6883       break;
 6884     default:
 6885       fatal("Unhandled type : %s", type2name(type));
 6886       break;
 6887   }
 6888 }
 6889 
 6890 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6891    //@IntrinsicCandidate
 6892    //private static int implEncodeISOArray(byte[] sa, int sp,
 6893    //byte[] da, int dp, int len) {
 6894    //  int i = 0;
 6895    //  for (; i < len; i++) {
 6896    //    char c = StringUTF16.getChar(sa, sp++);
 6897    //    if (c > '\u00FF')
 6898    //      break;
 6899    //    da[dp++] = (byte)c;
 6900    //  }
 6901    //  return i;
 6902    //}
 6903    //
 6904    //@IntrinsicCandidate
 6905    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6906    //    byte[] da, int dp, int len) {
 6907    //  int i = 0;
 6908    //  for (; i < len; i++) {
 6909    //    char c = sa[sp++];
 6910    //    if (c >= '\u0080')
 6911    //      break;
 6912    //    da[dp++] = (byte)c;
 6913    //  }
 6914    //  return i;
 6915    //}
 6916 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6917   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6918   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6919   Register tmp5, Register result, bool ascii) {
 6920 
 6921   // rsi: src
 6922   // rdi: dst
 6923   // rdx: len
 6924   // rcx: tmp5
 6925   // rax: result
 6926   ShortBranchVerifier sbv(this);
 6927   assert_different_registers(src, dst, len, tmp5, result);
 6928   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6929 
 6930   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6931   int short_mask = ascii ? 0xff80 : 0xff00;
 6932 
 6933   // set result
 6934   xorl(result, result);
 6935   // check for zero length
 6936   testl(len, len);
 6937   jcc(Assembler::zero, L_done);
 6938 
 6939   movl(result, len);
 6940 
 6941   // Setup pointers
 6942   lea(src, Address(src, len, Address::times_2)); // char[]
 6943   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6944   negptr(len);
 6945 
 6946   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6947     Label L_copy_8_chars, L_copy_8_chars_exit;
 6948     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6949 
 6950     if (UseAVX >= 2) {
 6951       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6952       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6953       movdl(tmp1Reg, tmp5);
 6954       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 6955       jmp(L_chars_32_check);
 6956 
 6957       bind(L_copy_32_chars);
 6958       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 6959       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 6960       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6961       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6962       jccb(Assembler::notZero, L_copy_32_chars_exit);
 6963       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6964       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 6965       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 6966 
 6967       bind(L_chars_32_check);
 6968       addptr(len, 32);
 6969       jcc(Assembler::lessEqual, L_copy_32_chars);
 6970 
 6971       bind(L_copy_32_chars_exit);
 6972       subptr(len, 16);
 6973       jccb(Assembler::greater, L_copy_16_chars_exit);
 6974 
 6975     } else if (UseSSE42Intrinsics) {
 6976       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6977       movdl(tmp1Reg, tmp5);
 6978       pshufd(tmp1Reg, tmp1Reg, 0);
 6979       jmpb(L_chars_16_check);
 6980     }
 6981 
 6982     bind(L_copy_16_chars);
 6983     if (UseAVX >= 2) {
 6984       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 6985       vptest(tmp2Reg, tmp1Reg);
 6986       jcc(Assembler::notZero, L_copy_16_chars_exit);
 6987       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 6988       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 6989     } else {
 6990       if (UseAVX > 0) {
 6991         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6992         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6993         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 6994       } else {
 6995         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6996         por(tmp2Reg, tmp3Reg);
 6997         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6998         por(tmp2Reg, tmp4Reg);
 6999       }
 7000       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7001       jccb(Assembler::notZero, L_copy_16_chars_exit);
 7002       packuswb(tmp3Reg, tmp4Reg);
 7003     }
 7004     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 7005 
 7006     bind(L_chars_16_check);
 7007     addptr(len, 16);
 7008     jcc(Assembler::lessEqual, L_copy_16_chars);
 7009 
 7010     bind(L_copy_16_chars_exit);
 7011     if (UseAVX >= 2) {
 7012       // clean upper bits of YMM registers
 7013       vpxor(tmp2Reg, tmp2Reg);
 7014       vpxor(tmp3Reg, tmp3Reg);
 7015       vpxor(tmp4Reg, tmp4Reg);
 7016       movdl(tmp1Reg, tmp5);
 7017       pshufd(tmp1Reg, tmp1Reg, 0);
 7018     }
 7019     subptr(len, 8);
 7020     jccb(Assembler::greater, L_copy_8_chars_exit);
 7021 
 7022     bind(L_copy_8_chars);
 7023     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 7024     ptest(tmp3Reg, tmp1Reg);
 7025     jccb(Assembler::notZero, L_copy_8_chars_exit);
 7026     packuswb(tmp3Reg, tmp1Reg);
 7027     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 7028     addptr(len, 8);
 7029     jccb(Assembler::lessEqual, L_copy_8_chars);
 7030 
 7031     bind(L_copy_8_chars_exit);
 7032     subptr(len, 8);
 7033     jccb(Assembler::zero, L_done);
 7034   }
 7035 
 7036   bind(L_copy_1_char);
 7037   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 7038   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 7039   jccb(Assembler::notZero, L_copy_1_char_exit);
 7040   movb(Address(dst, len, Address::times_1, 0), tmp5);
 7041   addptr(len, 1);
 7042   jccb(Assembler::less, L_copy_1_char);
 7043 
 7044   bind(L_copy_1_char_exit);
 7045   addptr(result, len); // len is negative count of not processed elements
 7046 
 7047   bind(L_done);
 7048 }
 7049 
 7050 #ifdef _LP64
 7051 /**
 7052  * Helper for multiply_to_len().
 7053  */
 7054 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 7055   addq(dest_lo, src1);
 7056   adcq(dest_hi, 0);
 7057   addq(dest_lo, src2);
 7058   adcq(dest_hi, 0);
 7059 }
 7060 
 7061 /**
 7062  * Multiply 64 bit by 64 bit first loop.
 7063  */
 7064 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 7065                                            Register y, Register y_idx, Register z,
 7066                                            Register carry, Register product,
 7067                                            Register idx, Register kdx) {
 7068   //
 7069   //  jlong carry, x[], y[], z[];
 7070   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7071   //    huge_128 product = y[idx] * x[xstart] + carry;
 7072   //    z[kdx] = (jlong)product;
 7073   //    carry  = (jlong)(product >>> 64);
 7074   //  }
 7075   //  z[xstart] = carry;
 7076   //
 7077 
 7078   Label L_first_loop, L_first_loop_exit;
 7079   Label L_one_x, L_one_y, L_multiply;
 7080 
 7081   decrementl(xstart);
 7082   jcc(Assembler::negative, L_one_x);
 7083 
 7084   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7085   rorq(x_xstart, 32); // convert big-endian to little-endian
 7086 
 7087   bind(L_first_loop);
 7088   decrementl(idx);
 7089   jcc(Assembler::negative, L_first_loop_exit);
 7090   decrementl(idx);
 7091   jcc(Assembler::negative, L_one_y);
 7092   movq(y_idx, Address(y, idx, Address::times_4,  0));
 7093   rorq(y_idx, 32); // convert big-endian to little-endian
 7094   bind(L_multiply);
 7095   movq(product, x_xstart);
 7096   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 7097   addq(product, carry);
 7098   adcq(rdx, 0);
 7099   subl(kdx, 2);
 7100   movl(Address(z, kdx, Address::times_4,  4), product);
 7101   shrq(product, 32);
 7102   movl(Address(z, kdx, Address::times_4,  0), product);
 7103   movq(carry, rdx);
 7104   jmp(L_first_loop);
 7105 
 7106   bind(L_one_y);
 7107   movl(y_idx, Address(y,  0));
 7108   jmp(L_multiply);
 7109 
 7110   bind(L_one_x);
 7111   movl(x_xstart, Address(x,  0));
 7112   jmp(L_first_loop);
 7113 
 7114   bind(L_first_loop_exit);
 7115 }
 7116 
 7117 /**
 7118  * Multiply 64 bit by 64 bit and add 128 bit.
 7119  */
 7120 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 7121                                             Register yz_idx, Register idx,
 7122                                             Register carry, Register product, int offset) {
 7123   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 7124   //     z[kdx] = (jlong)product;
 7125 
 7126   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 7127   rorq(yz_idx, 32); // convert big-endian to little-endian
 7128   movq(product, x_xstart);
 7129   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 7130   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 7131   rorq(yz_idx, 32); // convert big-endian to little-endian
 7132 
 7133   add2_with_carry(rdx, product, carry, yz_idx);
 7134 
 7135   movl(Address(z, idx, Address::times_4,  offset+4), product);
 7136   shrq(product, 32);
 7137   movl(Address(z, idx, Address::times_4,  offset), product);
 7138 
 7139 }
 7140 
 7141 /**
 7142  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 7143  */
 7144 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 7145                                              Register yz_idx, Register idx, Register jdx,
 7146                                              Register carry, Register product,
 7147                                              Register carry2) {
 7148   //   jlong carry, x[], y[], z[];
 7149   //   int kdx = ystart+1;
 7150   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7151   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 7152   //     z[kdx+idx+1] = (jlong)product;
 7153   //     jlong carry2  = (jlong)(product >>> 64);
 7154   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 7155   //     z[kdx+idx] = (jlong)product;
 7156   //     carry  = (jlong)(product >>> 64);
 7157   //   }
 7158   //   idx += 2;
 7159   //   if (idx > 0) {
 7160   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 7161   //     z[kdx+idx] = (jlong)product;
 7162   //     carry  = (jlong)(product >>> 64);
 7163   //   }
 7164   //
 7165 
 7166   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7167 
 7168   movl(jdx, idx);
 7169   andl(jdx, 0xFFFFFFFC);
 7170   shrl(jdx, 2);
 7171 
 7172   bind(L_third_loop);
 7173   subl(jdx, 1);
 7174   jcc(Assembler::negative, L_third_loop_exit);
 7175   subl(idx, 4);
 7176 
 7177   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 7178   movq(carry2, rdx);
 7179 
 7180   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 7181   movq(carry, rdx);
 7182   jmp(L_third_loop);
 7183 
 7184   bind (L_third_loop_exit);
 7185 
 7186   andl (idx, 0x3);
 7187   jcc(Assembler::zero, L_post_third_loop_done);
 7188 
 7189   Label L_check_1;
 7190   subl(idx, 2);
 7191   jcc(Assembler::negative, L_check_1);
 7192 
 7193   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 7194   movq(carry, rdx);
 7195 
 7196   bind (L_check_1);
 7197   addl (idx, 0x2);
 7198   andl (idx, 0x1);
 7199   subl(idx, 1);
 7200   jcc(Assembler::negative, L_post_third_loop_done);
 7201 
 7202   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 7203   movq(product, x_xstart);
 7204   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 7205   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 7206 
 7207   add2_with_carry(rdx, product, yz_idx, carry);
 7208 
 7209   movl(Address(z, idx, Address::times_4,  0), product);
 7210   shrq(product, 32);
 7211 
 7212   shlq(rdx, 32);
 7213   orq(product, rdx);
 7214   movq(carry, product);
 7215 
 7216   bind(L_post_third_loop_done);
 7217 }
 7218 
 7219 /**
 7220  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 7221  *
 7222  */
 7223 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 7224                                                   Register carry, Register carry2,
 7225                                                   Register idx, Register jdx,
 7226                                                   Register yz_idx1, Register yz_idx2,
 7227                                                   Register tmp, Register tmp3, Register tmp4) {
 7228   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 7229 
 7230   //   jlong carry, x[], y[], z[];
 7231   //   int kdx = ystart+1;
 7232   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7233   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 7234   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 7235   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 7236   //     carry  = (jlong)(tmp4 >>> 64);
 7237   //     z[kdx+idx+1] = (jlong)tmp3;
 7238   //     z[kdx+idx] = (jlong)tmp4;
 7239   //   }
 7240   //   idx += 2;
 7241   //   if (idx > 0) {
 7242   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 7243   //     z[kdx+idx] = (jlong)yz_idx1;
 7244   //     carry  = (jlong)(yz_idx1 >>> 64);
 7245   //   }
 7246   //
 7247 
 7248   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7249 
 7250   movl(jdx, idx);
 7251   andl(jdx, 0xFFFFFFFC);
 7252   shrl(jdx, 2);
 7253 
 7254   bind(L_third_loop);
 7255   subl(jdx, 1);
 7256   jcc(Assembler::negative, L_third_loop_exit);
 7257   subl(idx, 4);
 7258 
 7259   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 7260   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 7261   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 7262   rorxq(yz_idx2, yz_idx2, 32);
 7263 
 7264   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 7265   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 7266 
 7267   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 7268   rorxq(yz_idx1, yz_idx1, 32);
 7269   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7270   rorxq(yz_idx2, yz_idx2, 32);
 7271 
 7272   if (VM_Version::supports_adx()) {
 7273     adcxq(tmp3, carry);
 7274     adoxq(tmp3, yz_idx1);
 7275 
 7276     adcxq(tmp4, tmp);
 7277     adoxq(tmp4, yz_idx2);
 7278 
 7279     movl(carry, 0); // does not affect flags
 7280     adcxq(carry2, carry);
 7281     adoxq(carry2, carry);
 7282   } else {
 7283     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 7284     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 7285   }
 7286   movq(carry, carry2);
 7287 
 7288   movl(Address(z, idx, Address::times_4, 12), tmp3);
 7289   shrq(tmp3, 32);
 7290   movl(Address(z, idx, Address::times_4,  8), tmp3);
 7291 
 7292   movl(Address(z, idx, Address::times_4,  4), tmp4);
 7293   shrq(tmp4, 32);
 7294   movl(Address(z, idx, Address::times_4,  0), tmp4);
 7295 
 7296   jmp(L_third_loop);
 7297 
 7298   bind (L_third_loop_exit);
 7299 
 7300   andl (idx, 0x3);
 7301   jcc(Assembler::zero, L_post_third_loop_done);
 7302 
 7303   Label L_check_1;
 7304   subl(idx, 2);
 7305   jcc(Assembler::negative, L_check_1);
 7306 
 7307   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 7308   rorxq(yz_idx1, yz_idx1, 32);
 7309   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 7310   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7311   rorxq(yz_idx2, yz_idx2, 32);
 7312 
 7313   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 7314 
 7315   movl(Address(z, idx, Address::times_4,  4), tmp3);
 7316   shrq(tmp3, 32);
 7317   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7318   movq(carry, tmp4);
 7319 
 7320   bind (L_check_1);
 7321   addl (idx, 0x2);
 7322   andl (idx, 0x1);
 7323   subl(idx, 1);
 7324   jcc(Assembler::negative, L_post_third_loop_done);
 7325   movl(tmp4, Address(y, idx, Address::times_4,  0));
 7326   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 7327   movl(tmp4, Address(z, idx, Address::times_4,  0));
 7328 
 7329   add2_with_carry(carry2, tmp3, tmp4, carry);
 7330 
 7331   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7332   shrq(tmp3, 32);
 7333 
 7334   shlq(carry2, 32);
 7335   orq(tmp3, carry2);
 7336   movq(carry, tmp3);
 7337 
 7338   bind(L_post_third_loop_done);
 7339 }
 7340 
 7341 /**
 7342  * Code for BigInteger::multiplyToLen() intrinsic.
 7343  *
 7344  * rdi: x
 7345  * rax: xlen
 7346  * rsi: y
 7347  * rcx: ylen
 7348  * r8:  z
 7349  * r11: tmp0
 7350  * r12: tmp1
 7351  * r13: tmp2
 7352  * r14: tmp3
 7353  * r15: tmp4
 7354  * rbx: tmp5
 7355  *
 7356  */
 7357 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
 7358                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 7359   ShortBranchVerifier sbv(this);
 7360   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 7361 
 7362   push(tmp0);
 7363   push(tmp1);
 7364   push(tmp2);
 7365   push(tmp3);
 7366   push(tmp4);
 7367   push(tmp5);
 7368 
 7369   push(xlen);
 7370 
 7371   const Register idx = tmp1;
 7372   const Register kdx = tmp2;
 7373   const Register xstart = tmp3;
 7374 
 7375   const Register y_idx = tmp4;
 7376   const Register carry = tmp5;
 7377   const Register product  = xlen;
 7378   const Register x_xstart = tmp0;
 7379 
 7380   // First Loop.
 7381   //
 7382   //  final static long LONG_MASK = 0xffffffffL;
 7383   //  int xstart = xlen - 1;
 7384   //  int ystart = ylen - 1;
 7385   //  long carry = 0;
 7386   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7387   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7388   //    z[kdx] = (int)product;
 7389   //    carry = product >>> 32;
 7390   //  }
 7391   //  z[xstart] = (int)carry;
 7392   //
 7393 
 7394   movl(idx, ylen);               // idx = ylen;
 7395   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
 7396   xorq(carry, carry);            // carry = 0;
 7397 
 7398   Label L_done;
 7399 
 7400   movl(xstart, xlen);
 7401   decrementl(xstart);
 7402   jcc(Assembler::negative, L_done);
 7403 
 7404   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7405 
 7406   Label L_second_loop;
 7407   testl(kdx, kdx);
 7408   jcc(Assembler::zero, L_second_loop);
 7409 
 7410   Label L_carry;
 7411   subl(kdx, 1);
 7412   jcc(Assembler::zero, L_carry);
 7413 
 7414   movl(Address(z, kdx, Address::times_4,  0), carry);
 7415   shrq(carry, 32);
 7416   subl(kdx, 1);
 7417 
 7418   bind(L_carry);
 7419   movl(Address(z, kdx, Address::times_4,  0), carry);
 7420 
 7421   // Second and third (nested) loops.
 7422   //
 7423   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7424   //   carry = 0;
 7425   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7426   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7427   //                    (z[k] & LONG_MASK) + carry;
 7428   //     z[k] = (int)product;
 7429   //     carry = product >>> 32;
 7430   //   }
 7431   //   z[i] = (int)carry;
 7432   // }
 7433   //
 7434   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7435 
 7436   const Register jdx = tmp1;
 7437 
 7438   bind(L_second_loop);
 7439   xorl(carry, carry);    // carry = 0;
 7440   movl(jdx, ylen);       // j = ystart+1
 7441 
 7442   subl(xstart, 1);       // i = xstart-1;
 7443   jcc(Assembler::negative, L_done);
 7444 
 7445   push (z);
 7446 
 7447   Label L_last_x;
 7448   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7449   subl(xstart, 1);       // i = xstart-1;
 7450   jcc(Assembler::negative, L_last_x);
 7451 
 7452   if (UseBMI2Instructions) {
 7453     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7454     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7455   } else {
 7456     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7457     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7458   }
 7459 
 7460   Label L_third_loop_prologue;
 7461   bind(L_third_loop_prologue);
 7462 
 7463   push (x);
 7464   push (xstart);
 7465   push (ylen);
 7466 
 7467 
 7468   if (UseBMI2Instructions) {
 7469     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7470   } else { // !UseBMI2Instructions
 7471     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7472   }
 7473 
 7474   pop(ylen);
 7475   pop(xlen);
 7476   pop(x);
 7477   pop(z);
 7478 
 7479   movl(tmp3, xlen);
 7480   addl(tmp3, 1);
 7481   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7482   subl(tmp3, 1);
 7483   jccb(Assembler::negative, L_done);
 7484 
 7485   shrq(carry, 32);
 7486   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7487   jmp(L_second_loop);
 7488 
 7489   // Next infrequent code is moved outside loops.
 7490   bind(L_last_x);
 7491   if (UseBMI2Instructions) {
 7492     movl(rdx, Address(x,  0));
 7493   } else {
 7494     movl(x_xstart, Address(x,  0));
 7495   }
 7496   jmp(L_third_loop_prologue);
 7497 
 7498   bind(L_done);
 7499 
 7500   pop(xlen);
 7501 
 7502   pop(tmp5);
 7503   pop(tmp4);
 7504   pop(tmp3);
 7505   pop(tmp2);
 7506   pop(tmp1);
 7507   pop(tmp0);
 7508 }
 7509 
 7510 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7511   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7512   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7513   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7514   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7515   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7516   Label SAME_TILL_END, DONE;
 7517   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7518 
 7519   //scale is in rcx in both Win64 and Unix
 7520   ShortBranchVerifier sbv(this);
 7521 
 7522   shlq(length);
 7523   xorq(result, result);
 7524 
 7525   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7526       VM_Version::supports_avx512vlbw()) {
 7527     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7528 
 7529     cmpq(length, 64);
 7530     jcc(Assembler::less, VECTOR32_TAIL);
 7531 
 7532     movq(tmp1, length);
 7533     andq(tmp1, 0x3F);      // tail count
 7534     andq(length, ~(0x3F)); //vector count
 7535 
 7536     bind(VECTOR64_LOOP);
 7537     // AVX512 code to compare 64 byte vectors.
 7538     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7539     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7540     kortestql(k7, k7);
 7541     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7542     addq(result, 64);
 7543     subq(length, 64);
 7544     jccb(Assembler::notZero, VECTOR64_LOOP);
 7545 
 7546     //bind(VECTOR64_TAIL);
 7547     testq(tmp1, tmp1);
 7548     jcc(Assembler::zero, SAME_TILL_END);
 7549 
 7550     //bind(VECTOR64_TAIL);
 7551     // AVX512 code to compare up to 63 byte vectors.
 7552     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7553     shlxq(tmp2, tmp2, tmp1);
 7554     notq(tmp2);
 7555     kmovql(k3, tmp2);
 7556 
 7557     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7558     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7559 
 7560     ktestql(k7, k3);
 7561     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7562 
 7563     bind(VECTOR64_NOT_EQUAL);
 7564     kmovql(tmp1, k7);
 7565     notq(tmp1);
 7566     tzcntq(tmp1, tmp1);
 7567     addq(result, tmp1);
 7568     shrq(result);
 7569     jmp(DONE);
 7570     bind(VECTOR32_TAIL);
 7571   }
 7572 
 7573   cmpq(length, 8);
 7574   jcc(Assembler::equal, VECTOR8_LOOP);
 7575   jcc(Assembler::less, VECTOR4_TAIL);
 7576 
 7577   if (UseAVX >= 2) {
 7578     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7579 
 7580     cmpq(length, 16);
 7581     jcc(Assembler::equal, VECTOR16_LOOP);
 7582     jcc(Assembler::less, VECTOR8_LOOP);
 7583 
 7584     cmpq(length, 32);
 7585     jccb(Assembler::less, VECTOR16_TAIL);
 7586 
 7587     subq(length, 32);
 7588     bind(VECTOR32_LOOP);
 7589     vmovdqu(rymm0, Address(obja, result));
 7590     vmovdqu(rymm1, Address(objb, result));
 7591     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7592     vptest(rymm2, rymm2);
 7593     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7594     addq(result, 32);
 7595     subq(length, 32);
 7596     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7597     addq(length, 32);
 7598     jcc(Assembler::equal, SAME_TILL_END);
 7599     //falling through if less than 32 bytes left //close the branch here.
 7600 
 7601     bind(VECTOR16_TAIL);
 7602     cmpq(length, 16);
 7603     jccb(Assembler::less, VECTOR8_TAIL);
 7604     bind(VECTOR16_LOOP);
 7605     movdqu(rymm0, Address(obja, result));
 7606     movdqu(rymm1, Address(objb, result));
 7607     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7608     ptest(rymm2, rymm2);
 7609     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7610     addq(result, 16);
 7611     subq(length, 16);
 7612     jcc(Assembler::equal, SAME_TILL_END);
 7613     //falling through if less than 16 bytes left
 7614   } else {//regular intrinsics
 7615 
 7616     cmpq(length, 16);
 7617     jccb(Assembler::less, VECTOR8_TAIL);
 7618 
 7619     subq(length, 16);
 7620     bind(VECTOR16_LOOP);
 7621     movdqu(rymm0, Address(obja, result));
 7622     movdqu(rymm1, Address(objb, result));
 7623     pxor(rymm0, rymm1);
 7624     ptest(rymm0, rymm0);
 7625     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7626     addq(result, 16);
 7627     subq(length, 16);
 7628     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7629     addq(length, 16);
 7630     jcc(Assembler::equal, SAME_TILL_END);
 7631     //falling through if less than 16 bytes left
 7632   }
 7633 
 7634   bind(VECTOR8_TAIL);
 7635   cmpq(length, 8);
 7636   jccb(Assembler::less, VECTOR4_TAIL);
 7637   bind(VECTOR8_LOOP);
 7638   movq(tmp1, Address(obja, result));
 7639   movq(tmp2, Address(objb, result));
 7640   xorq(tmp1, tmp2);
 7641   testq(tmp1, tmp1);
 7642   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7643   addq(result, 8);
 7644   subq(length, 8);
 7645   jcc(Assembler::equal, SAME_TILL_END);
 7646   //falling through if less than 8 bytes left
 7647 
 7648   bind(VECTOR4_TAIL);
 7649   cmpq(length, 4);
 7650   jccb(Assembler::less, BYTES_TAIL);
 7651   bind(VECTOR4_LOOP);
 7652   movl(tmp1, Address(obja, result));
 7653   xorl(tmp1, Address(objb, result));
 7654   testl(tmp1, tmp1);
 7655   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7656   addq(result, 4);
 7657   subq(length, 4);
 7658   jcc(Assembler::equal, SAME_TILL_END);
 7659   //falling through if less than 4 bytes left
 7660 
 7661   bind(BYTES_TAIL);
 7662   bind(BYTES_LOOP);
 7663   load_unsigned_byte(tmp1, Address(obja, result));
 7664   load_unsigned_byte(tmp2, Address(objb, result));
 7665   xorl(tmp1, tmp2);
 7666   testl(tmp1, tmp1);
 7667   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7668   decq(length);
 7669   jcc(Assembler::zero, SAME_TILL_END);
 7670   incq(result);
 7671   load_unsigned_byte(tmp1, Address(obja, result));
 7672   load_unsigned_byte(tmp2, Address(objb, result));
 7673   xorl(tmp1, tmp2);
 7674   testl(tmp1, tmp1);
 7675   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7676   decq(length);
 7677   jcc(Assembler::zero, SAME_TILL_END);
 7678   incq(result);
 7679   load_unsigned_byte(tmp1, Address(obja, result));
 7680   load_unsigned_byte(tmp2, Address(objb, result));
 7681   xorl(tmp1, tmp2);
 7682   testl(tmp1, tmp1);
 7683   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7684   jmp(SAME_TILL_END);
 7685 
 7686   if (UseAVX >= 2) {
 7687     bind(VECTOR32_NOT_EQUAL);
 7688     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7689     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7690     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7691     vpmovmskb(tmp1, rymm0);
 7692     bsfq(tmp1, tmp1);
 7693     addq(result, tmp1);
 7694     shrq(result);
 7695     jmp(DONE);
 7696   }
 7697 
 7698   bind(VECTOR16_NOT_EQUAL);
 7699   if (UseAVX >= 2) {
 7700     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7701     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7702     pxor(rymm0, rymm2);
 7703   } else {
 7704     pcmpeqb(rymm2, rymm2);
 7705     pxor(rymm0, rymm1);
 7706     pcmpeqb(rymm0, rymm1);
 7707     pxor(rymm0, rymm2);
 7708   }
 7709   pmovmskb(tmp1, rymm0);
 7710   bsfq(tmp1, tmp1);
 7711   addq(result, tmp1);
 7712   shrq(result);
 7713   jmpb(DONE);
 7714 
 7715   bind(VECTOR8_NOT_EQUAL);
 7716   bind(VECTOR4_NOT_EQUAL);
 7717   bsfq(tmp1, tmp1);
 7718   shrq(tmp1, 3);
 7719   addq(result, tmp1);
 7720   bind(BYTES_NOT_EQUAL);
 7721   shrq(result);
 7722   jmpb(DONE);
 7723 
 7724   bind(SAME_TILL_END);
 7725   mov64(result, -1);
 7726 
 7727   bind(DONE);
 7728 }
 7729 
 7730 //Helper functions for square_to_len()
 7731 
 7732 /**
 7733  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7734  * Preserves x and z and modifies rest of the registers.
 7735  */
 7736 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7737   // Perform square and right shift by 1
 7738   // Handle odd xlen case first, then for even xlen do the following
 7739   // jlong carry = 0;
 7740   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7741   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7742   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7743   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7744   //     carry = (jlong)product;
 7745   // }
 7746 
 7747   xorq(tmp5, tmp5);     // carry
 7748   xorq(rdxReg, rdxReg);
 7749   xorl(tmp1, tmp1);     // index for x
 7750   xorl(tmp4, tmp4);     // index for z
 7751 
 7752   Label L_first_loop, L_first_loop_exit;
 7753 
 7754   testl(xlen, 1);
 7755   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7756 
 7757   // Square and right shift by 1 the odd element using 32 bit multiply
 7758   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7759   imulq(raxReg, raxReg);
 7760   shrq(raxReg, 1);
 7761   adcq(tmp5, 0);
 7762   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7763   incrementl(tmp1);
 7764   addl(tmp4, 2);
 7765 
 7766   // Square and  right shift by 1 the rest using 64 bit multiply
 7767   bind(L_first_loop);
 7768   cmpptr(tmp1, xlen);
 7769   jccb(Assembler::equal, L_first_loop_exit);
 7770 
 7771   // Square
 7772   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7773   rorq(raxReg, 32);    // convert big-endian to little-endian
 7774   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7775 
 7776   // Right shift by 1 and save carry
 7777   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7778   rcrq(rdxReg, 1);
 7779   rcrq(raxReg, 1);
 7780   adcq(tmp5, 0);
 7781 
 7782   // Store result in z
 7783   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7784   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7785 
 7786   // Update indices for x and z
 7787   addl(tmp1, 2);
 7788   addl(tmp4, 4);
 7789   jmp(L_first_loop);
 7790 
 7791   bind(L_first_loop_exit);
 7792 }
 7793 
 7794 
 7795 /**
 7796  * Perform the following multiply add operation using BMI2 instructions
 7797  * carry:sum = sum + op1*op2 + carry
 7798  * op2 should be in rdx
 7799  * op2 is preserved, all other registers are modified
 7800  */
 7801 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7802   // assert op2 is rdx
 7803   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7804   addq(sum, carry);
 7805   adcq(tmp2, 0);
 7806   addq(sum, op1);
 7807   adcq(tmp2, 0);
 7808   movq(carry, tmp2);
 7809 }
 7810 
 7811 /**
 7812  * Perform the following multiply add operation:
 7813  * carry:sum = sum + op1*op2 + carry
 7814  * Preserves op1, op2 and modifies rest of registers
 7815  */
 7816 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7817   // rdx:rax = op1 * op2
 7818   movq(raxReg, op2);
 7819   mulq(op1);
 7820 
 7821   //  rdx:rax = sum + carry + rdx:rax
 7822   addq(sum, carry);
 7823   adcq(rdxReg, 0);
 7824   addq(sum, raxReg);
 7825   adcq(rdxReg, 0);
 7826 
 7827   // carry:sum = rdx:sum
 7828   movq(carry, rdxReg);
 7829 }
 7830 
 7831 /**
 7832  * Add 64 bit long carry into z[] with carry propagation.
 7833  * Preserves z and carry register values and modifies rest of registers.
 7834  *
 7835  */
 7836 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7837   Label L_fourth_loop, L_fourth_loop_exit;
 7838 
 7839   movl(tmp1, 1);
 7840   subl(zlen, 2);
 7841   addq(Address(z, zlen, Address::times_4, 0), carry);
 7842 
 7843   bind(L_fourth_loop);
 7844   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7845   subl(zlen, 2);
 7846   jccb(Assembler::negative, L_fourth_loop_exit);
 7847   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7848   jmp(L_fourth_loop);
 7849   bind(L_fourth_loop_exit);
 7850 }
 7851 
 7852 /**
 7853  * Shift z[] left by 1 bit.
 7854  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7855  *
 7856  */
 7857 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7858 
 7859   Label L_fifth_loop, L_fifth_loop_exit;
 7860 
 7861   // Fifth loop
 7862   // Perform primitiveLeftShift(z, zlen, 1)
 7863 
 7864   const Register prev_carry = tmp1;
 7865   const Register new_carry = tmp4;
 7866   const Register value = tmp2;
 7867   const Register zidx = tmp3;
 7868 
 7869   // int zidx, carry;
 7870   // long value;
 7871   // carry = 0;
 7872   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7873   //    (carry:value)  = (z[i] << 1) | carry ;
 7874   //    z[i] = value;
 7875   // }
 7876 
 7877   movl(zidx, zlen);
 7878   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7879 
 7880   bind(L_fifth_loop);
 7881   decl(zidx);  // Use decl to preserve carry flag
 7882   decl(zidx);
 7883   jccb(Assembler::negative, L_fifth_loop_exit);
 7884 
 7885   if (UseBMI2Instructions) {
 7886      movq(value, Address(z, zidx, Address::times_4, 0));
 7887      rclq(value, 1);
 7888      rorxq(value, value, 32);
 7889      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7890   }
 7891   else {
 7892     // clear new_carry
 7893     xorl(new_carry, new_carry);
 7894 
 7895     // Shift z[i] by 1, or in previous carry and save new carry
 7896     movq(value, Address(z, zidx, Address::times_4, 0));
 7897     shlq(value, 1);
 7898     adcl(new_carry, 0);
 7899 
 7900     orq(value, prev_carry);
 7901     rorq(value, 0x20);
 7902     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7903 
 7904     // Set previous carry = new carry
 7905     movl(prev_carry, new_carry);
 7906   }
 7907   jmp(L_fifth_loop);
 7908 
 7909   bind(L_fifth_loop_exit);
 7910 }
 7911 
 7912 
 7913 /**
 7914  * Code for BigInteger::squareToLen() intrinsic
 7915  *
 7916  * rdi: x
 7917  * rsi: len
 7918  * r8:  z
 7919  * rcx: zlen
 7920  * r12: tmp1
 7921  * r13: tmp2
 7922  * r14: tmp3
 7923  * r15: tmp4
 7924  * rbx: tmp5
 7925  *
 7926  */
 7927 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7928 
 7929   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7930   push(tmp1);
 7931   push(tmp2);
 7932   push(tmp3);
 7933   push(tmp4);
 7934   push(tmp5);
 7935 
 7936   // First loop
 7937   // Store the squares, right shifted one bit (i.e., divided by 2).
 7938   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7939 
 7940   // Add in off-diagonal sums.
 7941   //
 7942   // Second, third (nested) and fourth loops.
 7943   // zlen +=2;
 7944   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7945   //    carry = 0;
 7946   //    long op2 = x[xidx:xidx+1];
 7947   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7948   //       k -= 2;
 7949   //       long op1 = x[j:j+1];
 7950   //       long sum = z[k:k+1];
 7951   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7952   //       z[k:k+1] = sum;
 7953   //    }
 7954   //    add_one_64(z, k, carry, tmp_regs);
 7955   // }
 7956 
 7957   const Register carry = tmp5;
 7958   const Register sum = tmp3;
 7959   const Register op1 = tmp4;
 7960   Register op2 = tmp2;
 7961 
 7962   push(zlen);
 7963   push(len);
 7964   addl(zlen,2);
 7965   bind(L_second_loop);
 7966   xorq(carry, carry);
 7967   subl(zlen, 4);
 7968   subl(len, 2);
 7969   push(zlen);
 7970   push(len);
 7971   cmpl(len, 0);
 7972   jccb(Assembler::lessEqual, L_second_loop_exit);
 7973 
 7974   // Multiply an array by one 64 bit long.
 7975   if (UseBMI2Instructions) {
 7976     op2 = rdxReg;
 7977     movq(op2, Address(x, len, Address::times_4,  0));
 7978     rorxq(op2, op2, 32);
 7979   }
 7980   else {
 7981     movq(op2, Address(x, len, Address::times_4,  0));
 7982     rorq(op2, 32);
 7983   }
 7984 
 7985   bind(L_third_loop);
 7986   decrementl(len);
 7987   jccb(Assembler::negative, L_third_loop_exit);
 7988   decrementl(len);
 7989   jccb(Assembler::negative, L_last_x);
 7990 
 7991   movq(op1, Address(x, len, Address::times_4,  0));
 7992   rorq(op1, 32);
 7993 
 7994   bind(L_multiply);
 7995   subl(zlen, 2);
 7996   movq(sum, Address(z, zlen, Address::times_4,  0));
 7997 
 7998   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 7999   if (UseBMI2Instructions) {
 8000     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 8001   }
 8002   else {
 8003     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8004   }
 8005 
 8006   movq(Address(z, zlen, Address::times_4, 0), sum);
 8007 
 8008   jmp(L_third_loop);
 8009   bind(L_third_loop_exit);
 8010 
 8011   // Fourth loop
 8012   // Add 64 bit long carry into z with carry propagation.
 8013   // Uses offsetted zlen.
 8014   add_one_64(z, zlen, carry, tmp1);
 8015 
 8016   pop(len);
 8017   pop(zlen);
 8018   jmp(L_second_loop);
 8019 
 8020   // Next infrequent code is moved outside loops.
 8021   bind(L_last_x);
 8022   movl(op1, Address(x, 0));
 8023   jmp(L_multiply);
 8024 
 8025   bind(L_second_loop_exit);
 8026   pop(len);
 8027   pop(zlen);
 8028   pop(len);
 8029   pop(zlen);
 8030 
 8031   // Fifth loop
 8032   // Shift z left 1 bit.
 8033   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 8034 
 8035   // z[zlen-1] |= x[len-1] & 1;
 8036   movl(tmp3, Address(x, len, Address::times_4, -4));
 8037   andl(tmp3, 1);
 8038   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 8039 
 8040   pop(tmp5);
 8041   pop(tmp4);
 8042   pop(tmp3);
 8043   pop(tmp2);
 8044   pop(tmp1);
 8045 }
 8046 
 8047 /**
 8048  * Helper function for mul_add()
 8049  * Multiply the in[] by int k and add to out[] starting at offset offs using
 8050  * 128 bit by 32 bit multiply and return the carry in tmp5.
 8051  * Only quad int aligned length of in[] is operated on in this function.
 8052  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 8053  * This function preserves out, in and k registers.
 8054  * len and offset point to the appropriate index in "in" & "out" correspondingly
 8055  * tmp5 has the carry.
 8056  * other registers are temporary and are modified.
 8057  *
 8058  */
 8059 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 8060   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 8061   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8062 
 8063   Label L_first_loop, L_first_loop_exit;
 8064 
 8065   movl(tmp1, len);
 8066   shrl(tmp1, 2);
 8067 
 8068   bind(L_first_loop);
 8069   subl(tmp1, 1);
 8070   jccb(Assembler::negative, L_first_loop_exit);
 8071 
 8072   subl(len, 4);
 8073   subl(offset, 4);
 8074 
 8075   Register op2 = tmp2;
 8076   const Register sum = tmp3;
 8077   const Register op1 = tmp4;
 8078   const Register carry = tmp5;
 8079 
 8080   if (UseBMI2Instructions) {
 8081     op2 = rdxReg;
 8082   }
 8083 
 8084   movq(op1, Address(in, len, Address::times_4,  8));
 8085   rorq(op1, 32);
 8086   movq(sum, Address(out, offset, Address::times_4,  8));
 8087   rorq(sum, 32);
 8088   if (UseBMI2Instructions) {
 8089     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8090   }
 8091   else {
 8092     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8093   }
 8094   // Store back in big endian from little endian
 8095   rorq(sum, 0x20);
 8096   movq(Address(out, offset, Address::times_4,  8), sum);
 8097 
 8098   movq(op1, Address(in, len, Address::times_4,  0));
 8099   rorq(op1, 32);
 8100   movq(sum, Address(out, offset, Address::times_4,  0));
 8101   rorq(sum, 32);
 8102   if (UseBMI2Instructions) {
 8103     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8104   }
 8105   else {
 8106     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8107   }
 8108   // Store back in big endian from little endian
 8109   rorq(sum, 0x20);
 8110   movq(Address(out, offset, Address::times_4,  0), sum);
 8111 
 8112   jmp(L_first_loop);
 8113   bind(L_first_loop_exit);
 8114 }
 8115 
 8116 /**
 8117  * Code for BigInteger::mulAdd() intrinsic
 8118  *
 8119  * rdi: out
 8120  * rsi: in
 8121  * r11: offs (out.length - offset)
 8122  * rcx: len
 8123  * r8:  k
 8124  * r12: tmp1
 8125  * r13: tmp2
 8126  * r14: tmp3
 8127  * r15: tmp4
 8128  * rbx: tmp5
 8129  * Multiply the in[] by word k and add to out[], return the carry in rax
 8130  */
 8131 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 8132    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 8133    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8134 
 8135   Label L_carry, L_last_in, L_done;
 8136 
 8137 // carry = 0;
 8138 // for (int j=len-1; j >= 0; j--) {
 8139 //    long product = (in[j] & LONG_MASK) * kLong +
 8140 //                   (out[offs] & LONG_MASK) + carry;
 8141 //    out[offs--] = (int)product;
 8142 //    carry = product >>> 32;
 8143 // }
 8144 //
 8145   push(tmp1);
 8146   push(tmp2);
 8147   push(tmp3);
 8148   push(tmp4);
 8149   push(tmp5);
 8150 
 8151   Register op2 = tmp2;
 8152   const Register sum = tmp3;
 8153   const Register op1 = tmp4;
 8154   const Register carry =  tmp5;
 8155 
 8156   if (UseBMI2Instructions) {
 8157     op2 = rdxReg;
 8158     movl(op2, k);
 8159   }
 8160   else {
 8161     movl(op2, k);
 8162   }
 8163 
 8164   xorq(carry, carry);
 8165 
 8166   //First loop
 8167 
 8168   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 8169   //The carry is in tmp5
 8170   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8171 
 8172   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 8173   decrementl(len);
 8174   jccb(Assembler::negative, L_carry);
 8175   decrementl(len);
 8176   jccb(Assembler::negative, L_last_in);
 8177 
 8178   movq(op1, Address(in, len, Address::times_4,  0));
 8179   rorq(op1, 32);
 8180 
 8181   subl(offs, 2);
 8182   movq(sum, Address(out, offs, Address::times_4,  0));
 8183   rorq(sum, 32);
 8184 
 8185   if (UseBMI2Instructions) {
 8186     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8187   }
 8188   else {
 8189     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8190   }
 8191 
 8192   // Store back in big endian from little endian
 8193   rorq(sum, 0x20);
 8194   movq(Address(out, offs, Address::times_4,  0), sum);
 8195 
 8196   testl(len, len);
 8197   jccb(Assembler::zero, L_carry);
 8198 
 8199   //Multiply the last in[] entry, if any
 8200   bind(L_last_in);
 8201   movl(op1, Address(in, 0));
 8202   movl(sum, Address(out, offs, Address::times_4,  -4));
 8203 
 8204   movl(raxReg, k);
 8205   mull(op1); //tmp4 * eax -> edx:eax
 8206   addl(sum, carry);
 8207   adcl(rdxReg, 0);
 8208   addl(sum, raxReg);
 8209   adcl(rdxReg, 0);
 8210   movl(carry, rdxReg);
 8211 
 8212   movl(Address(out, offs, Address::times_4,  -4), sum);
 8213 
 8214   bind(L_carry);
 8215   //return tmp5/carry as carry in rax
 8216   movl(rax, carry);
 8217 
 8218   bind(L_done);
 8219   pop(tmp5);
 8220   pop(tmp4);
 8221   pop(tmp3);
 8222   pop(tmp2);
 8223   pop(tmp1);
 8224 }
 8225 #endif
 8226 
 8227 /**
 8228  * Emits code to update CRC-32 with a byte value according to constants in table
 8229  *
 8230  * @param [in,out]crc   Register containing the crc.
 8231  * @param [in]val       Register containing the byte to fold into the CRC.
 8232  * @param [in]table     Register containing the table of crc constants.
 8233  *
 8234  * uint32_t crc;
 8235  * val = crc_table[(val ^ crc) & 0xFF];
 8236  * crc = val ^ (crc >> 8);
 8237  *
 8238  */
 8239 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 8240   xorl(val, crc);
 8241   andl(val, 0xFF);
 8242   shrl(crc, 8); // unsigned shift
 8243   xorl(crc, Address(table, val, Address::times_4, 0));
 8244 }
 8245 
 8246 /**
 8247  * Fold 128-bit data chunk
 8248  */
 8249 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 8250   if (UseAVX > 0) {
 8251     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 8252     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 8253     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 8254     pxor(xcrc, xtmp);
 8255   } else {
 8256     movdqa(xtmp, xcrc);
 8257     pclmulhdq(xtmp, xK);   // [123:64]
 8258     pclmulldq(xcrc, xK);   // [63:0]
 8259     pxor(xcrc, xtmp);
 8260     movdqu(xtmp, Address(buf, offset));
 8261     pxor(xcrc, xtmp);
 8262   }
 8263 }
 8264 
 8265 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 8266   if (UseAVX > 0) {
 8267     vpclmulhdq(xtmp, xK, xcrc);
 8268     vpclmulldq(xcrc, xK, xcrc);
 8269     pxor(xcrc, xbuf);
 8270     pxor(xcrc, xtmp);
 8271   } else {
 8272     movdqa(xtmp, xcrc);
 8273     pclmulhdq(xtmp, xK);
 8274     pclmulldq(xcrc, xK);
 8275     pxor(xcrc, xbuf);
 8276     pxor(xcrc, xtmp);
 8277   }
 8278 }
 8279 
 8280 /**
 8281  * 8-bit folds to compute 32-bit CRC
 8282  *
 8283  * uint64_t xcrc;
 8284  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 8285  */
 8286 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 8287   movdl(tmp, xcrc);
 8288   andl(tmp, 0xFF);
 8289   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 8290   psrldq(xcrc, 1); // unsigned shift one byte
 8291   pxor(xcrc, xtmp);
 8292 }
 8293 
 8294 /**
 8295  * uint32_t crc;
 8296  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 8297  */
 8298 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 8299   movl(tmp, crc);
 8300   andl(tmp, 0xFF);
 8301   shrl(crc, 8);
 8302   xorl(crc, Address(table, tmp, Address::times_4, 0));
 8303 }
 8304 
 8305 /**
 8306  * @param crc   register containing existing CRC (32-bit)
 8307  * @param buf   register pointing to input byte buffer (byte*)
 8308  * @param len   register containing number of bytes
 8309  * @param table register that will contain address of CRC table
 8310  * @param tmp   scratch register
 8311  */
 8312 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 8313   assert_different_registers(crc, buf, len, table, tmp, rax);
 8314 
 8315   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8316   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8317 
 8318   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8319   // context for the registers used, where all instructions below are using 128-bit mode
 8320   // On EVEX without VL and BW, these instructions will all be AVX.
 8321   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 8322   notl(crc); // ~crc
 8323   cmpl(len, 16);
 8324   jcc(Assembler::less, L_tail);
 8325 
 8326   // Align buffer to 16 bytes
 8327   movl(tmp, buf);
 8328   andl(tmp, 0xF);
 8329   jccb(Assembler::zero, L_aligned);
 8330   subl(tmp,  16);
 8331   addl(len, tmp);
 8332 
 8333   align(4);
 8334   BIND(L_align_loop);
 8335   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8336   update_byte_crc32(crc, rax, table);
 8337   increment(buf);
 8338   incrementl(tmp);
 8339   jccb(Assembler::less, L_align_loop);
 8340 
 8341   BIND(L_aligned);
 8342   movl(tmp, len); // save
 8343   shrl(len, 4);
 8344   jcc(Assembler::zero, L_tail_restore);
 8345 
 8346   // Fold crc into first bytes of vector
 8347   movdqa(xmm1, Address(buf, 0));
 8348   movdl(rax, xmm1);
 8349   xorl(crc, rax);
 8350   if (VM_Version::supports_sse4_1()) {
 8351     pinsrd(xmm1, crc, 0);
 8352   } else {
 8353     pinsrw(xmm1, crc, 0);
 8354     shrl(crc, 16);
 8355     pinsrw(xmm1, crc, 1);
 8356   }
 8357   addptr(buf, 16);
 8358   subl(len, 4); // len > 0
 8359   jcc(Assembler::less, L_fold_tail);
 8360 
 8361   movdqa(xmm2, Address(buf,  0));
 8362   movdqa(xmm3, Address(buf, 16));
 8363   movdqa(xmm4, Address(buf, 32));
 8364   addptr(buf, 48);
 8365   subl(len, 3);
 8366   jcc(Assembler::lessEqual, L_fold_512b);
 8367 
 8368   // Fold total 512 bits of polynomial on each iteration,
 8369   // 128 bits per each of 4 parallel streams.
 8370   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8371 
 8372   align32();
 8373   BIND(L_fold_512b_loop);
 8374   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8375   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8376   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8377   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8378   addptr(buf, 64);
 8379   subl(len, 4);
 8380   jcc(Assembler::greater, L_fold_512b_loop);
 8381 
 8382   // Fold 512 bits to 128 bits.
 8383   BIND(L_fold_512b);
 8384   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8385   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8386   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8387   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8388 
 8389   // Fold the rest of 128 bits data chunks
 8390   BIND(L_fold_tail);
 8391   addl(len, 3);
 8392   jccb(Assembler::lessEqual, L_fold_128b);
 8393   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8394 
 8395   BIND(L_fold_tail_loop);
 8396   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8397   addptr(buf, 16);
 8398   decrementl(len);
 8399   jccb(Assembler::greater, L_fold_tail_loop);
 8400 
 8401   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8402   BIND(L_fold_128b);
 8403   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8404   if (UseAVX > 0) {
 8405     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8406     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8407     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8408   } else {
 8409     movdqa(xmm2, xmm0);
 8410     pclmulqdq(xmm2, xmm1, 0x1);
 8411     movdqa(xmm3, xmm0);
 8412     pand(xmm3, xmm2);
 8413     pclmulqdq(xmm0, xmm3, 0x1);
 8414   }
 8415   psrldq(xmm1, 8);
 8416   psrldq(xmm2, 4);
 8417   pxor(xmm0, xmm1);
 8418   pxor(xmm0, xmm2);
 8419 
 8420   // 8 8-bit folds to compute 32-bit CRC.
 8421   for (int j = 0; j < 4; j++) {
 8422     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8423   }
 8424   movdl(crc, xmm0); // mov 32 bits to general register
 8425   for (int j = 0; j < 4; j++) {
 8426     fold_8bit_crc32(crc, table, rax);
 8427   }
 8428 
 8429   BIND(L_tail_restore);
 8430   movl(len, tmp); // restore
 8431   BIND(L_tail);
 8432   andl(len, 0xf);
 8433   jccb(Assembler::zero, L_exit);
 8434 
 8435   // Fold the rest of bytes
 8436   align(4);
 8437   BIND(L_tail_loop);
 8438   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8439   update_byte_crc32(crc, rax, table);
 8440   increment(buf);
 8441   decrementl(len);
 8442   jccb(Assembler::greater, L_tail_loop);
 8443 
 8444   BIND(L_exit);
 8445   notl(crc); // ~c
 8446 }
 8447 
 8448 #ifdef _LP64
 8449 // Helper function for AVX 512 CRC32
 8450 // Fold 512-bit data chunks
 8451 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8452                                              Register pos, int offset) {
 8453   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8454   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8455   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8456   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8457   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8458 }
 8459 
 8460 // Helper function for AVX 512 CRC32
 8461 // Compute CRC32 for < 256B buffers
 8462 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8463                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8464                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8465 
 8466   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8467   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8468   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8469 
 8470   // check if there is enough buffer to be able to fold 16B at a time
 8471   cmpl(len, 32);
 8472   jcc(Assembler::less, L_less_than_32);
 8473 
 8474   // if there is, load the constants
 8475   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8476   movdl(xmm0, crc);                        // get the initial crc value
 8477   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8478   pxor(xmm7, xmm0);
 8479 
 8480   // update the buffer pointer
 8481   addl(pos, 16);
 8482   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8483   subl(len, 32);
 8484   jmp(L_16B_reduction_loop);
 8485 
 8486   bind(L_less_than_32);
 8487   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8488   movl(rax, crc);
 8489   testl(len, len);
 8490   jcc(Assembler::equal, L_cleanup);
 8491 
 8492   movdl(xmm0, crc);                        //get the initial crc value
 8493 
 8494   cmpl(len, 16);
 8495   jcc(Assembler::equal, L_exact_16_left);
 8496   jcc(Assembler::less, L_less_than_16_left);
 8497 
 8498   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8499   pxor(xmm7, xmm0);                       //xor the initial crc value
 8500   addl(pos, 16);
 8501   subl(len, 16);
 8502   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8503   jmp(L_get_last_two_xmms);
 8504 
 8505   bind(L_less_than_16_left);
 8506   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8507   pxor(xmm1, xmm1);
 8508   movptr(tmp1, rsp);
 8509   movdqu(Address(tmp1, 0 * 16), xmm1);
 8510 
 8511   cmpl(len, 4);
 8512   jcc(Assembler::less, L_only_less_than_4);
 8513 
 8514   //backup the counter value
 8515   movl(tmp2, len);
 8516   cmpl(len, 8);
 8517   jcc(Assembler::less, L_less_than_8_left);
 8518 
 8519   //load 8 Bytes
 8520   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8521   movq(Address(tmp1, 0 * 16), rax);
 8522   addptr(tmp1, 8);
 8523   subl(len, 8);
 8524   addl(pos, 8);
 8525 
 8526   bind(L_less_than_8_left);
 8527   cmpl(len, 4);
 8528   jcc(Assembler::less, L_less_than_4_left);
 8529 
 8530   //load 4 Bytes
 8531   movl(rax, Address(buf, pos, Address::times_1, 0));
 8532   movl(Address(tmp1, 0 * 16), rax);
 8533   addptr(tmp1, 4);
 8534   subl(len, 4);
 8535   addl(pos, 4);
 8536 
 8537   bind(L_less_than_4_left);
 8538   cmpl(len, 2);
 8539   jcc(Assembler::less, L_less_than_2_left);
 8540 
 8541   // load 2 Bytes
 8542   movw(rax, Address(buf, pos, Address::times_1, 0));
 8543   movl(Address(tmp1, 0 * 16), rax);
 8544   addptr(tmp1, 2);
 8545   subl(len, 2);
 8546   addl(pos, 2);
 8547 
 8548   bind(L_less_than_2_left);
 8549   cmpl(len, 1);
 8550   jcc(Assembler::less, L_zero_left);
 8551 
 8552   // load 1 Byte
 8553   movb(rax, Address(buf, pos, Address::times_1, 0));
 8554   movb(Address(tmp1, 0 * 16), rax);
 8555 
 8556   bind(L_zero_left);
 8557   movdqu(xmm7, Address(rsp, 0));
 8558   pxor(xmm7, xmm0);                       //xor the initial crc value
 8559 
 8560   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8561   movdqu(xmm0, Address(rax, tmp2));
 8562   pshufb(xmm7, xmm0);
 8563   jmp(L_128_done);
 8564 
 8565   bind(L_exact_16_left);
 8566   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8567   pxor(xmm7, xmm0);                       //xor the initial crc value
 8568   jmp(L_128_done);
 8569 
 8570   bind(L_only_less_than_4);
 8571   cmpl(len, 3);
 8572   jcc(Assembler::less, L_only_less_than_3);
 8573 
 8574   // load 3 Bytes
 8575   movb(rax, Address(buf, pos, Address::times_1, 0));
 8576   movb(Address(tmp1, 0), rax);
 8577 
 8578   movb(rax, Address(buf, pos, Address::times_1, 1));
 8579   movb(Address(tmp1, 1), rax);
 8580 
 8581   movb(rax, Address(buf, pos, Address::times_1, 2));
 8582   movb(Address(tmp1, 2), rax);
 8583 
 8584   movdqu(xmm7, Address(rsp, 0));
 8585   pxor(xmm7, xmm0);                     //xor the initial crc value
 8586 
 8587   pslldq(xmm7, 0x5);
 8588   jmp(L_barrett);
 8589   bind(L_only_less_than_3);
 8590   cmpl(len, 2);
 8591   jcc(Assembler::less, L_only_less_than_2);
 8592 
 8593   // load 2 Bytes
 8594   movb(rax, Address(buf, pos, Address::times_1, 0));
 8595   movb(Address(tmp1, 0), rax);
 8596 
 8597   movb(rax, Address(buf, pos, Address::times_1, 1));
 8598   movb(Address(tmp1, 1), rax);
 8599 
 8600   movdqu(xmm7, Address(rsp, 0));
 8601   pxor(xmm7, xmm0);                     //xor the initial crc value
 8602 
 8603   pslldq(xmm7, 0x6);
 8604   jmp(L_barrett);
 8605 
 8606   bind(L_only_less_than_2);
 8607   //load 1 Byte
 8608   movb(rax, Address(buf, pos, Address::times_1, 0));
 8609   movb(Address(tmp1, 0), rax);
 8610 
 8611   movdqu(xmm7, Address(rsp, 0));
 8612   pxor(xmm7, xmm0);                     //xor the initial crc value
 8613 
 8614   pslldq(xmm7, 0x7);
 8615 }
 8616 
 8617 /**
 8618 * Compute CRC32 using AVX512 instructions
 8619 * param crc   register containing existing CRC (32-bit)
 8620 * param buf   register pointing to input byte buffer (byte*)
 8621 * param len   register containing number of bytes
 8622 * param table address of crc or crc32c table
 8623 * param tmp1  scratch register
 8624 * param tmp2  scratch register
 8625 * return rax  result register
 8626 *
 8627 * This routine is identical for crc32c with the exception of the precomputed constant
 8628 * table which will be passed as the table argument.  The calculation steps are
 8629 * the same for both variants.
 8630 */
 8631 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8632   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8633 
 8634   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8635   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8636   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8637   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8638   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8639 
 8640   const Register pos = r12;
 8641   push(r12);
 8642   subptr(rsp, 16 * 2 + 8);
 8643 
 8644   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8645   // context for the registers used, where all instructions below are using 128-bit mode
 8646   // On EVEX without VL and BW, these instructions will all be AVX.
 8647   movl(pos, 0);
 8648 
 8649   // check if smaller than 256B
 8650   cmpl(len, 256);
 8651   jcc(Assembler::less, L_less_than_256);
 8652 
 8653   // load the initial crc value
 8654   movdl(xmm10, crc);
 8655 
 8656   // receive the initial 64B data, xor the initial crc value
 8657   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8658   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8659   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8660   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8661 
 8662   subl(len, 256);
 8663   cmpl(len, 256);
 8664   jcc(Assembler::less, L_fold_128_B_loop);
 8665 
 8666   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8667   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8668   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8669   subl(len, 256);
 8670 
 8671   bind(L_fold_256_B_loop);
 8672   addl(pos, 256);
 8673   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8674   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8675   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8676   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8677 
 8678   subl(len, 256);
 8679   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8680 
 8681   // Fold 256 into 128
 8682   addl(pos, 256);
 8683   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8684   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8685   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8686 
 8687   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8688   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8689   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8690 
 8691   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8692   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8693 
 8694   addl(len, 128);
 8695   jmp(L_fold_128_B_register);
 8696 
 8697   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8698   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8699 
 8700   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8701   bind(L_fold_128_B_loop);
 8702   addl(pos, 128);
 8703   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8704   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8705 
 8706   subl(len, 128);
 8707   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8708 
 8709   addl(pos, 128);
 8710 
 8711   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8712   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8713   bind(L_fold_128_B_register);
 8714   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8715   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8716   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8717   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8718   // save last that has no multiplicand
 8719   vextracti64x2(xmm7, xmm4, 3);
 8720 
 8721   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8722   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8723   // Needed later in reduction loop
 8724   movdqu(xmm10, Address(table, 1 * 16));
 8725   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8726   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8727 
 8728   // Swap 1,0,3,2 - 01 00 11 10
 8729   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8730   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8731   vextracti128(xmm5, xmm8, 1);
 8732   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8733 
 8734   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8735   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8736   addl(len, 128 - 16);
 8737   jcc(Assembler::less, L_final_reduction_for_128);
 8738 
 8739   bind(L_16B_reduction_loop);
 8740   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8741   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8742   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8743   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8744   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8745   addl(pos, 16);
 8746   subl(len, 16);
 8747   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8748 
 8749   bind(L_final_reduction_for_128);
 8750   addl(len, 16);
 8751   jcc(Assembler::equal, L_128_done);
 8752 
 8753   bind(L_get_last_two_xmms);
 8754   movdqu(xmm2, xmm7);
 8755   addl(pos, len);
 8756   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8757   subl(pos, len);
 8758 
 8759   // get rid of the extra data that was loaded before
 8760   // load the shift constant
 8761   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8762   movdqu(xmm0, Address(rax, len));
 8763   addl(rax, len);
 8764 
 8765   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8766   //Change mask to 512
 8767   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8768   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8769 
 8770   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8771   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8772   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8773   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8774   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8775 
 8776   bind(L_128_done);
 8777   // compute crc of a 128-bit value
 8778   movdqu(xmm10, Address(table, 3 * 16));
 8779   movdqu(xmm0, xmm7);
 8780 
 8781   // 64b fold
 8782   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8783   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8784   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8785 
 8786   // 32b fold
 8787   movdqu(xmm0, xmm7);
 8788   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8789   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8790   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8791   jmp(L_barrett);
 8792 
 8793   bind(L_less_than_256);
 8794   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8795 
 8796   //barrett reduction
 8797   bind(L_barrett);
 8798   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8799   movdqu(xmm1, xmm7);
 8800   movdqu(xmm2, xmm7);
 8801   movdqu(xmm10, Address(table, 4 * 16));
 8802 
 8803   pclmulqdq(xmm7, xmm10, 0x0);
 8804   pxor(xmm7, xmm2);
 8805   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8806   movdqu(xmm2, xmm7);
 8807   pclmulqdq(xmm7, xmm10, 0x10);
 8808   pxor(xmm7, xmm2);
 8809   pxor(xmm7, xmm1);
 8810   pextrd(crc, xmm7, 2);
 8811 
 8812   bind(L_cleanup);
 8813   addptr(rsp, 16 * 2 + 8);
 8814   pop(r12);
 8815 }
 8816 
 8817 // S. Gueron / Information Processing Letters 112 (2012) 184
 8818 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8819 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8820 // Output: the 64-bit carry-less product of B * CONST
 8821 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8822                                      Register tmp1, Register tmp2, Register tmp3) {
 8823   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8824   if (n > 0) {
 8825     addq(tmp3, n * 256 * 8);
 8826   }
 8827   //    Q1 = TABLEExt[n][B & 0xFF];
 8828   movl(tmp1, in);
 8829   andl(tmp1, 0x000000FF);
 8830   shll(tmp1, 3);
 8831   addq(tmp1, tmp3);
 8832   movq(tmp1, Address(tmp1, 0));
 8833 
 8834   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8835   movl(tmp2, in);
 8836   shrl(tmp2, 8);
 8837   andl(tmp2, 0x000000FF);
 8838   shll(tmp2, 3);
 8839   addq(tmp2, tmp3);
 8840   movq(tmp2, Address(tmp2, 0));
 8841 
 8842   shlq(tmp2, 8);
 8843   xorq(tmp1, tmp2);
 8844 
 8845   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8846   movl(tmp2, in);
 8847   shrl(tmp2, 16);
 8848   andl(tmp2, 0x000000FF);
 8849   shll(tmp2, 3);
 8850   addq(tmp2, tmp3);
 8851   movq(tmp2, Address(tmp2, 0));
 8852 
 8853   shlq(tmp2, 16);
 8854   xorq(tmp1, tmp2);
 8855 
 8856   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8857   shrl(in, 24);
 8858   andl(in, 0x000000FF);
 8859   shll(in, 3);
 8860   addq(in, tmp3);
 8861   movq(in, Address(in, 0));
 8862 
 8863   shlq(in, 24);
 8864   xorq(in, tmp1);
 8865   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8866 }
 8867 
 8868 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8869                                       Register in_out,
 8870                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8871                                       XMMRegister w_xtmp2,
 8872                                       Register tmp1,
 8873                                       Register n_tmp2, Register n_tmp3) {
 8874   if (is_pclmulqdq_supported) {
 8875     movdl(w_xtmp1, in_out); // modified blindly
 8876 
 8877     movl(tmp1, const_or_pre_comp_const_index);
 8878     movdl(w_xtmp2, tmp1);
 8879     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8880 
 8881     movdq(in_out, w_xtmp1);
 8882   } else {
 8883     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8884   }
 8885 }
 8886 
 8887 // Recombination Alternative 2: No bit-reflections
 8888 // T1 = (CRC_A * U1) << 1
 8889 // T2 = (CRC_B * U2) << 1
 8890 // C1 = T1 >> 32
 8891 // C2 = T2 >> 32
 8892 // T1 = T1 & 0xFFFFFFFF
 8893 // T2 = T2 & 0xFFFFFFFF
 8894 // T1 = CRC32(0, T1)
 8895 // T2 = CRC32(0, T2)
 8896 // C1 = C1 ^ T1
 8897 // C2 = C2 ^ T2
 8898 // CRC = C1 ^ C2 ^ CRC_C
 8899 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8900                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8901                                      Register tmp1, Register tmp2,
 8902                                      Register n_tmp3) {
 8903   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8904   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8905   shlq(in_out, 1);
 8906   movl(tmp1, in_out);
 8907   shrq(in_out, 32);
 8908   xorl(tmp2, tmp2);
 8909   crc32(tmp2, tmp1, 4);
 8910   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8911   shlq(in1, 1);
 8912   movl(tmp1, in1);
 8913   shrq(in1, 32);
 8914   xorl(tmp2, tmp2);
 8915   crc32(tmp2, tmp1, 4);
 8916   xorl(in1, tmp2);
 8917   xorl(in_out, in1);
 8918   xorl(in_out, in2);
 8919 }
 8920 
 8921 // Set N to predefined value
 8922 // Subtract from a length of a buffer
 8923 // execute in a loop:
 8924 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8925 // for i = 1 to N do
 8926 //  CRC_A = CRC32(CRC_A, A[i])
 8927 //  CRC_B = CRC32(CRC_B, B[i])
 8928 //  CRC_C = CRC32(CRC_C, C[i])
 8929 // end for
 8930 // Recombine
 8931 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8932                                        Register in_out1, Register in_out2, Register in_out3,
 8933                                        Register tmp1, Register tmp2, Register tmp3,
 8934                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8935                                        Register tmp4, Register tmp5,
 8936                                        Register n_tmp6) {
 8937   Label L_processPartitions;
 8938   Label L_processPartition;
 8939   Label L_exit;
 8940 
 8941   bind(L_processPartitions);
 8942   cmpl(in_out1, 3 * size);
 8943   jcc(Assembler::less, L_exit);
 8944     xorl(tmp1, tmp1);
 8945     xorl(tmp2, tmp2);
 8946     movq(tmp3, in_out2);
 8947     addq(tmp3, size);
 8948 
 8949     bind(L_processPartition);
 8950       crc32(in_out3, Address(in_out2, 0), 8);
 8951       crc32(tmp1, Address(in_out2, size), 8);
 8952       crc32(tmp2, Address(in_out2, size * 2), 8);
 8953       addq(in_out2, 8);
 8954       cmpq(in_out2, tmp3);
 8955       jcc(Assembler::less, L_processPartition);
 8956     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8957             w_xtmp1, w_xtmp2, w_xtmp3,
 8958             tmp4, tmp5,
 8959             n_tmp6);
 8960     addq(in_out2, 2 * size);
 8961     subl(in_out1, 3 * size);
 8962     jmp(L_processPartitions);
 8963 
 8964   bind(L_exit);
 8965 }
 8966 #else
 8967 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 8968                                      Register tmp1, Register tmp2, Register tmp3,
 8969                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 8970   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8971   if (n > 0) {
 8972     addl(tmp3, n * 256 * 8);
 8973   }
 8974   //    Q1 = TABLEExt[n][B & 0xFF];
 8975   movl(tmp1, in_out);
 8976   andl(tmp1, 0x000000FF);
 8977   shll(tmp1, 3);
 8978   addl(tmp1, tmp3);
 8979   movq(xtmp1, Address(tmp1, 0));
 8980 
 8981   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8982   movl(tmp2, in_out);
 8983   shrl(tmp2, 8);
 8984   andl(tmp2, 0x000000FF);
 8985   shll(tmp2, 3);
 8986   addl(tmp2, tmp3);
 8987   movq(xtmp2, Address(tmp2, 0));
 8988 
 8989   psllq(xtmp2, 8);
 8990   pxor(xtmp1, xtmp2);
 8991 
 8992   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8993   movl(tmp2, in_out);
 8994   shrl(tmp2, 16);
 8995   andl(tmp2, 0x000000FF);
 8996   shll(tmp2, 3);
 8997   addl(tmp2, tmp3);
 8998   movq(xtmp2, Address(tmp2, 0));
 8999 
 9000   psllq(xtmp2, 16);
 9001   pxor(xtmp1, xtmp2);
 9002 
 9003   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 9004   shrl(in_out, 24);
 9005   andl(in_out, 0x000000FF);
 9006   shll(in_out, 3);
 9007   addl(in_out, tmp3);
 9008   movq(xtmp2, Address(in_out, 0));
 9009 
 9010   psllq(xtmp2, 24);
 9011   pxor(xtmp1, xtmp2); // Result in CXMM
 9012   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 9013 }
 9014 
 9015 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 9016                                       Register in_out,
 9017                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 9018                                       XMMRegister w_xtmp2,
 9019                                       Register tmp1,
 9020                                       Register n_tmp2, Register n_tmp3) {
 9021   if (is_pclmulqdq_supported) {
 9022     movdl(w_xtmp1, in_out);
 9023 
 9024     movl(tmp1, const_or_pre_comp_const_index);
 9025     movdl(w_xtmp2, tmp1);
 9026     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 9027     // Keep result in XMM since GPR is 32 bit in length
 9028   } else {
 9029     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 9030   }
 9031 }
 9032 
 9033 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9034                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9035                                      Register tmp1, Register tmp2,
 9036                                      Register n_tmp3) {
 9037   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9038   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9039 
 9040   psllq(w_xtmp1, 1);
 9041   movdl(tmp1, w_xtmp1);
 9042   psrlq(w_xtmp1, 32);
 9043   movdl(in_out, w_xtmp1);
 9044 
 9045   xorl(tmp2, tmp2);
 9046   crc32(tmp2, tmp1, 4);
 9047   xorl(in_out, tmp2);
 9048 
 9049   psllq(w_xtmp2, 1);
 9050   movdl(tmp1, w_xtmp2);
 9051   psrlq(w_xtmp2, 32);
 9052   movdl(in1, w_xtmp2);
 9053 
 9054   xorl(tmp2, tmp2);
 9055   crc32(tmp2, tmp1, 4);
 9056   xorl(in1, tmp2);
 9057   xorl(in_out, in1);
 9058   xorl(in_out, in2);
 9059 }
 9060 
 9061 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9062                                        Register in_out1, Register in_out2, Register in_out3,
 9063                                        Register tmp1, Register tmp2, Register tmp3,
 9064                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9065                                        Register tmp4, Register tmp5,
 9066                                        Register n_tmp6) {
 9067   Label L_processPartitions;
 9068   Label L_processPartition;
 9069   Label L_exit;
 9070 
 9071   bind(L_processPartitions);
 9072   cmpl(in_out1, 3 * size);
 9073   jcc(Assembler::less, L_exit);
 9074     xorl(tmp1, tmp1);
 9075     xorl(tmp2, tmp2);
 9076     movl(tmp3, in_out2);
 9077     addl(tmp3, size);
 9078 
 9079     bind(L_processPartition);
 9080       crc32(in_out3, Address(in_out2, 0), 4);
 9081       crc32(tmp1, Address(in_out2, size), 4);
 9082       crc32(tmp2, Address(in_out2, size*2), 4);
 9083       crc32(in_out3, Address(in_out2, 0+4), 4);
 9084       crc32(tmp1, Address(in_out2, size+4), 4);
 9085       crc32(tmp2, Address(in_out2, size*2+4), 4);
 9086       addl(in_out2, 8);
 9087       cmpl(in_out2, tmp3);
 9088       jcc(Assembler::less, L_processPartition);
 9089 
 9090         push(tmp3);
 9091         push(in_out1);
 9092         push(in_out2);
 9093         tmp4 = tmp3;
 9094         tmp5 = in_out1;
 9095         n_tmp6 = in_out2;
 9096 
 9097       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9098             w_xtmp1, w_xtmp2, w_xtmp3,
 9099             tmp4, tmp5,
 9100             n_tmp6);
 9101 
 9102         pop(in_out2);
 9103         pop(in_out1);
 9104         pop(tmp3);
 9105 
 9106     addl(in_out2, 2 * size);
 9107     subl(in_out1, 3 * size);
 9108     jmp(L_processPartitions);
 9109 
 9110   bind(L_exit);
 9111 }
 9112 #endif //LP64
 9113 
 9114 #ifdef _LP64
 9115 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 9116 // Input: A buffer I of L bytes.
 9117 // Output: the CRC32C value of the buffer.
 9118 // Notations:
 9119 // Write L = 24N + r, with N = floor (L/24).
 9120 // r = L mod 24 (0 <= r < 24).
 9121 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 9122 // N quadwords, and R consists of r bytes.
 9123 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 9124 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 9125 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 9126 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 9127 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9128                                           Register tmp1, Register tmp2, Register tmp3,
 9129                                           Register tmp4, Register tmp5, Register tmp6,
 9130                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9131                                           bool is_pclmulqdq_supported) {
 9132   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9133   Label L_wordByWord;
 9134   Label L_byteByByteProlog;
 9135   Label L_byteByByte;
 9136   Label L_exit;
 9137 
 9138   if (is_pclmulqdq_supported ) {
 9139     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
 9140     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
 9141 
 9142     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
 9143     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
 9144 
 9145     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
 9146     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
 9147     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 9148   } else {
 9149     const_or_pre_comp_const_index[0] = 1;
 9150     const_or_pre_comp_const_index[1] = 0;
 9151 
 9152     const_or_pre_comp_const_index[2] = 3;
 9153     const_or_pre_comp_const_index[3] = 2;
 9154 
 9155     const_or_pre_comp_const_index[4] = 5;
 9156     const_or_pre_comp_const_index[5] = 4;
 9157    }
 9158   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9159                     in2, in1, in_out,
 9160                     tmp1, tmp2, tmp3,
 9161                     w_xtmp1, w_xtmp2, w_xtmp3,
 9162                     tmp4, tmp5,
 9163                     tmp6);
 9164   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9165                     in2, in1, in_out,
 9166                     tmp1, tmp2, tmp3,
 9167                     w_xtmp1, w_xtmp2, w_xtmp3,
 9168                     tmp4, tmp5,
 9169                     tmp6);
 9170   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9171                     in2, in1, in_out,
 9172                     tmp1, tmp2, tmp3,
 9173                     w_xtmp1, w_xtmp2, w_xtmp3,
 9174                     tmp4, tmp5,
 9175                     tmp6);
 9176   movl(tmp1, in2);
 9177   andl(tmp1, 0x00000007);
 9178   negl(tmp1);
 9179   addl(tmp1, in2);
 9180   addq(tmp1, in1);
 9181 
 9182   cmpq(in1, tmp1);
 9183   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 9184   align(16);
 9185   BIND(L_wordByWord);
 9186     crc32(in_out, Address(in1, 0), 8);
 9187     addq(in1, 8);
 9188     cmpq(in1, tmp1);
 9189     jcc(Assembler::less, L_wordByWord);
 9190 
 9191   BIND(L_byteByByteProlog);
 9192   andl(in2, 0x00000007);
 9193   movl(tmp2, 1);
 9194 
 9195   cmpl(tmp2, in2);
 9196   jccb(Assembler::greater, L_exit);
 9197   BIND(L_byteByByte);
 9198     crc32(in_out, Address(in1, 0), 1);
 9199     incq(in1);
 9200     incl(tmp2);
 9201     cmpl(tmp2, in2);
 9202     jcc(Assembler::lessEqual, L_byteByByte);
 9203 
 9204   BIND(L_exit);
 9205 }
 9206 #else
 9207 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9208                                           Register tmp1, Register  tmp2, Register tmp3,
 9209                                           Register tmp4, Register  tmp5, Register tmp6,
 9210                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9211                                           bool is_pclmulqdq_supported) {
 9212   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9213   Label L_wordByWord;
 9214   Label L_byteByByteProlog;
 9215   Label L_byteByByte;
 9216   Label L_exit;
 9217 
 9218   if (is_pclmulqdq_supported) {
 9219     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
 9220     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
 9221 
 9222     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
 9223     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
 9224 
 9225     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
 9226     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
 9227   } else {
 9228     const_or_pre_comp_const_index[0] = 1;
 9229     const_or_pre_comp_const_index[1] = 0;
 9230 
 9231     const_or_pre_comp_const_index[2] = 3;
 9232     const_or_pre_comp_const_index[3] = 2;
 9233 
 9234     const_or_pre_comp_const_index[4] = 5;
 9235     const_or_pre_comp_const_index[5] = 4;
 9236   }
 9237   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9238                     in2, in1, in_out,
 9239                     tmp1, tmp2, tmp3,
 9240                     w_xtmp1, w_xtmp2, w_xtmp3,
 9241                     tmp4, tmp5,
 9242                     tmp6);
 9243   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9244                     in2, in1, in_out,
 9245                     tmp1, tmp2, tmp3,
 9246                     w_xtmp1, w_xtmp2, w_xtmp3,
 9247                     tmp4, tmp5,
 9248                     tmp6);
 9249   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9250                     in2, in1, in_out,
 9251                     tmp1, tmp2, tmp3,
 9252                     w_xtmp1, w_xtmp2, w_xtmp3,
 9253                     tmp4, tmp5,
 9254                     tmp6);
 9255   movl(tmp1, in2);
 9256   andl(tmp1, 0x00000007);
 9257   negl(tmp1);
 9258   addl(tmp1, in2);
 9259   addl(tmp1, in1);
 9260 
 9261   BIND(L_wordByWord);
 9262   cmpl(in1, tmp1);
 9263   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 9264     crc32(in_out, Address(in1,0), 4);
 9265     addl(in1, 4);
 9266     jmp(L_wordByWord);
 9267 
 9268   BIND(L_byteByByteProlog);
 9269   andl(in2, 0x00000007);
 9270   movl(tmp2, 1);
 9271 
 9272   BIND(L_byteByByte);
 9273   cmpl(tmp2, in2);
 9274   jccb(Assembler::greater, L_exit);
 9275     movb(tmp1, Address(in1, 0));
 9276     crc32(in_out, tmp1, 1);
 9277     incl(in1);
 9278     incl(tmp2);
 9279     jmp(L_byteByByte);
 9280 
 9281   BIND(L_exit);
 9282 }
 9283 #endif // LP64
 9284 #undef BIND
 9285 #undef BLOCK_COMMENT
 9286 
 9287 // Compress char[] array to byte[].
 9288 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
 9289 // Return the array length if every element in array can be encoded,
 9290 // otherwise, the index of first non-latin1 (> 0xff) character.
 9291 //   @IntrinsicCandidate
 9292 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 9293 //     for (int i = 0; i < len; i++) {
 9294 //       char c = src[srcOff];
 9295 //       if (c > 0xff) {
 9296 //           return i;  // return index of non-latin1 char
 9297 //       }
 9298 //       dst[dstOff] = (byte)c;
 9299 //       srcOff++;
 9300 //       dstOff++;
 9301 //     }
 9302 //     return len;
 9303 //   }
 9304 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 9305   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 9306   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 9307   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 9308   Label copy_chars_loop, done, reset_sp, copy_tail;
 9309 
 9310   // rsi: src
 9311   // rdi: dst
 9312   // rdx: len
 9313   // rcx: tmp5
 9314   // rax: result
 9315 
 9316   // rsi holds start addr of source char[] to be compressed
 9317   // rdi holds start addr of destination byte[]
 9318   // rdx holds length
 9319 
 9320   assert(len != result, "");
 9321 
 9322   // save length for return
 9323   movl(result, len);
 9324 
 9325   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 9326     VM_Version::supports_avx512vlbw() &&
 9327     VM_Version::supports_bmi2()) {
 9328 
 9329     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
 9330 
 9331     // alignment
 9332     Label post_alignment;
 9333 
 9334     // if length of the string is less than 32, handle it the old fashioned way
 9335     testl(len, -32);
 9336     jcc(Assembler::zero, below_threshold);
 9337 
 9338     // First check whether a character is compressible ( <= 0xFF).
 9339     // Create mask to test for Unicode chars inside zmm vector
 9340     movl(tmp5, 0x00FF);
 9341     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
 9342 
 9343     testl(len, -64);
 9344     jccb(Assembler::zero, post_alignment);
 9345 
 9346     movl(tmp5, dst);
 9347     andl(tmp5, (32 - 1));
 9348     negl(tmp5);
 9349     andl(tmp5, (32 - 1));
 9350 
 9351     // bail out when there is nothing to be done
 9352     testl(tmp5, 0xFFFFFFFF);
 9353     jccb(Assembler::zero, post_alignment);
 9354 
 9355     // ~(~0 << len), where len is the # of remaining elements to process
 9356     movl(len, 0xFFFFFFFF);
 9357     shlxl(len, len, tmp5);
 9358     notl(len);
 9359     kmovdl(mask2, len);
 9360     movl(len, result);
 9361 
 9362     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9363     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9364     ktestd(mask1, mask2);
 9365     jcc(Assembler::carryClear, copy_tail);
 9366 
 9367     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9368 
 9369     addptr(src, tmp5);
 9370     addptr(src, tmp5);
 9371     addptr(dst, tmp5);
 9372     subl(len, tmp5);
 9373 
 9374     bind(post_alignment);
 9375     // end of alignment
 9376 
 9377     movl(tmp5, len);
 9378     andl(tmp5, (32 - 1));    // tail count (in chars)
 9379     andl(len, ~(32 - 1));    // vector count (in chars)
 9380     jccb(Assembler::zero, copy_loop_tail);
 9381 
 9382     lea(src, Address(src, len, Address::times_2));
 9383     lea(dst, Address(dst, len, Address::times_1));
 9384     negptr(len);
 9385 
 9386     bind(copy_32_loop);
 9387     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9388     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9389     kortestdl(mask1, mask1);
 9390     jccb(Assembler::carryClear, reset_for_copy_tail);
 9391 
 9392     // All elements in current processed chunk are valid candidates for
 9393     // compression. Write a truncated byte elements to the memory.
 9394     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9395     addptr(len, 32);
 9396     jccb(Assembler::notZero, copy_32_loop);
 9397 
 9398     bind(copy_loop_tail);
 9399     // bail out when there is nothing to be done
 9400     testl(tmp5, 0xFFFFFFFF);
 9401     jcc(Assembler::zero, done);
 9402 
 9403     movl(len, tmp5);
 9404 
 9405     // ~(~0 << len), where len is the # of remaining elements to process
 9406     movl(tmp5, 0xFFFFFFFF);
 9407     shlxl(tmp5, tmp5, len);
 9408     notl(tmp5);
 9409 
 9410     kmovdl(mask2, tmp5);
 9411 
 9412     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9413     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9414     ktestd(mask1, mask2);
 9415     jcc(Assembler::carryClear, copy_tail);
 9416 
 9417     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9418     jmp(done);
 9419 
 9420     bind(reset_for_copy_tail);
 9421     lea(src, Address(src, tmp5, Address::times_2));
 9422     lea(dst, Address(dst, tmp5, Address::times_1));
 9423     subptr(len, tmp5);
 9424     jmp(copy_chars_loop);
 9425 
 9426     bind(below_threshold);
 9427   }
 9428 
 9429   if (UseSSE42Intrinsics) {
 9430     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
 9431 
 9432     // vectored compression
 9433     testl(len, 0xfffffff8);
 9434     jcc(Assembler::zero, copy_tail);
 9435 
 9436     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9437     movdl(tmp1Reg, tmp5);
 9438     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9439 
 9440     andl(len, 0xfffffff0);
 9441     jccb(Assembler::zero, copy_16);
 9442 
 9443     // compress 16 chars per iter
 9444     pxor(tmp4Reg, tmp4Reg);
 9445 
 9446     lea(src, Address(src, len, Address::times_2));
 9447     lea(dst, Address(dst, len, Address::times_1));
 9448     negptr(len);
 9449 
 9450     bind(copy_32_loop);
 9451     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9452     por(tmp4Reg, tmp2Reg);
 9453     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9454     por(tmp4Reg, tmp3Reg);
 9455     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9456     jccb(Assembler::notZero, reset_for_copy_tail);
 9457     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9458     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9459     addptr(len, 16);
 9460     jccb(Assembler::notZero, copy_32_loop);
 9461 
 9462     // compress next vector of 8 chars (if any)
 9463     bind(copy_16);
 9464     // len = 0
 9465     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
 9466     jccb(Assembler::zero, copy_tail_sse);
 9467 
 9468     pxor(tmp3Reg, tmp3Reg);
 9469 
 9470     movdqu(tmp2Reg, Address(src, 0));
 9471     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9472     jccb(Assembler::notZero, reset_for_copy_tail);
 9473     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9474     movq(Address(dst, 0), tmp2Reg);
 9475     addptr(src, 16);
 9476     addptr(dst, 8);
 9477     jmpb(copy_tail_sse);
 9478 
 9479     bind(reset_for_copy_tail);
 9480     movl(tmp5, result);
 9481     andl(tmp5, 0x0000000f);
 9482     lea(src, Address(src, tmp5, Address::times_2));
 9483     lea(dst, Address(dst, tmp5, Address::times_1));
 9484     subptr(len, tmp5);
 9485     jmpb(copy_chars_loop);
 9486 
 9487     bind(copy_tail_sse);
 9488     movl(len, result);
 9489     andl(len, 0x00000007);    // tail count (in chars)
 9490   }
 9491   // compress 1 char per iter
 9492   bind(copy_tail);
 9493   testl(len, len);
 9494   jccb(Assembler::zero, done);
 9495   lea(src, Address(src, len, Address::times_2));
 9496   lea(dst, Address(dst, len, Address::times_1));
 9497   negptr(len);
 9498 
 9499   bind(copy_chars_loop);
 9500   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
 9501   testl(tmp5, 0xff00);      // check if Unicode char
 9502   jccb(Assembler::notZero, reset_sp);
 9503   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
 9504   increment(len);
 9505   jccb(Assembler::notZero, copy_chars_loop);
 9506 
 9507   // add len then return (len will be zero if compress succeeded, otherwise negative)
 9508   bind(reset_sp);
 9509   addl(result, len);
 9510 
 9511   bind(done);
 9512 }
 9513 
 9514 // Inflate byte[] array to char[].
 9515 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9516 //   @IntrinsicCandidate
 9517 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9518 //     for (int i = 0; i < len; i++) {
 9519 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9520 //     }
 9521 //   }
 9522 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9523   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9524   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9525   // rsi: src
 9526   // rdi: dst
 9527   // rdx: len
 9528   // rcx: tmp2
 9529 
 9530   // rsi holds start addr of source byte[] to be inflated
 9531   // rdi holds start addr of destination char[]
 9532   // rdx holds length
 9533   assert_different_registers(src, dst, len, tmp2);
 9534   movl(tmp2, len);
 9535   if ((UseAVX > 2) && // AVX512
 9536     VM_Version::supports_avx512vlbw() &&
 9537     VM_Version::supports_bmi2()) {
 9538 
 9539     Label copy_32_loop, copy_tail;
 9540     Register tmp3_aliased = len;
 9541 
 9542     // if length of the string is less than 16, handle it in an old fashioned way
 9543     testl(len, -16);
 9544     jcc(Assembler::zero, below_threshold);
 9545 
 9546     testl(len, -1 * AVX3Threshold);
 9547     jcc(Assembler::zero, avx3_threshold);
 9548 
 9549     // In order to use only one arithmetic operation for the main loop we use
 9550     // this pre-calculation
 9551     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9552     andl(len, -32);     // vector count
 9553     jccb(Assembler::zero, copy_tail);
 9554 
 9555     lea(src, Address(src, len, Address::times_1));
 9556     lea(dst, Address(dst, len, Address::times_2));
 9557     negptr(len);
 9558 
 9559 
 9560     // inflate 32 chars per iter
 9561     bind(copy_32_loop);
 9562     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9563     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9564     addptr(len, 32);
 9565     jcc(Assembler::notZero, copy_32_loop);
 9566 
 9567     bind(copy_tail);
 9568     // bail out when there is nothing to be done
 9569     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9570     jcc(Assembler::zero, done);
 9571 
 9572     // ~(~0 << length), where length is the # of remaining elements to process
 9573     movl(tmp3_aliased, -1);
 9574     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9575     notl(tmp3_aliased);
 9576     kmovdl(mask, tmp3_aliased);
 9577     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9578     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9579 
 9580     jmp(done);
 9581     bind(avx3_threshold);
 9582   }
 9583   if (UseSSE42Intrinsics) {
 9584     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9585 
 9586     if (UseAVX > 1) {
 9587       andl(tmp2, (16 - 1));
 9588       andl(len, -16);
 9589       jccb(Assembler::zero, copy_new_tail);
 9590     } else {
 9591       andl(tmp2, 0x00000007);   // tail count (in chars)
 9592       andl(len, 0xfffffff8);    // vector count (in chars)
 9593       jccb(Assembler::zero, copy_tail);
 9594     }
 9595 
 9596     // vectored inflation
 9597     lea(src, Address(src, len, Address::times_1));
 9598     lea(dst, Address(dst, len, Address::times_2));
 9599     negptr(len);
 9600 
 9601     if (UseAVX > 1) {
 9602       bind(copy_16_loop);
 9603       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9604       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9605       addptr(len, 16);
 9606       jcc(Assembler::notZero, copy_16_loop);
 9607 
 9608       bind(below_threshold);
 9609       bind(copy_new_tail);
 9610       movl(len, tmp2);
 9611       andl(tmp2, 0x00000007);
 9612       andl(len, 0xFFFFFFF8);
 9613       jccb(Assembler::zero, copy_tail);
 9614 
 9615       pmovzxbw(tmp1, Address(src, 0));
 9616       movdqu(Address(dst, 0), tmp1);
 9617       addptr(src, 8);
 9618       addptr(dst, 2 * 8);
 9619 
 9620       jmp(copy_tail, true);
 9621     }
 9622 
 9623     // inflate 8 chars per iter
 9624     bind(copy_8_loop);
 9625     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9626     movdqu(Address(dst, len, Address::times_2), tmp1);
 9627     addptr(len, 8);
 9628     jcc(Assembler::notZero, copy_8_loop);
 9629 
 9630     bind(copy_tail);
 9631     movl(len, tmp2);
 9632 
 9633     cmpl(len, 4);
 9634     jccb(Assembler::less, copy_bytes);
 9635 
 9636     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9637     pmovzxbw(tmp1, tmp1);
 9638     movq(Address(dst, 0), tmp1);
 9639     subptr(len, 4);
 9640     addptr(src, 4);
 9641     addptr(dst, 8);
 9642 
 9643     bind(copy_bytes);
 9644   } else {
 9645     bind(below_threshold);
 9646   }
 9647 
 9648   testl(len, len);
 9649   jccb(Assembler::zero, done);
 9650   lea(src, Address(src, len, Address::times_1));
 9651   lea(dst, Address(dst, len, Address::times_2));
 9652   negptr(len);
 9653 
 9654   // inflate 1 char per iter
 9655   bind(copy_chars_loop);
 9656   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9657   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9658   increment(len);
 9659   jcc(Assembler::notZero, copy_chars_loop);
 9660 
 9661   bind(done);
 9662 }
 9663 
 9664 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 9665   switch(type) {
 9666     case T_BYTE:
 9667     case T_BOOLEAN:
 9668       evmovdqub(dst, kmask, src, merge, vector_len);
 9669       break;
 9670     case T_CHAR:
 9671     case T_SHORT:
 9672       evmovdquw(dst, kmask, src, merge, vector_len);
 9673       break;
 9674     case T_INT:
 9675     case T_FLOAT:
 9676       evmovdqul(dst, kmask, src, merge, vector_len);
 9677       break;
 9678     case T_LONG:
 9679     case T_DOUBLE:
 9680       evmovdquq(dst, kmask, src, merge, vector_len);
 9681       break;
 9682     default:
 9683       fatal("Unexpected type argument %s", type2name(type));
 9684       break;
 9685   }
 9686 }
 9687 
 9688 
 9689 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9690   switch(type) {
 9691     case T_BYTE:
 9692     case T_BOOLEAN:
 9693       evmovdqub(dst, kmask, src, merge, vector_len);
 9694       break;
 9695     case T_CHAR:
 9696     case T_SHORT:
 9697       evmovdquw(dst, kmask, src, merge, vector_len);
 9698       break;
 9699     case T_INT:
 9700     case T_FLOAT:
 9701       evmovdqul(dst, kmask, src, merge, vector_len);
 9702       break;
 9703     case T_LONG:
 9704     case T_DOUBLE:
 9705       evmovdquq(dst, kmask, src, merge, vector_len);
 9706       break;
 9707     default:
 9708       fatal("Unexpected type argument %s", type2name(type));
 9709       break;
 9710   }
 9711 }
 9712 
 9713 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9714   switch(type) {
 9715     case T_BYTE:
 9716     case T_BOOLEAN:
 9717       evmovdqub(dst, kmask, src, merge, vector_len);
 9718       break;
 9719     case T_CHAR:
 9720     case T_SHORT:
 9721       evmovdquw(dst, kmask, src, merge, vector_len);
 9722       break;
 9723     case T_INT:
 9724     case T_FLOAT:
 9725       evmovdqul(dst, kmask, src, merge, vector_len);
 9726       break;
 9727     case T_LONG:
 9728     case T_DOUBLE:
 9729       evmovdquq(dst, kmask, src, merge, vector_len);
 9730       break;
 9731     default:
 9732       fatal("Unexpected type argument %s", type2name(type));
 9733       break;
 9734   }
 9735 }
 9736 
 9737 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9738   switch(masklen) {
 9739     case 2:
 9740        knotbl(dst, src);
 9741        movl(rtmp, 3);
 9742        kmovbl(ktmp, rtmp);
 9743        kandbl(dst, ktmp, dst);
 9744        break;
 9745     case 4:
 9746        knotbl(dst, src);
 9747        movl(rtmp, 15);
 9748        kmovbl(ktmp, rtmp);
 9749        kandbl(dst, ktmp, dst);
 9750        break;
 9751     case 8:
 9752        knotbl(dst, src);
 9753        break;
 9754     case 16:
 9755        knotwl(dst, src);
 9756        break;
 9757     case 32:
 9758        knotdl(dst, src);
 9759        break;
 9760     case 64:
 9761        knotql(dst, src);
 9762        break;
 9763     default:
 9764       fatal("Unexpected vector length %d", masklen);
 9765       break;
 9766   }
 9767 }
 9768 
 9769 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9770   switch(type) {
 9771     case T_BOOLEAN:
 9772     case T_BYTE:
 9773        kandbl(dst, src1, src2);
 9774        break;
 9775     case T_CHAR:
 9776     case T_SHORT:
 9777        kandwl(dst, src1, src2);
 9778        break;
 9779     case T_INT:
 9780     case T_FLOAT:
 9781        kanddl(dst, src1, src2);
 9782        break;
 9783     case T_LONG:
 9784     case T_DOUBLE:
 9785        kandql(dst, src1, src2);
 9786        break;
 9787     default:
 9788       fatal("Unexpected type argument %s", type2name(type));
 9789       break;
 9790   }
 9791 }
 9792 
 9793 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9794   switch(type) {
 9795     case T_BOOLEAN:
 9796     case T_BYTE:
 9797        korbl(dst, src1, src2);
 9798        break;
 9799     case T_CHAR:
 9800     case T_SHORT:
 9801        korwl(dst, src1, src2);
 9802        break;
 9803     case T_INT:
 9804     case T_FLOAT:
 9805        kordl(dst, src1, src2);
 9806        break;
 9807     case T_LONG:
 9808     case T_DOUBLE:
 9809        korql(dst, src1, src2);
 9810        break;
 9811     default:
 9812       fatal("Unexpected type argument %s", type2name(type));
 9813       break;
 9814   }
 9815 }
 9816 
 9817 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9818   switch(type) {
 9819     case T_BOOLEAN:
 9820     case T_BYTE:
 9821        kxorbl(dst, src1, src2);
 9822        break;
 9823     case T_CHAR:
 9824     case T_SHORT:
 9825        kxorwl(dst, src1, src2);
 9826        break;
 9827     case T_INT:
 9828     case T_FLOAT:
 9829        kxordl(dst, src1, src2);
 9830        break;
 9831     case T_LONG:
 9832     case T_DOUBLE:
 9833        kxorql(dst, src1, src2);
 9834        break;
 9835     default:
 9836       fatal("Unexpected type argument %s", type2name(type));
 9837       break;
 9838   }
 9839 }
 9840 
 9841 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9842   switch(type) {
 9843     case T_BOOLEAN:
 9844     case T_BYTE:
 9845       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9846     case T_CHAR:
 9847     case T_SHORT:
 9848       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9849     case T_INT:
 9850     case T_FLOAT:
 9851       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9852     case T_LONG:
 9853     case T_DOUBLE:
 9854       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9855     default:
 9856       fatal("Unexpected type argument %s", type2name(type)); break;
 9857   }
 9858 }
 9859 
 9860 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9861   switch(type) {
 9862     case T_BOOLEAN:
 9863     case T_BYTE:
 9864       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9865     case T_CHAR:
 9866     case T_SHORT:
 9867       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9868     case T_INT:
 9869     case T_FLOAT:
 9870       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9871     case T_LONG:
 9872     case T_DOUBLE:
 9873       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9874     default:
 9875       fatal("Unexpected type argument %s", type2name(type)); break;
 9876   }
 9877 }
 9878 
 9879 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9880   switch(type) {
 9881     case T_BYTE:
 9882       evpminub(dst, mask, nds, src, merge, vector_len); break;
 9883     case T_SHORT:
 9884       evpminuw(dst, mask, nds, src, merge, vector_len); break;
 9885     case T_INT:
 9886       evpminud(dst, mask, nds, src, merge, vector_len); break;
 9887     case T_LONG:
 9888       evpminuq(dst, mask, nds, src, merge, vector_len); break;
 9889     default:
 9890       fatal("Unexpected type argument %s", type2name(type)); break;
 9891   }
 9892 }
 9893 
 9894 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9895   switch(type) {
 9896     case T_BYTE:
 9897       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
 9898     case T_SHORT:
 9899       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
 9900     case T_INT:
 9901       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
 9902     case T_LONG:
 9903       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
 9904     default:
 9905       fatal("Unexpected type argument %s", type2name(type)); break;
 9906   }
 9907 }
 9908 
 9909 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9910   switch(type) {
 9911     case T_BYTE:
 9912       evpminub(dst, mask, nds, src, merge, vector_len); break;
 9913     case T_SHORT:
 9914       evpminuw(dst, mask, nds, src, merge, vector_len); break;
 9915     case T_INT:
 9916       evpminud(dst, mask, nds, src, merge, vector_len); break;
 9917     case T_LONG:
 9918       evpminuq(dst, mask, nds, src, merge, vector_len); break;
 9919     default:
 9920       fatal("Unexpected type argument %s", type2name(type)); break;
 9921   }
 9922 }
 9923 
 9924 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9925   switch(type) {
 9926     case T_BYTE:
 9927       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
 9928     case T_SHORT:
 9929       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
 9930     case T_INT:
 9931       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
 9932     case T_LONG:
 9933       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
 9934     default:
 9935       fatal("Unexpected type argument %s", type2name(type)); break;
 9936   }
 9937 }
 9938 
 9939 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9940   switch(type) {
 9941     case T_BYTE:
 9942       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9943     case T_SHORT:
 9944       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9945     case T_INT:
 9946       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9947     case T_LONG:
 9948       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9949     default:
 9950       fatal("Unexpected type argument %s", type2name(type)); break;
 9951   }
 9952 }
 9953 
 9954 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9955   switch(type) {
 9956     case T_BYTE:
 9957       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9958     case T_SHORT:
 9959       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9960     case T_INT:
 9961       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9962     case T_LONG:
 9963       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9964     default:
 9965       fatal("Unexpected type argument %s", type2name(type)); break;
 9966   }
 9967 }
 9968 
 9969 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9970   switch(type) {
 9971     case T_BYTE:
 9972       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9973     case T_SHORT:
 9974       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9975     case T_INT:
 9976       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9977     case T_LONG:
 9978       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9979     default:
 9980       fatal("Unexpected type argument %s", type2name(type)); break;
 9981   }
 9982 }
 9983 
 9984 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9985   switch(type) {
 9986     case T_BYTE:
 9987       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9988     case T_SHORT:
 9989       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9990     case T_INT:
 9991       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9992     case T_LONG:
 9993       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9994     default:
 9995       fatal("Unexpected type argument %s", type2name(type)); break;
 9996   }
 9997 }
 9998 
 9999 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10000   switch(type) {
10001     case T_INT:
10002       evpxord(dst, mask, nds, src, merge, vector_len); break;
10003     case T_LONG:
10004       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10005     default:
10006       fatal("Unexpected type argument %s", type2name(type)); break;
10007   }
10008 }
10009 
10010 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10011   switch(type) {
10012     case T_INT:
10013       evpxord(dst, mask, nds, src, merge, vector_len); break;
10014     case T_LONG:
10015       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10016     default:
10017       fatal("Unexpected type argument %s", type2name(type)); break;
10018   }
10019 }
10020 
10021 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10022   switch(type) {
10023     case T_INT:
10024       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10025     case T_LONG:
10026       evporq(dst, mask, nds, src, merge, vector_len); break;
10027     default:
10028       fatal("Unexpected type argument %s", type2name(type)); break;
10029   }
10030 }
10031 
10032 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10033   switch(type) {
10034     case T_INT:
10035       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10036     case T_LONG:
10037       evporq(dst, mask, nds, src, merge, vector_len); break;
10038     default:
10039       fatal("Unexpected type argument %s", type2name(type)); break;
10040   }
10041 }
10042 
10043 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10044   switch(type) {
10045     case T_INT:
10046       evpandd(dst, mask, nds, src, merge, vector_len); break;
10047     case T_LONG:
10048       evpandq(dst, mask, nds, src, merge, vector_len); break;
10049     default:
10050       fatal("Unexpected type argument %s", type2name(type)); break;
10051   }
10052 }
10053 
10054 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10055   switch(type) {
10056     case T_INT:
10057       evpandd(dst, mask, nds, src, merge, vector_len); break;
10058     case T_LONG:
10059       evpandq(dst, mask, nds, src, merge, vector_len); break;
10060     default:
10061       fatal("Unexpected type argument %s", type2name(type)); break;
10062   }
10063 }
10064 
10065 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
10066   switch(masklen) {
10067     case 8:
10068        kortestbl(src1, src2);
10069        break;
10070     case 16:
10071        kortestwl(src1, src2);
10072        break;
10073     case 32:
10074        kortestdl(src1, src2);
10075        break;
10076     case 64:
10077        kortestql(src1, src2);
10078        break;
10079     default:
10080       fatal("Unexpected mask length %d", masklen);
10081       break;
10082   }
10083 }
10084 
10085 
10086 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
10087   switch(masklen)  {
10088     case 8:
10089        ktestbl(src1, src2);
10090        break;
10091     case 16:
10092        ktestwl(src1, src2);
10093        break;
10094     case 32:
10095        ktestdl(src1, src2);
10096        break;
10097     case 64:
10098        ktestql(src1, src2);
10099        break;
10100     default:
10101       fatal("Unexpected mask length %d", masklen);
10102       break;
10103   }
10104 }
10105 
10106 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10107   switch(type) {
10108     case T_INT:
10109       evprold(dst, mask, src, shift, merge, vlen_enc); break;
10110     case T_LONG:
10111       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
10112     default:
10113       fatal("Unexpected type argument %s", type2name(type)); break;
10114       break;
10115   }
10116 }
10117 
10118 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10119   switch(type) {
10120     case T_INT:
10121       evprord(dst, mask, src, shift, merge, vlen_enc); break;
10122     case T_LONG:
10123       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
10124     default:
10125       fatal("Unexpected type argument %s", type2name(type)); break;
10126   }
10127 }
10128 
10129 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10130   switch(type) {
10131     case T_INT:
10132       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
10133     case T_LONG:
10134       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
10135     default:
10136       fatal("Unexpected type argument %s", type2name(type)); break;
10137   }
10138 }
10139 
10140 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10141   switch(type) {
10142     case T_INT:
10143       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
10144     case T_LONG:
10145       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
10146     default:
10147       fatal("Unexpected type argument %s", type2name(type)); break;
10148   }
10149 }
10150 
10151 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10152   assert(rscratch != noreg || always_reachable(src), "missing");
10153 
10154   if (reachable(src)) {
10155     evpandq(dst, nds, as_Address(src), vector_len);
10156   } else {
10157     lea(rscratch, src);
10158     evpandq(dst, nds, Address(rscratch, 0), vector_len);
10159   }
10160 }
10161 
10162 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
10163   assert(rscratch != noreg || always_reachable(src), "missing");
10164 
10165   if (reachable(src)) {
10166     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
10167   } else {
10168     lea(rscratch, src);
10169     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
10170   }
10171 }
10172 
10173 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10174   assert(rscratch != noreg || always_reachable(src), "missing");
10175 
10176   if (reachable(src)) {
10177     evporq(dst, nds, as_Address(src), vector_len);
10178   } else {
10179     lea(rscratch, src);
10180     evporq(dst, nds, Address(rscratch, 0), vector_len);
10181   }
10182 }
10183 
10184 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10185   assert(rscratch != noreg || always_reachable(src), "missing");
10186 
10187   if (reachable(src)) {
10188     vpshufb(dst, nds, as_Address(src), vector_len);
10189   } else {
10190     lea(rscratch, src);
10191     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
10192   }
10193 }
10194 
10195 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10196   assert(rscratch != noreg || always_reachable(src), "missing");
10197 
10198   if (reachable(src)) {
10199     Assembler::vpor(dst, nds, as_Address(src), vector_len);
10200   } else {
10201     lea(rscratch, src);
10202     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
10203   }
10204 }
10205 
10206 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
10207   assert(rscratch != noreg || always_reachable(src3), "missing");
10208 
10209   if (reachable(src3)) {
10210     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
10211   } else {
10212     lea(rscratch, src3);
10213     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
10214   }
10215 }
10216 
10217 #if COMPILER2_OR_JVMCI
10218 
10219 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
10220                                  Register length, Register temp, int vec_enc) {
10221   // Computing mask for predicated vector store.
10222   movptr(temp, -1);
10223   bzhiq(temp, temp, length);
10224   kmov(mask, temp);
10225   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
10226 }
10227 
10228 // Set memory operation for length "less than" 64 bytes.
10229 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
10230                                        XMMRegister xmm, KRegister mask, Register length,
10231                                        Register temp, bool use64byteVector) {
10232   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10233   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10234   if (!use64byteVector) {
10235     fill32(dst, disp, xmm);
10236     subptr(length, 32 >> shift);
10237     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
10238   } else {
10239     assert(MaxVectorSize == 64, "vector length != 64");
10240     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
10241   }
10242 }
10243 
10244 
10245 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
10246                                        XMMRegister xmm, KRegister mask, Register length,
10247                                        Register temp) {
10248   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10249   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10250   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
10251 }
10252 
10253 
10254 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
10255   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10256   vmovdqu(dst, xmm);
10257 }
10258 
10259 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
10260   fill32(Address(dst, disp), xmm);
10261 }
10262 
10263 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
10264   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10265   if (!use64byteVector) {
10266     fill32(dst, xmm);
10267     fill32(dst.plus_disp(32), xmm);
10268   } else {
10269     evmovdquq(dst, xmm, Assembler::AVX_512bit);
10270   }
10271 }
10272 
10273 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
10274   fill64(Address(dst, disp), xmm, use64byteVector);
10275 }
10276 
10277 #ifdef _LP64
10278 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
10279                                         Register count, Register rtmp, XMMRegister xtmp) {
10280   Label L_exit;
10281   Label L_fill_start;
10282   Label L_fill_64_bytes;
10283   Label L_fill_96_bytes;
10284   Label L_fill_128_bytes;
10285   Label L_fill_128_bytes_loop;
10286   Label L_fill_128_loop_header;
10287   Label L_fill_128_bytes_loop_header;
10288   Label L_fill_128_bytes_loop_pre_header;
10289   Label L_fill_zmm_sequence;
10290 
10291   int shift = -1;
10292   int avx3threshold = VM_Version::avx3_threshold();
10293   switch(type) {
10294     case T_BYTE:  shift = 0;
10295       break;
10296     case T_SHORT: shift = 1;
10297       break;
10298     case T_INT:   shift = 2;
10299       break;
10300     /* Uncomment when LONG fill stubs are supported.
10301     case T_LONG:  shift = 3;
10302       break;
10303     */
10304     default:
10305       fatal("Unhandled type: %s\n", type2name(type));
10306   }
10307 
10308   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
10309 
10310     if (MaxVectorSize == 64) {
10311       cmpq(count, avx3threshold >> shift);
10312       jcc(Assembler::greater, L_fill_zmm_sequence);
10313     }
10314 
10315     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
10316 
10317     bind(L_fill_start);
10318 
10319     cmpq(count, 32 >> shift);
10320     jccb(Assembler::greater, L_fill_64_bytes);
10321     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
10322     jmp(L_exit);
10323 
10324     bind(L_fill_64_bytes);
10325     cmpq(count, 64 >> shift);
10326     jccb(Assembler::greater, L_fill_96_bytes);
10327     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
10328     jmp(L_exit);
10329 
10330     bind(L_fill_96_bytes);
10331     cmpq(count, 96 >> shift);
10332     jccb(Assembler::greater, L_fill_128_bytes);
10333     fill64(to, 0, xtmp);
10334     subq(count, 64 >> shift);
10335     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
10336     jmp(L_exit);
10337 
10338     bind(L_fill_128_bytes);
10339     cmpq(count, 128 >> shift);
10340     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
10341     fill64(to, 0, xtmp);
10342     fill32(to, 64, xtmp);
10343     subq(count, 96 >> shift);
10344     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
10345     jmp(L_exit);
10346 
10347     bind(L_fill_128_bytes_loop_pre_header);
10348     {
10349       mov(rtmp, to);
10350       andq(rtmp, 31);
10351       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
10352       negq(rtmp);
10353       addq(rtmp, 32);
10354       mov64(r8, -1L);
10355       bzhiq(r8, r8, rtmp);
10356       kmovql(k2, r8);
10357       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
10358       addq(to, rtmp);
10359       shrq(rtmp, shift);
10360       subq(count, rtmp);
10361     }
10362 
10363     cmpq(count, 128 >> shift);
10364     jcc(Assembler::less, L_fill_start);
10365 
10366     bind(L_fill_128_bytes_loop_header);
10367     subq(count, 128 >> shift);
10368 
10369     align32();
10370     bind(L_fill_128_bytes_loop);
10371       fill64(to, 0, xtmp);
10372       fill64(to, 64, xtmp);
10373       addq(to, 128);
10374       subq(count, 128 >> shift);
10375       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
10376 
10377     addq(count, 128 >> shift);
10378     jcc(Assembler::zero, L_exit);
10379     jmp(L_fill_start);
10380   }
10381 
10382   if (MaxVectorSize == 64) {
10383     // Sequence using 64 byte ZMM register.
10384     Label L_fill_128_bytes_zmm;
10385     Label L_fill_192_bytes_zmm;
10386     Label L_fill_192_bytes_loop_zmm;
10387     Label L_fill_192_bytes_loop_header_zmm;
10388     Label L_fill_192_bytes_loop_pre_header_zmm;
10389     Label L_fill_start_zmm_sequence;
10390 
10391     bind(L_fill_zmm_sequence);
10392     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
10393 
10394     bind(L_fill_start_zmm_sequence);
10395     cmpq(count, 64 >> shift);
10396     jccb(Assembler::greater, L_fill_128_bytes_zmm);
10397     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
10398     jmp(L_exit);
10399 
10400     bind(L_fill_128_bytes_zmm);
10401     cmpq(count, 128 >> shift);
10402     jccb(Assembler::greater, L_fill_192_bytes_zmm);
10403     fill64(to, 0, xtmp, true);
10404     subq(count, 64 >> shift);
10405     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
10406     jmp(L_exit);
10407 
10408     bind(L_fill_192_bytes_zmm);
10409     cmpq(count, 192 >> shift);
10410     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
10411     fill64(to, 0, xtmp, true);
10412     fill64(to, 64, xtmp, true);
10413     subq(count, 128 >> shift);
10414     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
10415     jmp(L_exit);
10416 
10417     bind(L_fill_192_bytes_loop_pre_header_zmm);
10418     {
10419       movq(rtmp, to);
10420       andq(rtmp, 63);
10421       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
10422       negq(rtmp);
10423       addq(rtmp, 64);
10424       mov64(r8, -1L);
10425       bzhiq(r8, r8, rtmp);
10426       kmovql(k2, r8);
10427       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
10428       addq(to, rtmp);
10429       shrq(rtmp, shift);
10430       subq(count, rtmp);
10431     }
10432 
10433     cmpq(count, 192 >> shift);
10434     jcc(Assembler::less, L_fill_start_zmm_sequence);
10435 
10436     bind(L_fill_192_bytes_loop_header_zmm);
10437     subq(count, 192 >> shift);
10438 
10439     align32();
10440     bind(L_fill_192_bytes_loop_zmm);
10441       fill64(to, 0, xtmp, true);
10442       fill64(to, 64, xtmp, true);
10443       fill64(to, 128, xtmp, true);
10444       addq(to, 192);
10445       subq(count, 192 >> shift);
10446       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10447 
10448     addq(count, 192 >> shift);
10449     jcc(Assembler::zero, L_exit);
10450     jmp(L_fill_start_zmm_sequence);
10451   }
10452   bind(L_exit);
10453 }
10454 #endif
10455 #endif //COMPILER2_OR_JVMCI
10456 
10457 
10458 #ifdef _LP64
10459 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10460   Label done;
10461   cvttss2sil(dst, src);
10462   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10463   cmpl(dst, 0x80000000); // float_sign_flip
10464   jccb(Assembler::notEqual, done);
10465   subptr(rsp, 8);
10466   movflt(Address(rsp, 0), src);
10467   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10468   pop(dst);
10469   bind(done);
10470 }
10471 
10472 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10473   Label done;
10474   cvttsd2sil(dst, src);
10475   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10476   cmpl(dst, 0x80000000); // float_sign_flip
10477   jccb(Assembler::notEqual, done);
10478   subptr(rsp, 8);
10479   movdbl(Address(rsp, 0), src);
10480   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10481   pop(dst);
10482   bind(done);
10483 }
10484 
10485 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10486   Label done;
10487   cvttss2siq(dst, src);
10488   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10489   jccb(Assembler::notEqual, done);
10490   subptr(rsp, 8);
10491   movflt(Address(rsp, 0), src);
10492   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10493   pop(dst);
10494   bind(done);
10495 }
10496 
10497 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10498   // Following code is line by line assembly translation rounding algorithm.
10499   // Please refer to java.lang.Math.round(float) algorithm for details.
10500   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10501   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10502   const int32_t FloatConsts_EXP_BIAS = 127;
10503   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10504   const int32_t MINUS_32 = 0xFFFFFFE0;
10505   Label L_special_case, L_block1, L_exit;
10506   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10507   movdl(dst, src);
10508   andl(dst, rtmp);
10509   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10510   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10511   subl(rtmp, dst);
10512   movl(rcx, rtmp);
10513   movl(dst, MINUS_32);
10514   testl(rtmp, dst);
10515   jccb(Assembler::notEqual, L_special_case);
10516   movdl(dst, src);
10517   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10518   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10519   movdl(rtmp, src);
10520   testl(rtmp, rtmp);
10521   jccb(Assembler::greaterEqual, L_block1);
10522   negl(dst);
10523   bind(L_block1);
10524   sarl(dst);
10525   addl(dst, 0x1);
10526   sarl(dst, 0x1);
10527   jmp(L_exit);
10528   bind(L_special_case);
10529   convert_f2i(dst, src);
10530   bind(L_exit);
10531 }
10532 
10533 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10534   // Following code is line by line assembly translation rounding algorithm.
10535   // Please refer to java.lang.Math.round(double) algorithm for details.
10536   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10537   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10538   const int64_t DoubleConsts_EXP_BIAS = 1023;
10539   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10540   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10541   Label L_special_case, L_block1, L_exit;
10542   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10543   movq(dst, src);
10544   andq(dst, rtmp);
10545   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10546   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10547   subq(rtmp, dst);
10548   movq(rcx, rtmp);
10549   mov64(dst, MINUS_64);
10550   testq(rtmp, dst);
10551   jccb(Assembler::notEqual, L_special_case);
10552   movq(dst, src);
10553   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10554   andq(dst, rtmp);
10555   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10556   orq(dst, rtmp);
10557   movq(rtmp, src);
10558   testq(rtmp, rtmp);
10559   jccb(Assembler::greaterEqual, L_block1);
10560   negq(dst);
10561   bind(L_block1);
10562   sarq(dst);
10563   addq(dst, 0x1);
10564   sarq(dst, 0x1);
10565   jmp(L_exit);
10566   bind(L_special_case);
10567   convert_d2l(dst, src);
10568   bind(L_exit);
10569 }
10570 
10571 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10572   Label done;
10573   cvttsd2siq(dst, src);
10574   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10575   jccb(Assembler::notEqual, done);
10576   subptr(rsp, 8);
10577   movdbl(Address(rsp, 0), src);
10578   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10579   pop(dst);
10580   bind(done);
10581 }
10582 
10583 void MacroAssembler::cache_wb(Address line)
10584 {
10585   // 64 bit cpus always support clflush
10586   assert(VM_Version::supports_clflush(), "clflush should be available");
10587   bool optimized = VM_Version::supports_clflushopt();
10588   bool no_evict = VM_Version::supports_clwb();
10589 
10590   // prefer clwb (writeback without evict) otherwise
10591   // prefer clflushopt (potentially parallel writeback with evict)
10592   // otherwise fallback on clflush (serial writeback with evict)
10593 
10594   if (optimized) {
10595     if (no_evict) {
10596       clwb(line);
10597     } else {
10598       clflushopt(line);
10599     }
10600   } else {
10601     // no need for fence when using CLFLUSH
10602     clflush(line);
10603   }
10604 }
10605 
10606 void MacroAssembler::cache_wbsync(bool is_pre)
10607 {
10608   assert(VM_Version::supports_clflush(), "clflush should be available");
10609   bool optimized = VM_Version::supports_clflushopt();
10610   bool no_evict = VM_Version::supports_clwb();
10611 
10612   // pick the correct implementation
10613 
10614   if (!is_pre && (optimized || no_evict)) {
10615     // need an sfence for post flush when using clflushopt or clwb
10616     // otherwise no no need for any synchroniaztion
10617 
10618     sfence();
10619   }
10620 }
10621 
10622 #endif // _LP64
10623 
10624 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10625   switch (cond) {
10626     // Note some conditions are synonyms for others
10627     case Assembler::zero:         return Assembler::notZero;
10628     case Assembler::notZero:      return Assembler::zero;
10629     case Assembler::less:         return Assembler::greaterEqual;
10630     case Assembler::lessEqual:    return Assembler::greater;
10631     case Assembler::greater:      return Assembler::lessEqual;
10632     case Assembler::greaterEqual: return Assembler::less;
10633     case Assembler::below:        return Assembler::aboveEqual;
10634     case Assembler::belowEqual:   return Assembler::above;
10635     case Assembler::above:        return Assembler::belowEqual;
10636     case Assembler::aboveEqual:   return Assembler::below;
10637     case Assembler::overflow:     return Assembler::noOverflow;
10638     case Assembler::noOverflow:   return Assembler::overflow;
10639     case Assembler::negative:     return Assembler::positive;
10640     case Assembler::positive:     return Assembler::negative;
10641     case Assembler::parity:       return Assembler::noParity;
10642     case Assembler::noParity:     return Assembler::parity;
10643   }
10644   ShouldNotReachHere(); return Assembler::overflow;
10645 }
10646 
10647 // This is simply a call to Thread::current()
10648 void MacroAssembler::get_thread(Register thread) {
10649   if (thread != rax) {
10650     push(rax);
10651   }
10652   LP64_ONLY(push(rdi);)
10653   LP64_ONLY(push(rsi);)
10654   push(rdx);
10655   push(rcx);
10656 #ifdef _LP64
10657   push(r8);
10658   push(r9);
10659   push(r10);
10660   push(r11);
10661 #endif
10662 
10663   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10664 
10665 #ifdef _LP64
10666   pop(r11);
10667   pop(r10);
10668   pop(r9);
10669   pop(r8);
10670 #endif
10671   pop(rcx);
10672   pop(rdx);
10673   LP64_ONLY(pop(rsi);)
10674   LP64_ONLY(pop(rdi);)
10675   if (thread != rax) {
10676     mov(thread, rax);
10677     pop(rax);
10678   }
10679 }
10680 
10681 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10682   Label L_stack_ok;
10683   if (bias == 0) {
10684     testptr(sp, 2 * wordSize - 1);
10685   } else {
10686     // lea(tmp, Address(rsp, bias);
10687     mov(tmp, sp);
10688     addptr(tmp, bias);
10689     testptr(tmp, 2 * wordSize - 1);
10690   }
10691   jcc(Assembler::equal, L_stack_ok);
10692   block_comment(msg);
10693   stop(msg);
10694   bind(L_stack_ok);
10695 }
10696 
10697 // Implements lightweight-locking.
10698 //
10699 // obj: the object to be locked
10700 // reg_rax: rax
10701 // thread: the thread which attempts to lock obj
10702 // tmp: a temporary register
10703 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10704   assert(reg_rax == rax, "");
10705   assert_different_registers(basic_lock, obj, reg_rax, thread, tmp);
10706 
10707   Label push;
10708   const Register top = tmp;
10709 
10710   // Preload the markWord. It is important that this is the first
10711   // instruction emitted as it is part of C1's null check semantics.
10712   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10713 
10714   if (UseObjectMonitorTable) {
10715     // Clear cache in case fast locking succeeds.
10716     movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0);
10717   }
10718 
10719   // Load top.
10720   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10721 
10722   // Check if the lock-stack is full.
10723   cmpl(top, LockStack::end_offset());
10724   jcc(Assembler::greaterEqual, slow);
10725 
10726   // Check for recursion.
10727   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10728   jcc(Assembler::equal, push);
10729 
10730   // Check header for monitor (0b10).
10731   testptr(reg_rax, markWord::monitor_value);
10732   jcc(Assembler::notZero, slow);
10733 
10734   // Try to lock. Transition lock bits 0b01 => 0b00
10735   movptr(tmp, reg_rax);
10736   andptr(tmp, ~(int32_t)markWord::unlocked_value);
10737   orptr(reg_rax, markWord::unlocked_value);
10738   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10739   jcc(Assembler::notEqual, slow);
10740 
10741   // Restore top, CAS clobbers register.
10742   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10743 
10744   bind(push);
10745   // After successful lock, push object on lock-stack.
10746   movptr(Address(thread, top), obj);
10747   incrementl(top, oopSize);
10748   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
10749 }
10750 
10751 // Implements lightweight-unlocking.
10752 //
10753 // obj: the object to be unlocked
10754 // reg_rax: rax
10755 // thread: the thread
10756 // tmp: a temporary register
10757 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10758   assert(reg_rax == rax, "");
10759   assert_different_registers(obj, reg_rax, thread, tmp);
10760 
10761   Label unlocked, push_and_slow;
10762   const Register top = tmp;
10763 
10764   // Check if obj is top of lock-stack.
10765   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10766   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10767   jcc(Assembler::notEqual, slow);
10768 
10769   // Pop lock-stack.
10770   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
10771   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10772 
10773   // Check if recursive.
10774   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
10775   jcc(Assembler::equal, unlocked);
10776 
10777   // Not recursive. Check header for monitor (0b10).
10778   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10779   testptr(reg_rax, markWord::monitor_value);
10780   jcc(Assembler::notZero, push_and_slow);
10781 
10782 #ifdef ASSERT
10783   // Check header not unlocked (0b01).
10784   Label not_unlocked;
10785   testptr(reg_rax, markWord::unlocked_value);
10786   jcc(Assembler::zero, not_unlocked);
10787   stop("lightweight_unlock already unlocked");
10788   bind(not_unlocked);
10789 #endif
10790 
10791   // Try to unlock. Transition lock bits 0b00 => 0b01
10792   movptr(tmp, reg_rax);
10793   orptr(tmp, markWord::unlocked_value);
10794   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10795   jcc(Assembler::equal, unlocked);
10796 
10797   bind(push_and_slow);
10798   // Restore lock-stack and handle the unlock in runtime.
10799 #ifdef ASSERT
10800   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10801   movptr(Address(thread, top), obj);
10802 #endif
10803   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10804   jmp(slow);
10805 
10806   bind(unlocked);
10807 }
10808 
10809 #ifdef _LP64
10810 // Saves legacy GPRs state on stack.
10811 void MacroAssembler::save_legacy_gprs() {
10812   subq(rsp, 16 * wordSize);
10813   movq(Address(rsp, 15 * wordSize), rax);
10814   movq(Address(rsp, 14 * wordSize), rcx);
10815   movq(Address(rsp, 13 * wordSize), rdx);
10816   movq(Address(rsp, 12 * wordSize), rbx);
10817   movq(Address(rsp, 10 * wordSize), rbp);
10818   movq(Address(rsp, 9 * wordSize), rsi);
10819   movq(Address(rsp, 8 * wordSize), rdi);
10820   movq(Address(rsp, 7 * wordSize), r8);
10821   movq(Address(rsp, 6 * wordSize), r9);
10822   movq(Address(rsp, 5 * wordSize), r10);
10823   movq(Address(rsp, 4 * wordSize), r11);
10824   movq(Address(rsp, 3 * wordSize), r12);
10825   movq(Address(rsp, 2 * wordSize), r13);
10826   movq(Address(rsp, wordSize), r14);
10827   movq(Address(rsp, 0), r15);
10828 }
10829 
10830 // Resotres back legacy GPRs state from stack.
10831 void MacroAssembler::restore_legacy_gprs() {
10832   movq(r15, Address(rsp, 0));
10833   movq(r14, Address(rsp, wordSize));
10834   movq(r13, Address(rsp, 2 * wordSize));
10835   movq(r12, Address(rsp, 3 * wordSize));
10836   movq(r11, Address(rsp, 4 * wordSize));
10837   movq(r10, Address(rsp, 5 * wordSize));
10838   movq(r9,  Address(rsp, 6 * wordSize));
10839   movq(r8,  Address(rsp, 7 * wordSize));
10840   movq(rdi, Address(rsp, 8 * wordSize));
10841   movq(rsi, Address(rsp, 9 * wordSize));
10842   movq(rbp, Address(rsp, 10 * wordSize));
10843   movq(rbx, Address(rsp, 12 * wordSize));
10844   movq(rdx, Address(rsp, 13 * wordSize));
10845   movq(rcx, Address(rsp, 14 * wordSize));
10846   movq(rax, Address(rsp, 15 * wordSize));
10847   addq(rsp, 16 * wordSize);
10848 }
10849 
10850 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
10851   if (VM_Version::supports_apx_f()) {
10852     esetzucc(comparison, dst);
10853   } else {
10854     setb(comparison, dst);
10855     movzbl(dst, dst);
10856   }
10857 }
10858 #endif