1 /*
    2  * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "code/compiledIC.hpp"
   29 #include "compiler/compiler_globals.hpp"
   30 #include "compiler/disassembler.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "jvm.h"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedKlass.inline.hpp"
   43 #include "oops/compressedOops.inline.hpp"
   44 #include "oops/klass.inline.hpp"
   45 #include "prims/methodHandles.hpp"
   46 #include "runtime/continuation.hpp"
   47 #include "runtime/interfaceSupport.inline.hpp"
   48 #include "runtime/javaThread.hpp"
   49 #include "runtime/jniHandles.hpp"
   50 #include "runtime/objectMonitor.hpp"
   51 #include "runtime/os.hpp"
   52 #include "runtime/safepoint.hpp"
   53 #include "runtime/safepointMechanism.hpp"
   54 #include "runtime/sharedRuntime.hpp"
   55 #include "runtime/stubRoutines.hpp"
   56 #include "utilities/checkedCast.hpp"
   57 #include "utilities/macros.hpp"
   58 
   59 #ifdef PRODUCT
   60 #define BLOCK_COMMENT(str) /* nothing */
   61 #define STOP(error) stop(error)
   62 #else
   63 #define BLOCK_COMMENT(str) block_comment(str)
   64 #define STOP(error) block_comment(error); stop(error)
   65 #endif
   66 
   67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   68 
   69 #ifdef ASSERT
   70 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   71 #endif
   72 
   73 static const Assembler::Condition reverse[] = {
   74     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   75     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   76     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   77     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   78     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   79     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   80     Assembler::above          /* belowEqual    = 0x6 */ ,
   81     Assembler::belowEqual     /* above         = 0x7 */ ,
   82     Assembler::positive       /* negative      = 0x8 */ ,
   83     Assembler::negative       /* positive      = 0x9 */ ,
   84     Assembler::noParity       /* parity        = 0xa */ ,
   85     Assembler::parity         /* noParity      = 0xb */ ,
   86     Assembler::greaterEqual   /* less          = 0xc */ ,
   87     Assembler::less           /* greaterEqual  = 0xd */ ,
   88     Assembler::greater        /* lessEqual     = 0xe */ ,
   89     Assembler::lessEqual      /* greater       = 0xf, */
   90 
   91 };
   92 
   93 
   94 // Implementation of MacroAssembler
   95 
   96 // First all the versions that have distinct versions depending on 32/64 bit
   97 // Unless the difference is trivial (1 line or so).
   98 
   99 #ifndef _LP64
  100 
  101 // 32bit versions
  102 
  103 Address MacroAssembler::as_Address(AddressLiteral adr) {
  104   return Address(adr.target(), adr.rspec());
  105 }
  106 
  107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  108   assert(rscratch == noreg, "");
  109   return Address::make_array(adr);
  110 }
  111 
  112 void MacroAssembler::call_VM_leaf_base(address entry_point,
  113                                        int number_of_arguments) {
  114   call(RuntimeAddress(entry_point));
  115   increment(rsp, number_of_arguments * wordSize);
  116 }
  117 
  118 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  119   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  120 }
  121 
  122 
  123 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  124   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  125 }
  126 
  127 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  128   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  129 }
  130 
  131 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  132   assert(rscratch == noreg, "redundant");
  133   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  134 }
  135 
  136 void MacroAssembler::extend_sign(Register hi, Register lo) {
  137   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  138   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  139     cdql();
  140   } else {
  141     movl(hi, lo);
  142     sarl(hi, 31);
  143   }
  144 }
  145 
  146 void MacroAssembler::jC2(Register tmp, Label& L) {
  147   // set parity bit if FPU flag C2 is set (via rax)
  148   save_rax(tmp);
  149   fwait(); fnstsw_ax();
  150   sahf();
  151   restore_rax(tmp);
  152   // branch
  153   jcc(Assembler::parity, L);
  154 }
  155 
  156 void MacroAssembler::jnC2(Register tmp, Label& L) {
  157   // set parity bit if FPU flag C2 is set (via rax)
  158   save_rax(tmp);
  159   fwait(); fnstsw_ax();
  160   sahf();
  161   restore_rax(tmp);
  162   // branch
  163   jcc(Assembler::noParity, L);
  164 }
  165 
  166 // 32bit can do a case table jump in one instruction but we no longer allow the base
  167 // to be installed in the Address class
  168 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  169   assert(rscratch == noreg, "not needed");
  170   jmp(as_Address(entry, noreg));
  171 }
  172 
  173 // Note: y_lo will be destroyed
  174 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  175   // Long compare for Java (semantics as described in JVM spec.)
  176   Label high, low, done;
  177 
  178   cmpl(x_hi, y_hi);
  179   jcc(Assembler::less, low);
  180   jcc(Assembler::greater, high);
  181   // x_hi is the return register
  182   xorl(x_hi, x_hi);
  183   cmpl(x_lo, y_lo);
  184   jcc(Assembler::below, low);
  185   jcc(Assembler::equal, done);
  186 
  187   bind(high);
  188   xorl(x_hi, x_hi);
  189   increment(x_hi);
  190   jmp(done);
  191 
  192   bind(low);
  193   xorl(x_hi, x_hi);
  194   decrementl(x_hi);
  195 
  196   bind(done);
  197 }
  198 
  199 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  200   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  201 }
  202 
  203 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  204   assert(rscratch == noreg, "not needed");
  205 
  206   // leal(dst, as_Address(adr));
  207   // see note in movl as to why we must use a move
  208   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  209 }
  210 
  211 void MacroAssembler::leave() {
  212   mov(rsp, rbp);
  213   pop(rbp);
  214 }
  215 
  216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  217   // Multiplication of two Java long values stored on the stack
  218   // as illustrated below. Result is in rdx:rax.
  219   //
  220   // rsp ---> [  ??  ] \               \
  221   //            ....    | y_rsp_offset  |
  222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  223   //          [ y_hi ]                  | (in bytes)
  224   //            ....                    |
  225   //          [ x_lo ]                 /
  226   //          [ x_hi ]
  227   //            ....
  228   //
  229   // Basic idea: lo(result) = lo(x_lo * y_lo)
  230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  233   Label quick;
  234   // load x_hi, y_hi and check if quick
  235   // multiplication is possible
  236   movl(rbx, x_hi);
  237   movl(rcx, y_hi);
  238   movl(rax, rbx);
  239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  241   // do full multiplication
  242   // 1st step
  243   mull(y_lo);                                    // x_hi * y_lo
  244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  245   // 2nd step
  246   movl(rax, x_lo);
  247   mull(rcx);                                     // x_lo * y_hi
  248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  249   // 3rd step
  250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  251   movl(rax, x_lo);
  252   mull(y_lo);                                    // x_lo * y_lo
  253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  254 }
  255 
  256 void MacroAssembler::lneg(Register hi, Register lo) {
  257   negl(lo);
  258   adcl(hi, 0);
  259   negl(hi);
  260 }
  261 
  262 void MacroAssembler::lshl(Register hi, Register lo) {
  263   // Java shift left long support (semantics as described in JVM spec., p.305)
  264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  265   // shift value is in rcx !
  266   assert(hi != rcx, "must not use rcx");
  267   assert(lo != rcx, "must not use rcx");
  268   const Register s = rcx;                        // shift count
  269   const int      n = BitsPerWord;
  270   Label L;
  271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  272   cmpl(s, n);                                    // if (s < n)
  273   jcc(Assembler::less, L);                       // else (s >= n)
  274   movl(hi, lo);                                  // x := x << n
  275   xorl(lo, lo);
  276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  277   bind(L);                                       // s (mod n) < n
  278   shldl(hi, lo);                                 // x := x << s
  279   shll(lo);
  280 }
  281 
  282 
  283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  286   assert(hi != rcx, "must not use rcx");
  287   assert(lo != rcx, "must not use rcx");
  288   const Register s = rcx;                        // shift count
  289   const int      n = BitsPerWord;
  290   Label L;
  291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  292   cmpl(s, n);                                    // if (s < n)
  293   jcc(Assembler::less, L);                       // else (s >= n)
  294   movl(lo, hi);                                  // x := x >> n
  295   if (sign_extension) sarl(hi, 31);
  296   else                xorl(hi, hi);
  297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  298   bind(L);                                       // s (mod n) < n
  299   shrdl(lo, hi);                                 // x := x >> s
  300   if (sign_extension) sarl(hi);
  301   else                shrl(hi);
  302 }
  303 
  304 void MacroAssembler::movoop(Register dst, jobject obj) {
  305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  306 }
  307 
  308 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  309   assert(rscratch == noreg, "redundant");
  310   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  311 }
  312 
  313 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  314   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  315 }
  316 
  317 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  318   assert(rscratch == noreg, "redundant");
  319   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  320 }
  321 
  322 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  323   if (src.is_lval()) {
  324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  325   } else {
  326     movl(dst, as_Address(src));
  327   }
  328 }
  329 
  330 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  331   assert(rscratch == noreg, "redundant");
  332   movl(as_Address(dst, noreg), src);
  333 }
  334 
  335 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  336   movl(dst, as_Address(src, noreg));
  337 }
  338 
  339 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  340   assert(rscratch == noreg, "redundant");
  341   movl(dst, src);
  342 }
  343 
  344 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  345   assert(rscratch == noreg, "redundant");
  346   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  347 }
  348 
  349 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  350   assert(rscratch == noreg, "redundant");
  351   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  352 }
  353 
  354 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  355   assert(rscratch == noreg, "redundant");
  356   if (src.is_lval()) {
  357     push_literal32((int32_t)src.target(), src.rspec());
  358   } else {
  359     pushl(as_Address(src));
  360   }
  361 }
  362 
  363 static void pass_arg0(MacroAssembler* masm, Register arg) {
  364   masm->push(arg);
  365 }
  366 
  367 static void pass_arg1(MacroAssembler* masm, Register arg) {
  368   masm->push(arg);
  369 }
  370 
  371 static void pass_arg2(MacroAssembler* masm, Register arg) {
  372   masm->push(arg);
  373 }
  374 
  375 static void pass_arg3(MacroAssembler* masm, Register arg) {
  376   masm->push(arg);
  377 }
  378 
  379 #ifndef PRODUCT
  380 extern "C" void findpc(intptr_t x);
  381 #endif
  382 
  383 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  384   // In order to get locks to work, we need to fake a in_VM state
  385   JavaThread* thread = JavaThread::current();
  386   JavaThreadState saved_state = thread->thread_state();
  387   thread->set_thread_state(_thread_in_vm);
  388   if (ShowMessageBoxOnError) {
  389     JavaThread* thread = JavaThread::current();
  390     JavaThreadState saved_state = thread->thread_state();
  391     thread->set_thread_state(_thread_in_vm);
  392     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  393       ttyLocker ttyl;
  394       BytecodeCounter::print();
  395     }
  396     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  397     // This is the value of eip which points to where verify_oop will return.
  398     if (os::message_box(msg, "Execution stopped, print registers?")) {
  399       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  400       BREAKPOINT;
  401     }
  402   }
  403   fatal("DEBUG MESSAGE: %s", msg);
  404 }
  405 
  406 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  407   ttyLocker ttyl;
  408   DebuggingContext debugging{};
  409   tty->print_cr("eip = 0x%08x", eip);
  410 #ifndef PRODUCT
  411   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  412     tty->cr();
  413     findpc(eip);
  414     tty->cr();
  415   }
  416 #endif
  417 #define PRINT_REG(rax) \
  418   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  419   PRINT_REG(rax);
  420   PRINT_REG(rbx);
  421   PRINT_REG(rcx);
  422   PRINT_REG(rdx);
  423   PRINT_REG(rdi);
  424   PRINT_REG(rsi);
  425   PRINT_REG(rbp);
  426   PRINT_REG(rsp);
  427 #undef PRINT_REG
  428   // Print some words near top of staack.
  429   int* dump_sp = (int*) rsp;
  430   for (int col1 = 0; col1 < 8; col1++) {
  431     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  432     os::print_location(tty, *dump_sp++);
  433   }
  434   for (int row = 0; row < 16; row++) {
  435     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  436     for (int col = 0; col < 8; col++) {
  437       tty->print(" 0x%08x", *dump_sp++);
  438     }
  439     tty->cr();
  440   }
  441   // Print some instructions around pc:
  442   Disassembler::decode((address)eip-64, (address)eip);
  443   tty->print_cr("--------");
  444   Disassembler::decode((address)eip, (address)eip+32);
  445 }
  446 
  447 void MacroAssembler::stop(const char* msg) {
  448   // push address of message
  449   ExternalAddress message((address)msg);
  450   pushptr(message.addr(), noreg);
  451   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  452   pusha();                                            // push registers
  453   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  454   hlt();
  455 }
  456 
  457 void MacroAssembler::warn(const char* msg) {
  458   push_CPU_state();
  459 
  460   // push address of message
  461   ExternalAddress message((address)msg);
  462   pushptr(message.addr(), noreg);
  463 
  464   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  465   addl(rsp, wordSize);       // discard argument
  466   pop_CPU_state();
  467 }
  468 
  469 void MacroAssembler::print_state() {
  470   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  471   pusha();                                            // push registers
  472 
  473   push_CPU_state();
  474   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  475   pop_CPU_state();
  476 
  477   popa();
  478   addl(rsp, wordSize);
  479 }
  480 
  481 #else // _LP64
  482 
  483 // 64 bit versions
  484 
  485 Address MacroAssembler::as_Address(AddressLiteral adr) {
  486   // amd64 always does this as a pc-rel
  487   // we can be absolute or disp based on the instruction type
  488   // jmp/call are displacements others are absolute
  489   assert(!adr.is_lval(), "must be rval");
  490   assert(reachable(adr), "must be");
  491   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  492 
  493 }
  494 
  495 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  496   AddressLiteral base = adr.base();
  497   lea(rscratch, base);
  498   Address index = adr.index();
  499   assert(index._disp == 0, "must not have disp"); // maybe it can?
  500   Address array(rscratch, index._index, index._scale, index._disp);
  501   return array;
  502 }
  503 
  504 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  505   Label L, E;
  506 
  507 #ifdef _WIN64
  508   // Windows always allocates space for it's register args
  509   assert(num_args <= 4, "only register arguments supported");
  510   subq(rsp,  frame::arg_reg_save_area_bytes);
  511 #endif
  512 
  513   // Align stack if necessary
  514   testl(rsp, 15);
  515   jcc(Assembler::zero, L);
  516 
  517   subq(rsp, 8);
  518   call(RuntimeAddress(entry_point));
  519   addq(rsp, 8);
  520   jmp(E);
  521 
  522   bind(L);
  523   call(RuntimeAddress(entry_point));
  524 
  525   bind(E);
  526 
  527 #ifdef _WIN64
  528   // restore stack pointer
  529   addq(rsp, frame::arg_reg_save_area_bytes);
  530 #endif
  531 
  532 }
  533 
  534 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  535   assert(!src2.is_lval(), "should use cmpptr");
  536   assert(rscratch != noreg || always_reachable(src2), "missing");
  537 
  538   if (reachable(src2)) {
  539     cmpq(src1, as_Address(src2));
  540   } else {
  541     lea(rscratch, src2);
  542     Assembler::cmpq(src1, Address(rscratch, 0));
  543   }
  544 }
  545 
  546 int MacroAssembler::corrected_idivq(Register reg) {
  547   // Full implementation of Java ldiv and lrem; checks for special
  548   // case as described in JVM spec., p.243 & p.271.  The function
  549   // returns the (pc) offset of the idivl instruction - may be needed
  550   // for implicit exceptions.
  551   //
  552   //         normal case                           special case
  553   //
  554   // input : rax: dividend                         min_long
  555   //         reg: divisor   (may not be eax/edx)   -1
  556   //
  557   // output: rax: quotient  (= rax idiv reg)       min_long
  558   //         rdx: remainder (= rax irem reg)       0
  559   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  560   static const int64_t min_long = 0x8000000000000000;
  561   Label normal_case, special_case;
  562 
  563   // check for special case
  564   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  565   jcc(Assembler::notEqual, normal_case);
  566   xorl(rdx, rdx); // prepare rdx for possible special case (where
  567                   // remainder = 0)
  568   cmpq(reg, -1);
  569   jcc(Assembler::equal, special_case);
  570 
  571   // handle normal case
  572   bind(normal_case);
  573   cdqq();
  574   int idivq_offset = offset();
  575   idivq(reg);
  576 
  577   // normal and special case exit
  578   bind(special_case);
  579 
  580   return idivq_offset;
  581 }
  582 
  583 void MacroAssembler::decrementq(Register reg, int value) {
  584   if (value == min_jint) { subq(reg, value); return; }
  585   if (value <  0) { incrementq(reg, -value); return; }
  586   if (value == 0) {                        ; return; }
  587   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  588   /* else */      { subq(reg, value)       ; return; }
  589 }
  590 
  591 void MacroAssembler::decrementq(Address dst, int value) {
  592   if (value == min_jint) { subq(dst, value); return; }
  593   if (value <  0) { incrementq(dst, -value); return; }
  594   if (value == 0) {                        ; return; }
  595   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  596   /* else */      { subq(dst, value)       ; return; }
  597 }
  598 
  599 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  600   assert(rscratch != noreg || always_reachable(dst), "missing");
  601 
  602   if (reachable(dst)) {
  603     incrementq(as_Address(dst));
  604   } else {
  605     lea(rscratch, dst);
  606     incrementq(Address(rscratch, 0));
  607   }
  608 }
  609 
  610 void MacroAssembler::incrementq(Register reg, int value) {
  611   if (value == min_jint) { addq(reg, value); return; }
  612   if (value <  0) { decrementq(reg, -value); return; }
  613   if (value == 0) {                        ; return; }
  614   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  615   /* else */      { addq(reg, value)       ; return; }
  616 }
  617 
  618 void MacroAssembler::incrementq(Address dst, int value) {
  619   if (value == min_jint) { addq(dst, value); return; }
  620   if (value <  0) { decrementq(dst, -value); return; }
  621   if (value == 0) {                        ; return; }
  622   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  623   /* else */      { addq(dst, value)       ; return; }
  624 }
  625 
  626 // 32bit can do a case table jump in one instruction but we no longer allow the base
  627 // to be installed in the Address class
  628 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  629   lea(rscratch, entry.base());
  630   Address dispatch = entry.index();
  631   assert(dispatch._base == noreg, "must be");
  632   dispatch._base = rscratch;
  633   jmp(dispatch);
  634 }
  635 
  636 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  637   ShouldNotReachHere(); // 64bit doesn't use two regs
  638   cmpq(x_lo, y_lo);
  639 }
  640 
  641 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  642   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  643 }
  644 
  645 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  646   lea(rscratch, adr);
  647   movptr(dst, rscratch);
  648 }
  649 
  650 void MacroAssembler::leave() {
  651   // %%% is this really better? Why not on 32bit too?
  652   emit_int8((unsigned char)0xC9); // LEAVE
  653 }
  654 
  655 void MacroAssembler::lneg(Register hi, Register lo) {
  656   ShouldNotReachHere(); // 64bit doesn't use two regs
  657   negq(lo);
  658 }
  659 
  660 void MacroAssembler::movoop(Register dst, jobject obj) {
  661   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  662 }
  663 
  664 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  665   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  666   movq(dst, rscratch);
  667 }
  668 
  669 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  670   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  671 }
  672 
  673 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  674   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  675   movq(dst, rscratch);
  676 }
  677 
  678 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  679   if (src.is_lval()) {
  680     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  681   } else {
  682     if (reachable(src)) {
  683       movq(dst, as_Address(src));
  684     } else {
  685       lea(dst, src);
  686       movq(dst, Address(dst, 0));
  687     }
  688   }
  689 }
  690 
  691 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  692   movq(as_Address(dst, rscratch), src);
  693 }
  694 
  695 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  696   movq(dst, as_Address(src, dst /*rscratch*/));
  697 }
  698 
  699 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  700 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  701   if (is_simm32(src)) {
  702     movptr(dst, checked_cast<int32_t>(src));
  703   } else {
  704     mov64(rscratch, src);
  705     movq(dst, rscratch);
  706   }
  707 }
  708 
  709 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  710   movoop(rscratch, obj);
  711   push(rscratch);
  712 }
  713 
  714 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  715   mov_metadata(rscratch, obj);
  716   push(rscratch);
  717 }
  718 
  719 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  720   lea(rscratch, src);
  721   if (src.is_lval()) {
  722     push(rscratch);
  723   } else {
  724     pushq(Address(rscratch, 0));
  725   }
  726 }
  727 
  728 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  729   reset_last_Java_frame(r15_thread, clear_fp);
  730 }
  731 
  732 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  733                                          Register last_java_fp,
  734                                          address  last_java_pc,
  735                                          Register rscratch) {
  736   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  737 }
  738 
  739 static void pass_arg0(MacroAssembler* masm, Register arg) {
  740   if (c_rarg0 != arg ) {
  741     masm->mov(c_rarg0, arg);
  742   }
  743 }
  744 
  745 static void pass_arg1(MacroAssembler* masm, Register arg) {
  746   if (c_rarg1 != arg ) {
  747     masm->mov(c_rarg1, arg);
  748   }
  749 }
  750 
  751 static void pass_arg2(MacroAssembler* masm, Register arg) {
  752   if (c_rarg2 != arg ) {
  753     masm->mov(c_rarg2, arg);
  754   }
  755 }
  756 
  757 static void pass_arg3(MacroAssembler* masm, Register arg) {
  758   if (c_rarg3 != arg ) {
  759     masm->mov(c_rarg3, arg);
  760   }
  761 }
  762 
  763 void MacroAssembler::stop(const char* msg) {
  764   if (ShowMessageBoxOnError) {
  765     address rip = pc();
  766     pusha(); // get regs on stack
  767     lea(c_rarg1, InternalAddress(rip));
  768     movq(c_rarg2, rsp); // pass pointer to regs array
  769   }
  770   lea(c_rarg0, ExternalAddress((address) msg));
  771   andq(rsp, -16); // align stack as required by ABI
  772   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  773   hlt();
  774 }
  775 
  776 void MacroAssembler::warn(const char* msg) {
  777   push(rbp);
  778   movq(rbp, rsp);
  779   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  780   push_CPU_state();   // keeps alignment at 16 bytes
  781 
  782   lea(c_rarg0, ExternalAddress((address) msg));
  783   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  784 
  785   pop_CPU_state();
  786   mov(rsp, rbp);
  787   pop(rbp);
  788 }
  789 
  790 void MacroAssembler::print_state() {
  791   address rip = pc();
  792   pusha();            // get regs on stack
  793   push(rbp);
  794   movq(rbp, rsp);
  795   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  796   push_CPU_state();   // keeps alignment at 16 bytes
  797 
  798   lea(c_rarg0, InternalAddress(rip));
  799   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  800   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  801 
  802   pop_CPU_state();
  803   mov(rsp, rbp);
  804   pop(rbp);
  805   popa();
  806 }
  807 
  808 #ifndef PRODUCT
  809 extern "C" void findpc(intptr_t x);
  810 #endif
  811 
  812 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  813   // In order to get locks to work, we need to fake a in_VM state
  814   if (ShowMessageBoxOnError) {
  815     JavaThread* thread = JavaThread::current();
  816     JavaThreadState saved_state = thread->thread_state();
  817     thread->set_thread_state(_thread_in_vm);
  818 #ifndef PRODUCT
  819     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  820       ttyLocker ttyl;
  821       BytecodeCounter::print();
  822     }
  823 #endif
  824     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  825     // XXX correct this offset for amd64
  826     // This is the value of eip which points to where verify_oop will return.
  827     if (os::message_box(msg, "Execution stopped, print registers?")) {
  828       print_state64(pc, regs);
  829       BREAKPOINT;
  830     }
  831   }
  832   fatal("DEBUG MESSAGE: %s", msg);
  833 }
  834 
  835 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  836   ttyLocker ttyl;
  837   DebuggingContext debugging{};
  838   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  839 #ifndef PRODUCT
  840   tty->cr();
  841   findpc(pc);
  842   tty->cr();
  843 #endif
  844 #define PRINT_REG(rax, value) \
  845   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  846   PRINT_REG(rax, regs[15]);
  847   PRINT_REG(rbx, regs[12]);
  848   PRINT_REG(rcx, regs[14]);
  849   PRINT_REG(rdx, regs[13]);
  850   PRINT_REG(rdi, regs[8]);
  851   PRINT_REG(rsi, regs[9]);
  852   PRINT_REG(rbp, regs[10]);
  853   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  854   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  855   PRINT_REG(r8 , regs[7]);
  856   PRINT_REG(r9 , regs[6]);
  857   PRINT_REG(r10, regs[5]);
  858   PRINT_REG(r11, regs[4]);
  859   PRINT_REG(r12, regs[3]);
  860   PRINT_REG(r13, regs[2]);
  861   PRINT_REG(r14, regs[1]);
  862   PRINT_REG(r15, regs[0]);
  863 #undef PRINT_REG
  864   // Print some words near the top of the stack.
  865   int64_t* rsp = &regs[16];
  866   int64_t* dump_sp = rsp;
  867   for (int col1 = 0; col1 < 8; col1++) {
  868     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  869     os::print_location(tty, *dump_sp++);
  870   }
  871   for (int row = 0; row < 25; row++) {
  872     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  873     for (int col = 0; col < 4; col++) {
  874       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  875     }
  876     tty->cr();
  877   }
  878   // Print some instructions around pc:
  879   Disassembler::decode((address)pc-64, (address)pc);
  880   tty->print_cr("--------");
  881   Disassembler::decode((address)pc, (address)pc+32);
  882 }
  883 
  884 // The java_calling_convention describes stack locations as ideal slots on
  885 // a frame with no abi restrictions. Since we must observe abi restrictions
  886 // (like the placement of the register window) the slots must be biased by
  887 // the following value.
  888 static int reg2offset_in(VMReg r) {
  889   // Account for saved rbp and return address
  890   // This should really be in_preserve_stack_slots
  891   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  892 }
  893 
  894 static int reg2offset_out(VMReg r) {
  895   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  896 }
  897 
  898 // A long move
  899 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  900 
  901   // The calling conventions assures us that each VMregpair is either
  902   // all really one physical register or adjacent stack slots.
  903 
  904   if (src.is_single_phys_reg() ) {
  905     if (dst.is_single_phys_reg()) {
  906       if (dst.first() != src.first()) {
  907         mov(dst.first()->as_Register(), src.first()->as_Register());
  908       }
  909     } else {
  910       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  911              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  912       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  913     }
  914   } else if (dst.is_single_phys_reg()) {
  915     assert(src.is_single_reg(),  "not a stack pair");
  916     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  917   } else {
  918     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  919     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  920     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  921   }
  922 }
  923 
  924 // A double move
  925 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  926 
  927   // The calling conventions assures us that each VMregpair is either
  928   // all really one physical register or adjacent stack slots.
  929 
  930   if (src.is_single_phys_reg() ) {
  931     if (dst.is_single_phys_reg()) {
  932       // In theory these overlap but the ordering is such that this is likely a nop
  933       if ( src.first() != dst.first()) {
  934         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  935       }
  936     } else {
  937       assert(dst.is_single_reg(), "not a stack pair");
  938       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  939     }
  940   } else if (dst.is_single_phys_reg()) {
  941     assert(src.is_single_reg(),  "not a stack pair");
  942     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  943   } else {
  944     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  945     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  946     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  947   }
  948 }
  949 
  950 
  951 // A float arg may have to do float reg int reg conversion
  952 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  953   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  954 
  955   // The calling conventions assures us that each VMregpair is either
  956   // all really one physical register or adjacent stack slots.
  957 
  958   if (src.first()->is_stack()) {
  959     if (dst.first()->is_stack()) {
  960       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  961       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  962     } else {
  963       // stack to reg
  964       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  965       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  966     }
  967   } else if (dst.first()->is_stack()) {
  968     // reg to stack
  969     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  970     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  971   } else {
  972     // reg to reg
  973     // In theory these overlap but the ordering is such that this is likely a nop
  974     if ( src.first() != dst.first()) {
  975       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  976     }
  977   }
  978 }
  979 
  980 // On 64 bit we will store integer like items to the stack as
  981 // 64 bits items (x86_32/64 abi) even though java would only store
  982 // 32bits for a parameter. On 32bit it will simply be 32 bits
  983 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  984 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  985   if (src.first()->is_stack()) {
  986     if (dst.first()->is_stack()) {
  987       // stack to stack
  988       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  989       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  990     } else {
  991       // stack to reg
  992       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  993     }
  994   } else if (dst.first()->is_stack()) {
  995     // reg to stack
  996     // Do we really have to sign extend???
  997     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
  998     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  999   } else {
 1000     // Do we really have to sign extend???
 1001     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1002     if (dst.first() != src.first()) {
 1003       movq(dst.first()->as_Register(), src.first()->as_Register());
 1004     }
 1005   }
 1006 }
 1007 
 1008 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1009   if (src.first()->is_stack()) {
 1010     if (dst.first()->is_stack()) {
 1011       // stack to stack
 1012       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1013       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1014     } else {
 1015       // stack to reg
 1016       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1017     }
 1018   } else if (dst.first()->is_stack()) {
 1019     // reg to stack
 1020     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1021   } else {
 1022     if (dst.first() != src.first()) {
 1023       movq(dst.first()->as_Register(), src.first()->as_Register());
 1024     }
 1025   }
 1026 }
 1027 
 1028 // An oop arg. Must pass a handle not the oop itself
 1029 void MacroAssembler::object_move(OopMap* map,
 1030                         int oop_handle_offset,
 1031                         int framesize_in_slots,
 1032                         VMRegPair src,
 1033                         VMRegPair dst,
 1034                         bool is_receiver,
 1035                         int* receiver_offset) {
 1036 
 1037   // must pass a handle. First figure out the location we use as a handle
 1038 
 1039   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1040 
 1041   // See if oop is null if it is we need no handle
 1042 
 1043   if (src.first()->is_stack()) {
 1044 
 1045     // Oop is already on the stack as an argument
 1046     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1047     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1048     if (is_receiver) {
 1049       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1050     }
 1051 
 1052     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1053     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1054     // conditionally move a null
 1055     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1056   } else {
 1057 
 1058     // Oop is in a register we must store it to the space we reserve
 1059     // on the stack for oop_handles and pass a handle if oop is non-null
 1060 
 1061     const Register rOop = src.first()->as_Register();
 1062     int oop_slot;
 1063     if (rOop == j_rarg0)
 1064       oop_slot = 0;
 1065     else if (rOop == j_rarg1)
 1066       oop_slot = 1;
 1067     else if (rOop == j_rarg2)
 1068       oop_slot = 2;
 1069     else if (rOop == j_rarg3)
 1070       oop_slot = 3;
 1071     else if (rOop == j_rarg4)
 1072       oop_slot = 4;
 1073     else {
 1074       assert(rOop == j_rarg5, "wrong register");
 1075       oop_slot = 5;
 1076     }
 1077 
 1078     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1079     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1080 
 1081     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1082     // Store oop in handle area, may be null
 1083     movptr(Address(rsp, offset), rOop);
 1084     if (is_receiver) {
 1085       *receiver_offset = offset;
 1086     }
 1087 
 1088     cmpptr(rOop, NULL_WORD);
 1089     lea(rHandle, Address(rsp, offset));
 1090     // conditionally move a null from the handle area where it was just stored
 1091     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1092   }
 1093 
 1094   // If arg is on the stack then place it otherwise it is already in correct reg.
 1095   if (dst.first()->is_stack()) {
 1096     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1097   }
 1098 }
 1099 
 1100 #endif // _LP64
 1101 
 1102 // Now versions that are common to 32/64 bit
 1103 
 1104 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1105   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1106 }
 1107 
 1108 void MacroAssembler::addptr(Register dst, Register src) {
 1109   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1110 }
 1111 
 1112 void MacroAssembler::addptr(Address dst, Register src) {
 1113   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1114 }
 1115 
 1116 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1117   assert(rscratch != noreg || always_reachable(src), "missing");
 1118 
 1119   if (reachable(src)) {
 1120     Assembler::addsd(dst, as_Address(src));
 1121   } else {
 1122     lea(rscratch, src);
 1123     Assembler::addsd(dst, Address(rscratch, 0));
 1124   }
 1125 }
 1126 
 1127 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1128   assert(rscratch != noreg || always_reachable(src), "missing");
 1129 
 1130   if (reachable(src)) {
 1131     addss(dst, as_Address(src));
 1132   } else {
 1133     lea(rscratch, src);
 1134     addss(dst, Address(rscratch, 0));
 1135   }
 1136 }
 1137 
 1138 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1139   assert(rscratch != noreg || always_reachable(src), "missing");
 1140 
 1141   if (reachable(src)) {
 1142     Assembler::addpd(dst, as_Address(src));
 1143   } else {
 1144     lea(rscratch, src);
 1145     Assembler::addpd(dst, Address(rscratch, 0));
 1146   }
 1147 }
 1148 
 1149 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1150 // Stub code is generated once and never copied.
 1151 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1152 void MacroAssembler::align64() {
 1153   align(64, (uint)(uintptr_t)pc());
 1154 }
 1155 
 1156 void MacroAssembler::align32() {
 1157   align(32, (uint)(uintptr_t)pc());
 1158 }
 1159 
 1160 void MacroAssembler::align(uint modulus) {
 1161   // 8273459: Ensure alignment is possible with current segment alignment
 1162   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1163   align(modulus, offset());
 1164 }
 1165 
 1166 void MacroAssembler::align(uint modulus, uint target) {
 1167   if (target % modulus != 0) {
 1168     nop(modulus - (target % modulus));
 1169   }
 1170 }
 1171 
 1172 void MacroAssembler::push_f(XMMRegister r) {
 1173   subptr(rsp, wordSize);
 1174   movflt(Address(rsp, 0), r);
 1175 }
 1176 
 1177 void MacroAssembler::pop_f(XMMRegister r) {
 1178   movflt(r, Address(rsp, 0));
 1179   addptr(rsp, wordSize);
 1180 }
 1181 
 1182 void MacroAssembler::push_d(XMMRegister r) {
 1183   subptr(rsp, 2 * wordSize);
 1184   movdbl(Address(rsp, 0), r);
 1185 }
 1186 
 1187 void MacroAssembler::pop_d(XMMRegister r) {
 1188   movdbl(r, Address(rsp, 0));
 1189   addptr(rsp, 2 * Interpreter::stackElementSize);
 1190 }
 1191 
 1192 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1193   // Used in sign-masking with aligned address.
 1194   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1195   assert(rscratch != noreg || always_reachable(src), "missing");
 1196 
 1197   if (reachable(src)) {
 1198     Assembler::andpd(dst, as_Address(src));
 1199   } else {
 1200     lea(rscratch, src);
 1201     Assembler::andpd(dst, Address(rscratch, 0));
 1202   }
 1203 }
 1204 
 1205 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1206   // Used in sign-masking with aligned address.
 1207   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1208   assert(rscratch != noreg || always_reachable(src), "missing");
 1209 
 1210   if (reachable(src)) {
 1211     Assembler::andps(dst, as_Address(src));
 1212   } else {
 1213     lea(rscratch, src);
 1214     Assembler::andps(dst, Address(rscratch, 0));
 1215   }
 1216 }
 1217 
 1218 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1219   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1220 }
 1221 
 1222 #ifdef _LP64
 1223 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1224   assert(rscratch != noreg || always_reachable(src), "missing");
 1225 
 1226   if (reachable(src)) {
 1227     andq(dst, as_Address(src));
 1228   } else {
 1229     lea(rscratch, src);
 1230     andq(dst, Address(rscratch, 0));
 1231   }
 1232 }
 1233 #endif
 1234 
 1235 void MacroAssembler::atomic_incl(Address counter_addr) {
 1236   lock();
 1237   incrementl(counter_addr);
 1238 }
 1239 
 1240 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1241   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1242 
 1243   if (reachable(counter_addr)) {
 1244     atomic_incl(as_Address(counter_addr));
 1245   } else {
 1246     lea(rscratch, counter_addr);
 1247     atomic_incl(Address(rscratch, 0));
 1248   }
 1249 }
 1250 
 1251 #ifdef _LP64
 1252 void MacroAssembler::atomic_incq(Address counter_addr) {
 1253   lock();
 1254   incrementq(counter_addr);
 1255 }
 1256 
 1257 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1258   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1259 
 1260   if (reachable(counter_addr)) {
 1261     atomic_incq(as_Address(counter_addr));
 1262   } else {
 1263     lea(rscratch, counter_addr);
 1264     atomic_incq(Address(rscratch, 0));
 1265   }
 1266 }
 1267 #endif
 1268 
 1269 // Writes to stack successive pages until offset reached to check for
 1270 // stack overflow + shadow pages.  This clobbers tmp.
 1271 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1272   movptr(tmp, rsp);
 1273   // Bang stack for total size given plus shadow page size.
 1274   // Bang one page at a time because large size can bang beyond yellow and
 1275   // red zones.
 1276   Label loop;
 1277   bind(loop);
 1278   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1279   subptr(tmp, (int)os::vm_page_size());
 1280   subl(size, (int)os::vm_page_size());
 1281   jcc(Assembler::greater, loop);
 1282 
 1283   // Bang down shadow pages too.
 1284   // At this point, (tmp-0) is the last address touched, so don't
 1285   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1286   // was post-decremented.)  Skip this address by starting at i=1, and
 1287   // touch a few more pages below.  N.B.  It is important to touch all
 1288   // the way down including all pages in the shadow zone.
 1289   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1290     // this could be any sized move but this is can be a debugging crumb
 1291     // so the bigger the better.
 1292     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1293   }
 1294 }
 1295 
 1296 void MacroAssembler::reserved_stack_check() {
 1297   // testing if reserved zone needs to be enabled
 1298   Label no_reserved_zone_enabling;
 1299   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1300   NOT_LP64(get_thread(rsi);)
 1301 
 1302   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1303   jcc(Assembler::below, no_reserved_zone_enabling);
 1304 
 1305   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1306   jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
 1307   should_not_reach_here();
 1308 
 1309   bind(no_reserved_zone_enabling);
 1310 }
 1311 
 1312 void MacroAssembler::c2bool(Register x) {
 1313   // implements x == 0 ? 0 : 1
 1314   // note: must only look at least-significant byte of x
 1315   //       since C-style booleans are stored in one byte
 1316   //       only! (was bug)
 1317   andl(x, 0xFF);
 1318   setb(Assembler::notZero, x);
 1319 }
 1320 
 1321 // Wouldn't need if AddressLiteral version had new name
 1322 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1323   Assembler::call(L, rtype);
 1324 }
 1325 
 1326 void MacroAssembler::call(Register entry) {
 1327   Assembler::call(entry);
 1328 }
 1329 
 1330 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1331   assert(rscratch != noreg || always_reachable(entry), "missing");
 1332 
 1333   if (reachable(entry)) {
 1334     Assembler::call_literal(entry.target(), entry.rspec());
 1335   } else {
 1336     lea(rscratch, entry);
 1337     Assembler::call(rscratch);
 1338   }
 1339 }
 1340 
 1341 void MacroAssembler::ic_call(address entry, jint method_index) {
 1342   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1343 #ifdef _LP64
 1344   // Needs full 64-bit immediate for later patching.
 1345   mov64(rax, (int64_t)Universe::non_oop_word());
 1346 #else
 1347   movptr(rax, (intptr_t)Universe::non_oop_word());
 1348 #endif
 1349   call(AddressLiteral(entry, rh));
 1350 }
 1351 
 1352 int MacroAssembler::ic_check_size() {
 1353   return LP64_ONLY(14) NOT_LP64(12);
 1354 }
 1355 
 1356 int MacroAssembler::ic_check(int end_alignment) {
 1357   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1358   Register data = rax;
 1359   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1360 
 1361   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1362   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1363   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1364   // before the inline cache check here, and not after
 1365   align(end_alignment, offset() + ic_check_size());
 1366 
 1367   int uep_offset = offset();
 1368 
 1369   if (UseCompressedClassPointers) {
 1370     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1371     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1372   } else {
 1373     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1374     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1375   }
 1376 
 1377   // if inline cache check fails, then jump to runtime routine
 1378   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1379   assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
 1380 
 1381   return uep_offset;
 1382 }
 1383 
 1384 void MacroAssembler::emit_static_call_stub() {
 1385   // Static stub relocation also tags the Method* in the code-stream.
 1386   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1387   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1388   jump(RuntimeAddress(pc()));
 1389 }
 1390 
 1391 // Implementation of call_VM versions
 1392 
 1393 void MacroAssembler::call_VM(Register oop_result,
 1394                              address entry_point,
 1395                              bool check_exceptions) {
 1396   Label C, E;
 1397   call(C, relocInfo::none);
 1398   jmp(E);
 1399 
 1400   bind(C);
 1401   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1402   ret(0);
 1403 
 1404   bind(E);
 1405 }
 1406 
 1407 void MacroAssembler::call_VM(Register oop_result,
 1408                              address entry_point,
 1409                              Register arg_1,
 1410                              bool check_exceptions) {
 1411   Label C, E;
 1412   call(C, relocInfo::none);
 1413   jmp(E);
 1414 
 1415   bind(C);
 1416   pass_arg1(this, arg_1);
 1417   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1418   ret(0);
 1419 
 1420   bind(E);
 1421 }
 1422 
 1423 void MacroAssembler::call_VM(Register oop_result,
 1424                              address entry_point,
 1425                              Register arg_1,
 1426                              Register arg_2,
 1427                              bool check_exceptions) {
 1428   Label C, E;
 1429   call(C, relocInfo::none);
 1430   jmp(E);
 1431 
 1432   bind(C);
 1433 
 1434   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1435 
 1436   pass_arg2(this, arg_2);
 1437   pass_arg1(this, arg_1);
 1438   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1439   ret(0);
 1440 
 1441   bind(E);
 1442 }
 1443 
 1444 void MacroAssembler::call_VM(Register oop_result,
 1445                              address entry_point,
 1446                              Register arg_1,
 1447                              Register arg_2,
 1448                              Register arg_3,
 1449                              bool check_exceptions) {
 1450   Label C, E;
 1451   call(C, relocInfo::none);
 1452   jmp(E);
 1453 
 1454   bind(C);
 1455 
 1456   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1457   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1458   pass_arg3(this, arg_3);
 1459   pass_arg2(this, arg_2);
 1460   pass_arg1(this, arg_1);
 1461   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1462   ret(0);
 1463 
 1464   bind(E);
 1465 }
 1466 
 1467 void MacroAssembler::call_VM(Register oop_result,
 1468                              Register last_java_sp,
 1469                              address entry_point,
 1470                              int number_of_arguments,
 1471                              bool check_exceptions) {
 1472   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1473   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1474 }
 1475 
 1476 void MacroAssembler::call_VM(Register oop_result,
 1477                              Register last_java_sp,
 1478                              address entry_point,
 1479                              Register arg_1,
 1480                              bool check_exceptions) {
 1481   pass_arg1(this, arg_1);
 1482   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1483 }
 1484 
 1485 void MacroAssembler::call_VM(Register oop_result,
 1486                              Register last_java_sp,
 1487                              address entry_point,
 1488                              Register arg_1,
 1489                              Register arg_2,
 1490                              bool check_exceptions) {
 1491 
 1492   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1493   pass_arg2(this, arg_2);
 1494   pass_arg1(this, arg_1);
 1495   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1496 }
 1497 
 1498 void MacroAssembler::call_VM(Register oop_result,
 1499                              Register last_java_sp,
 1500                              address entry_point,
 1501                              Register arg_1,
 1502                              Register arg_2,
 1503                              Register arg_3,
 1504                              bool check_exceptions) {
 1505   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1506   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1507   pass_arg3(this, arg_3);
 1508   pass_arg2(this, arg_2);
 1509   pass_arg1(this, arg_1);
 1510   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1511 }
 1512 
 1513 void MacroAssembler::super_call_VM(Register oop_result,
 1514                                    Register last_java_sp,
 1515                                    address entry_point,
 1516                                    int number_of_arguments,
 1517                                    bool check_exceptions) {
 1518   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1519   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1520 }
 1521 
 1522 void MacroAssembler::super_call_VM(Register oop_result,
 1523                                    Register last_java_sp,
 1524                                    address entry_point,
 1525                                    Register arg_1,
 1526                                    bool check_exceptions) {
 1527   pass_arg1(this, arg_1);
 1528   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1529 }
 1530 
 1531 void MacroAssembler::super_call_VM(Register oop_result,
 1532                                    Register last_java_sp,
 1533                                    address entry_point,
 1534                                    Register arg_1,
 1535                                    Register arg_2,
 1536                                    bool check_exceptions) {
 1537 
 1538   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1539   pass_arg2(this, arg_2);
 1540   pass_arg1(this, arg_1);
 1541   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1542 }
 1543 
 1544 void MacroAssembler::super_call_VM(Register oop_result,
 1545                                    Register last_java_sp,
 1546                                    address entry_point,
 1547                                    Register arg_1,
 1548                                    Register arg_2,
 1549                                    Register arg_3,
 1550                                    bool check_exceptions) {
 1551   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1552   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1553   pass_arg3(this, arg_3);
 1554   pass_arg2(this, arg_2);
 1555   pass_arg1(this, arg_1);
 1556   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1557 }
 1558 
 1559 void MacroAssembler::call_VM_base(Register oop_result,
 1560                                   Register java_thread,
 1561                                   Register last_java_sp,
 1562                                   address  entry_point,
 1563                                   int      number_of_arguments,
 1564                                   bool     check_exceptions) {
 1565   // determine java_thread register
 1566   if (!java_thread->is_valid()) {
 1567 #ifdef _LP64
 1568     java_thread = r15_thread;
 1569 #else
 1570     java_thread = rdi;
 1571     get_thread(java_thread);
 1572 #endif // LP64
 1573   }
 1574   // determine last_java_sp register
 1575   if (!last_java_sp->is_valid()) {
 1576     last_java_sp = rsp;
 1577   }
 1578   // debugging support
 1579   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1580   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1581 #ifdef ASSERT
 1582   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1583   // r12 is the heapbase.
 1584   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1585 #endif // ASSERT
 1586 
 1587   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1588   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1589 
 1590   // push java thread (becomes first argument of C function)
 1591 
 1592   NOT_LP64(push(java_thread); number_of_arguments++);
 1593   LP64_ONLY(mov(c_rarg0, r15_thread));
 1594 
 1595   // set last Java frame before call
 1596   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1597 
 1598   // Only interpreter should have to set fp
 1599   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1600 
 1601   // do the call, remove parameters
 1602   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1603 
 1604   // restore the thread (cannot use the pushed argument since arguments
 1605   // may be overwritten by C code generated by an optimizing compiler);
 1606   // however can use the register value directly if it is callee saved.
 1607   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1608     // rdi & rsi (also r15) are callee saved -> nothing to do
 1609 #ifdef ASSERT
 1610     guarantee(java_thread != rax, "change this code");
 1611     push(rax);
 1612     { Label L;
 1613       get_thread(rax);
 1614       cmpptr(java_thread, rax);
 1615       jcc(Assembler::equal, L);
 1616       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1617       bind(L);
 1618     }
 1619     pop(rax);
 1620 #endif
 1621   } else {
 1622     get_thread(java_thread);
 1623   }
 1624   // reset last Java frame
 1625   // Only interpreter should have to clear fp
 1626   reset_last_Java_frame(java_thread, true);
 1627 
 1628    // C++ interp handles this in the interpreter
 1629   check_and_handle_popframe(java_thread);
 1630   check_and_handle_earlyret(java_thread);
 1631 
 1632   if (check_exceptions) {
 1633     // check for pending exceptions (java_thread is set upon return)
 1634     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1635 #ifndef _LP64
 1636     jump_cc(Assembler::notEqual,
 1637             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1638 #else
 1639     // This used to conditionally jump to forward_exception however it is
 1640     // possible if we relocate that the branch will not reach. So we must jump
 1641     // around so we can always reach
 1642 
 1643     Label ok;
 1644     jcc(Assembler::equal, ok);
 1645     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1646     bind(ok);
 1647 #endif // LP64
 1648   }
 1649 
 1650   // get oop result if there is one and reset the value in the thread
 1651   if (oop_result->is_valid()) {
 1652     get_vm_result(oop_result, java_thread);
 1653   }
 1654 }
 1655 
 1656 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1657 
 1658   // Calculate the value for last_Java_sp
 1659   // somewhat subtle. call_VM does an intermediate call
 1660   // which places a return address on the stack just under the
 1661   // stack pointer as the user finished with it. This allows
 1662   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1663   // On 32bit we then have to push additional args on the stack to accomplish
 1664   // the actual requested call. On 64bit call_VM only can use register args
 1665   // so the only extra space is the return address that call_VM created.
 1666   // This hopefully explains the calculations here.
 1667 
 1668 #ifdef _LP64
 1669   // We've pushed one address, correct last_Java_sp
 1670   lea(rax, Address(rsp, wordSize));
 1671 #else
 1672   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1673 #endif // LP64
 1674 
 1675   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1676 
 1677 }
 1678 
 1679 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1680 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1681   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1682 }
 1683 
 1684 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1685   call_VM_leaf_base(entry_point, number_of_arguments);
 1686 }
 1687 
 1688 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1689   pass_arg0(this, arg_0);
 1690   call_VM_leaf(entry_point, 1);
 1691 }
 1692 
 1693 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1694 
 1695   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1696   pass_arg1(this, arg_1);
 1697   pass_arg0(this, arg_0);
 1698   call_VM_leaf(entry_point, 2);
 1699 }
 1700 
 1701 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1702   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1703   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1704   pass_arg2(this, arg_2);
 1705   pass_arg1(this, arg_1);
 1706   pass_arg0(this, arg_0);
 1707   call_VM_leaf(entry_point, 3);
 1708 }
 1709 
 1710 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1711   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1712   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1713   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1714   pass_arg3(this, arg_3);
 1715   pass_arg2(this, arg_2);
 1716   pass_arg1(this, arg_1);
 1717   pass_arg0(this, arg_0);
 1718   call_VM_leaf(entry_point, 3);
 1719 }
 1720 
 1721 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1722   pass_arg0(this, arg_0);
 1723   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1724 }
 1725 
 1726 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1727   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1728   pass_arg1(this, arg_1);
 1729   pass_arg0(this, arg_0);
 1730   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1731 }
 1732 
 1733 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1734   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1735   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1736   pass_arg2(this, arg_2);
 1737   pass_arg1(this, arg_1);
 1738   pass_arg0(this, arg_0);
 1739   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1740 }
 1741 
 1742 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1743   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1744   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1745   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1746   pass_arg3(this, arg_3);
 1747   pass_arg2(this, arg_2);
 1748   pass_arg1(this, arg_1);
 1749   pass_arg0(this, arg_0);
 1750   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1751 }
 1752 
 1753 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1754   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1755   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1756   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1757 }
 1758 
 1759 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1760   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1761   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1762 }
 1763 
 1764 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1765 }
 1766 
 1767 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1768 }
 1769 
 1770 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1771   assert(rscratch != noreg || always_reachable(src1), "missing");
 1772 
 1773   if (reachable(src1)) {
 1774     cmpl(as_Address(src1), imm);
 1775   } else {
 1776     lea(rscratch, src1);
 1777     cmpl(Address(rscratch, 0), imm);
 1778   }
 1779 }
 1780 
 1781 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1782   assert(!src2.is_lval(), "use cmpptr");
 1783   assert(rscratch != noreg || always_reachable(src2), "missing");
 1784 
 1785   if (reachable(src2)) {
 1786     cmpl(src1, as_Address(src2));
 1787   } else {
 1788     lea(rscratch, src2);
 1789     cmpl(src1, Address(rscratch, 0));
 1790   }
 1791 }
 1792 
 1793 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1794   Assembler::cmpl(src1, imm);
 1795 }
 1796 
 1797 void MacroAssembler::cmp32(Register src1, Address src2) {
 1798   Assembler::cmpl(src1, src2);
 1799 }
 1800 
 1801 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1802   ucomisd(opr1, opr2);
 1803 
 1804   Label L;
 1805   if (unordered_is_less) {
 1806     movl(dst, -1);
 1807     jcc(Assembler::parity, L);
 1808     jcc(Assembler::below , L);
 1809     movl(dst, 0);
 1810     jcc(Assembler::equal , L);
 1811     increment(dst);
 1812   } else { // unordered is greater
 1813     movl(dst, 1);
 1814     jcc(Assembler::parity, L);
 1815     jcc(Assembler::above , L);
 1816     movl(dst, 0);
 1817     jcc(Assembler::equal , L);
 1818     decrementl(dst);
 1819   }
 1820   bind(L);
 1821 }
 1822 
 1823 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1824   ucomiss(opr1, opr2);
 1825 
 1826   Label L;
 1827   if (unordered_is_less) {
 1828     movl(dst, -1);
 1829     jcc(Assembler::parity, L);
 1830     jcc(Assembler::below , L);
 1831     movl(dst, 0);
 1832     jcc(Assembler::equal , L);
 1833     increment(dst);
 1834   } else { // unordered is greater
 1835     movl(dst, 1);
 1836     jcc(Assembler::parity, L);
 1837     jcc(Assembler::above , L);
 1838     movl(dst, 0);
 1839     jcc(Assembler::equal , L);
 1840     decrementl(dst);
 1841   }
 1842   bind(L);
 1843 }
 1844 
 1845 
 1846 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1847   assert(rscratch != noreg || always_reachable(src1), "missing");
 1848 
 1849   if (reachable(src1)) {
 1850     cmpb(as_Address(src1), imm);
 1851   } else {
 1852     lea(rscratch, src1);
 1853     cmpb(Address(rscratch, 0), imm);
 1854   }
 1855 }
 1856 
 1857 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1858 #ifdef _LP64
 1859   assert(rscratch != noreg || always_reachable(src2), "missing");
 1860 
 1861   if (src2.is_lval()) {
 1862     movptr(rscratch, src2);
 1863     Assembler::cmpq(src1, rscratch);
 1864   } else if (reachable(src2)) {
 1865     cmpq(src1, as_Address(src2));
 1866   } else {
 1867     lea(rscratch, src2);
 1868     Assembler::cmpq(src1, Address(rscratch, 0));
 1869   }
 1870 #else
 1871   assert(rscratch == noreg, "not needed");
 1872   if (src2.is_lval()) {
 1873     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1874   } else {
 1875     cmpl(src1, as_Address(src2));
 1876   }
 1877 #endif // _LP64
 1878 }
 1879 
 1880 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1881   assert(src2.is_lval(), "not a mem-mem compare");
 1882 #ifdef _LP64
 1883   // moves src2's literal address
 1884   movptr(rscratch, src2);
 1885   Assembler::cmpq(src1, rscratch);
 1886 #else
 1887   assert(rscratch == noreg, "not needed");
 1888   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1889 #endif // _LP64
 1890 }
 1891 
 1892 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1893   cmpptr(src1, src2);
 1894 }
 1895 
 1896 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1897   cmpptr(src1, src2);
 1898 }
 1899 
 1900 #ifdef _LP64
 1901 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1902   movoop(rscratch, src2);
 1903   cmpptr(src1, rscratch);
 1904 }
 1905 #endif
 1906 
 1907 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1908   assert(rscratch != noreg || always_reachable(adr), "missing");
 1909 
 1910   if (reachable(adr)) {
 1911     lock();
 1912     cmpxchgptr(reg, as_Address(adr));
 1913   } else {
 1914     lea(rscratch, adr);
 1915     lock();
 1916     cmpxchgptr(reg, Address(rscratch, 0));
 1917   }
 1918 }
 1919 
 1920 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1921   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1922 }
 1923 
 1924 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1925   assert(rscratch != noreg || always_reachable(src), "missing");
 1926 
 1927   if (reachable(src)) {
 1928     Assembler::comisd(dst, as_Address(src));
 1929   } else {
 1930     lea(rscratch, src);
 1931     Assembler::comisd(dst, Address(rscratch, 0));
 1932   }
 1933 }
 1934 
 1935 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1936   assert(rscratch != noreg || always_reachable(src), "missing");
 1937 
 1938   if (reachable(src)) {
 1939     Assembler::comiss(dst, as_Address(src));
 1940   } else {
 1941     lea(rscratch, src);
 1942     Assembler::comiss(dst, Address(rscratch, 0));
 1943   }
 1944 }
 1945 
 1946 
 1947 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1948   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1949 
 1950   Condition negated_cond = negate_condition(cond);
 1951   Label L;
 1952   jcc(negated_cond, L);
 1953   pushf(); // Preserve flags
 1954   atomic_incl(counter_addr, rscratch);
 1955   popf();
 1956   bind(L);
 1957 }
 1958 
 1959 int MacroAssembler::corrected_idivl(Register reg) {
 1960   // Full implementation of Java idiv and irem; checks for
 1961   // special case as described in JVM spec., p.243 & p.271.
 1962   // The function returns the (pc) offset of the idivl
 1963   // instruction - may be needed for implicit exceptions.
 1964   //
 1965   //         normal case                           special case
 1966   //
 1967   // input : rax,: dividend                         min_int
 1968   //         reg: divisor   (may not be rax,/rdx)   -1
 1969   //
 1970   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1971   //         rdx: remainder (= rax, irem reg)       0
 1972   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1973   const int min_int = 0x80000000;
 1974   Label normal_case, special_case;
 1975 
 1976   // check for special case
 1977   cmpl(rax, min_int);
 1978   jcc(Assembler::notEqual, normal_case);
 1979   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1980   cmpl(reg, -1);
 1981   jcc(Assembler::equal, special_case);
 1982 
 1983   // handle normal case
 1984   bind(normal_case);
 1985   cdql();
 1986   int idivl_offset = offset();
 1987   idivl(reg);
 1988 
 1989   // normal and special case exit
 1990   bind(special_case);
 1991 
 1992   return idivl_offset;
 1993 }
 1994 
 1995 
 1996 
 1997 void MacroAssembler::decrementl(Register reg, int value) {
 1998   if (value == min_jint) {subl(reg, value) ; return; }
 1999   if (value <  0) { incrementl(reg, -value); return; }
 2000   if (value == 0) {                        ; return; }
 2001   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2002   /* else */      { subl(reg, value)       ; return; }
 2003 }
 2004 
 2005 void MacroAssembler::decrementl(Address dst, int value) {
 2006   if (value == min_jint) {subl(dst, value) ; return; }
 2007   if (value <  0) { incrementl(dst, -value); return; }
 2008   if (value == 0) {                        ; return; }
 2009   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2010   /* else */      { subl(dst, value)       ; return; }
 2011 }
 2012 
 2013 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2014   assert(shift_value > 0, "illegal shift value");
 2015   Label _is_positive;
 2016   testl (reg, reg);
 2017   jcc (Assembler::positive, _is_positive);
 2018   int offset = (1 << shift_value) - 1 ;
 2019 
 2020   if (offset == 1) {
 2021     incrementl(reg);
 2022   } else {
 2023     addl(reg, offset);
 2024   }
 2025 
 2026   bind (_is_positive);
 2027   sarl(reg, shift_value);
 2028 }
 2029 
 2030 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2031   assert(rscratch != noreg || always_reachable(src), "missing");
 2032 
 2033   if (reachable(src)) {
 2034     Assembler::divsd(dst, as_Address(src));
 2035   } else {
 2036     lea(rscratch, src);
 2037     Assembler::divsd(dst, Address(rscratch, 0));
 2038   }
 2039 }
 2040 
 2041 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2042   assert(rscratch != noreg || always_reachable(src), "missing");
 2043 
 2044   if (reachable(src)) {
 2045     Assembler::divss(dst, as_Address(src));
 2046   } else {
 2047     lea(rscratch, src);
 2048     Assembler::divss(dst, Address(rscratch, 0));
 2049   }
 2050 }
 2051 
 2052 void MacroAssembler::enter() {
 2053   push(rbp);
 2054   mov(rbp, rsp);
 2055 }
 2056 
 2057 void MacroAssembler::post_call_nop() {
 2058   if (!Continuations::enabled()) {
 2059     return;
 2060   }
 2061   InstructionMark im(this);
 2062   relocate(post_call_nop_Relocation::spec());
 2063   InlineSkippedInstructionsCounter skipCounter(this);
 2064   emit_int8((uint8_t)0x0f);
 2065   emit_int8((uint8_t)0x1f);
 2066   emit_int8((uint8_t)0x84);
 2067   emit_int8((uint8_t)0x00);
 2068   emit_int32(0x00);
 2069 }
 2070 
 2071 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2072 void MacroAssembler::fat_nop() {
 2073   if (UseAddressNop) {
 2074     addr_nop_5();
 2075   } else {
 2076     emit_int8((uint8_t)0x26); // es:
 2077     emit_int8((uint8_t)0x2e); // cs:
 2078     emit_int8((uint8_t)0x64); // fs:
 2079     emit_int8((uint8_t)0x65); // gs:
 2080     emit_int8((uint8_t)0x90);
 2081   }
 2082 }
 2083 
 2084 #ifndef _LP64
 2085 void MacroAssembler::fcmp(Register tmp) {
 2086   fcmp(tmp, 1, true, true);
 2087 }
 2088 
 2089 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2090   assert(!pop_right || pop_left, "usage error");
 2091   if (VM_Version::supports_cmov()) {
 2092     assert(tmp == noreg, "unneeded temp");
 2093     if (pop_left) {
 2094       fucomip(index);
 2095     } else {
 2096       fucomi(index);
 2097     }
 2098     if (pop_right) {
 2099       fpop();
 2100     }
 2101   } else {
 2102     assert(tmp != noreg, "need temp");
 2103     if (pop_left) {
 2104       if (pop_right) {
 2105         fcompp();
 2106       } else {
 2107         fcomp(index);
 2108       }
 2109     } else {
 2110       fcom(index);
 2111     }
 2112     // convert FPU condition into eflags condition via rax,
 2113     save_rax(tmp);
 2114     fwait(); fnstsw_ax();
 2115     sahf();
 2116     restore_rax(tmp);
 2117   }
 2118   // condition codes set as follows:
 2119   //
 2120   // CF (corresponds to C0) if x < y
 2121   // PF (corresponds to C2) if unordered
 2122   // ZF (corresponds to C3) if x = y
 2123 }
 2124 
 2125 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2126   fcmp2int(dst, unordered_is_less, 1, true, true);
 2127 }
 2128 
 2129 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2130   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2131   Label L;
 2132   if (unordered_is_less) {
 2133     movl(dst, -1);
 2134     jcc(Assembler::parity, L);
 2135     jcc(Assembler::below , L);
 2136     movl(dst, 0);
 2137     jcc(Assembler::equal , L);
 2138     increment(dst);
 2139   } else { // unordered is greater
 2140     movl(dst, 1);
 2141     jcc(Assembler::parity, L);
 2142     jcc(Assembler::above , L);
 2143     movl(dst, 0);
 2144     jcc(Assembler::equal , L);
 2145     decrementl(dst);
 2146   }
 2147   bind(L);
 2148 }
 2149 
 2150 void MacroAssembler::fld_d(AddressLiteral src) {
 2151   fld_d(as_Address(src));
 2152 }
 2153 
 2154 void MacroAssembler::fld_s(AddressLiteral src) {
 2155   fld_s(as_Address(src));
 2156 }
 2157 
 2158 void MacroAssembler::fldcw(AddressLiteral src) {
 2159   fldcw(as_Address(src));
 2160 }
 2161 
 2162 void MacroAssembler::fpop() {
 2163   ffree();
 2164   fincstp();
 2165 }
 2166 
 2167 void MacroAssembler::fremr(Register tmp) {
 2168   save_rax(tmp);
 2169   { Label L;
 2170     bind(L);
 2171     fprem();
 2172     fwait(); fnstsw_ax();
 2173     sahf();
 2174     jcc(Assembler::parity, L);
 2175   }
 2176   restore_rax(tmp);
 2177   // Result is in ST0.
 2178   // Note: fxch & fpop to get rid of ST1
 2179   // (otherwise FPU stack could overflow eventually)
 2180   fxch(1);
 2181   fpop();
 2182 }
 2183 
 2184 void MacroAssembler::empty_FPU_stack() {
 2185   if (VM_Version::supports_mmx()) {
 2186     emms();
 2187   } else {
 2188     for (int i = 8; i-- > 0; ) ffree(i);
 2189   }
 2190 }
 2191 #endif // !LP64
 2192 
 2193 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2194   assert(rscratch != noreg || always_reachable(src), "missing");
 2195   if (reachable(src)) {
 2196     Assembler::mulpd(dst, as_Address(src));
 2197   } else {
 2198     lea(rscratch, src);
 2199     Assembler::mulpd(dst, Address(rscratch, 0));
 2200   }
 2201 }
 2202 
 2203 void MacroAssembler::load_float(Address src) {
 2204 #ifdef _LP64
 2205   movflt(xmm0, src);
 2206 #else
 2207   if (UseSSE >= 1) {
 2208     movflt(xmm0, src);
 2209   } else {
 2210     fld_s(src);
 2211   }
 2212 #endif // LP64
 2213 }
 2214 
 2215 void MacroAssembler::store_float(Address dst) {
 2216 #ifdef _LP64
 2217   movflt(dst, xmm0);
 2218 #else
 2219   if (UseSSE >= 1) {
 2220     movflt(dst, xmm0);
 2221   } else {
 2222     fstp_s(dst);
 2223   }
 2224 #endif // LP64
 2225 }
 2226 
 2227 void MacroAssembler::load_double(Address src) {
 2228 #ifdef _LP64
 2229   movdbl(xmm0, src);
 2230 #else
 2231   if (UseSSE >= 2) {
 2232     movdbl(xmm0, src);
 2233   } else {
 2234     fld_d(src);
 2235   }
 2236 #endif // LP64
 2237 }
 2238 
 2239 void MacroAssembler::store_double(Address dst) {
 2240 #ifdef _LP64
 2241   movdbl(dst, xmm0);
 2242 #else
 2243   if (UseSSE >= 2) {
 2244     movdbl(dst, xmm0);
 2245   } else {
 2246     fstp_d(dst);
 2247   }
 2248 #endif // LP64
 2249 }
 2250 
 2251 // dst = c = a * b + c
 2252 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2253   Assembler::vfmadd231sd(c, a, b);
 2254   if (dst != c) {
 2255     movdbl(dst, c);
 2256   }
 2257 }
 2258 
 2259 // dst = c = a * b + c
 2260 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2261   Assembler::vfmadd231ss(c, a, b);
 2262   if (dst != c) {
 2263     movflt(dst, c);
 2264   }
 2265 }
 2266 
 2267 // dst = c = a * b + c
 2268 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2269   Assembler::vfmadd231pd(c, a, b, vector_len);
 2270   if (dst != c) {
 2271     vmovdqu(dst, c);
 2272   }
 2273 }
 2274 
 2275 // dst = c = a * b + c
 2276 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2277   Assembler::vfmadd231ps(c, a, b, vector_len);
 2278   if (dst != c) {
 2279     vmovdqu(dst, c);
 2280   }
 2281 }
 2282 
 2283 // dst = c = a * b + c
 2284 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2285   Assembler::vfmadd231pd(c, a, b, vector_len);
 2286   if (dst != c) {
 2287     vmovdqu(dst, c);
 2288   }
 2289 }
 2290 
 2291 // dst = c = a * b + c
 2292 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2293   Assembler::vfmadd231ps(c, a, b, vector_len);
 2294   if (dst != c) {
 2295     vmovdqu(dst, c);
 2296   }
 2297 }
 2298 
 2299 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2300   assert(rscratch != noreg || always_reachable(dst), "missing");
 2301 
 2302   if (reachable(dst)) {
 2303     incrementl(as_Address(dst));
 2304   } else {
 2305     lea(rscratch, dst);
 2306     incrementl(Address(rscratch, 0));
 2307   }
 2308 }
 2309 
 2310 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2311   incrementl(as_Address(dst, rscratch));
 2312 }
 2313 
 2314 void MacroAssembler::incrementl(Register reg, int value) {
 2315   if (value == min_jint) {addl(reg, value) ; return; }
 2316   if (value <  0) { decrementl(reg, -value); return; }
 2317   if (value == 0) {                        ; return; }
 2318   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2319   /* else */      { addl(reg, value)       ; return; }
 2320 }
 2321 
 2322 void MacroAssembler::incrementl(Address dst, int value) {
 2323   if (value == min_jint) {addl(dst, value) ; return; }
 2324   if (value <  0) { decrementl(dst, -value); return; }
 2325   if (value == 0) {                        ; return; }
 2326   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2327   /* else */      { addl(dst, value)       ; return; }
 2328 }
 2329 
 2330 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2331   assert(rscratch != noreg || always_reachable(dst), "missing");
 2332 
 2333   if (reachable(dst)) {
 2334     jmp_literal(dst.target(), dst.rspec());
 2335   } else {
 2336     lea(rscratch, dst);
 2337     jmp(rscratch);
 2338   }
 2339 }
 2340 
 2341 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2342   assert(rscratch != noreg || always_reachable(dst), "missing");
 2343 
 2344   if (reachable(dst)) {
 2345     InstructionMark im(this);
 2346     relocate(dst.reloc());
 2347     const int short_size = 2;
 2348     const int long_size = 6;
 2349     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2350     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2351       // 0111 tttn #8-bit disp
 2352       emit_int8(0x70 | cc);
 2353       emit_int8((offs - short_size) & 0xFF);
 2354     } else {
 2355       // 0000 1111 1000 tttn #32-bit disp
 2356       emit_int8(0x0F);
 2357       emit_int8((unsigned char)(0x80 | cc));
 2358       emit_int32(offs - long_size);
 2359     }
 2360   } else {
 2361 #ifdef ASSERT
 2362     warning("reversing conditional branch");
 2363 #endif /* ASSERT */
 2364     Label skip;
 2365     jccb(reverse[cc], skip);
 2366     lea(rscratch, dst);
 2367     Assembler::jmp(rscratch);
 2368     bind(skip);
 2369   }
 2370 }
 2371 
 2372 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2373   assert(rscratch != noreg || always_reachable(src), "missing");
 2374 
 2375   if (reachable(src)) {
 2376     Assembler::ldmxcsr(as_Address(src));
 2377   } else {
 2378     lea(rscratch, src);
 2379     Assembler::ldmxcsr(Address(rscratch, 0));
 2380   }
 2381 }
 2382 
 2383 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2384   int off;
 2385   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2386     off = offset();
 2387     movsbl(dst, src); // movsxb
 2388   } else {
 2389     off = load_unsigned_byte(dst, src);
 2390     shll(dst, 24);
 2391     sarl(dst, 24);
 2392   }
 2393   return off;
 2394 }
 2395 
 2396 // Note: load_signed_short used to be called load_signed_word.
 2397 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2398 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2399 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2400 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2401   int off;
 2402   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2403     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2404     // version but this is what 64bit has always done. This seems to imply
 2405     // that users are only using 32bits worth.
 2406     off = offset();
 2407     movswl(dst, src); // movsxw
 2408   } else {
 2409     off = load_unsigned_short(dst, src);
 2410     shll(dst, 16);
 2411     sarl(dst, 16);
 2412   }
 2413   return off;
 2414 }
 2415 
 2416 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2417   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2418   // and "3.9 Partial Register Penalties", p. 22).
 2419   int off;
 2420   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2421     off = offset();
 2422     movzbl(dst, src); // movzxb
 2423   } else {
 2424     xorl(dst, dst);
 2425     off = offset();
 2426     movb(dst, src);
 2427   }
 2428   return off;
 2429 }
 2430 
 2431 // Note: load_unsigned_short used to be called load_unsigned_word.
 2432 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2433   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2434   // and "3.9 Partial Register Penalties", p. 22).
 2435   int off;
 2436   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2437     off = offset();
 2438     movzwl(dst, src); // movzxw
 2439   } else {
 2440     xorl(dst, dst);
 2441     off = offset();
 2442     movw(dst, src);
 2443   }
 2444   return off;
 2445 }
 2446 
 2447 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2448   switch (size_in_bytes) {
 2449 #ifndef _LP64
 2450   case  8:
 2451     assert(dst2 != noreg, "second dest register required");
 2452     movl(dst,  src);
 2453     movl(dst2, src.plus_disp(BytesPerInt));
 2454     break;
 2455 #else
 2456   case  8:  movq(dst, src); break;
 2457 #endif
 2458   case  4:  movl(dst, src); break;
 2459   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2460   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2461   default:  ShouldNotReachHere();
 2462   }
 2463 }
 2464 
 2465 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2466   switch (size_in_bytes) {
 2467 #ifndef _LP64
 2468   case  8:
 2469     assert(src2 != noreg, "second source register required");
 2470     movl(dst,                        src);
 2471     movl(dst.plus_disp(BytesPerInt), src2);
 2472     break;
 2473 #else
 2474   case  8:  movq(dst, src); break;
 2475 #endif
 2476   case  4:  movl(dst, src); break;
 2477   case  2:  movw(dst, src); break;
 2478   case  1:  movb(dst, src); break;
 2479   default:  ShouldNotReachHere();
 2480   }
 2481 }
 2482 
 2483 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2484   assert(rscratch != noreg || always_reachable(dst), "missing");
 2485 
 2486   if (reachable(dst)) {
 2487     movl(as_Address(dst), src);
 2488   } else {
 2489     lea(rscratch, dst);
 2490     movl(Address(rscratch, 0), src);
 2491   }
 2492 }
 2493 
 2494 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2495   if (reachable(src)) {
 2496     movl(dst, as_Address(src));
 2497   } else {
 2498     lea(dst, src);
 2499     movl(dst, Address(dst, 0));
 2500   }
 2501 }
 2502 
 2503 // C++ bool manipulation
 2504 
 2505 void MacroAssembler::movbool(Register dst, Address src) {
 2506   if(sizeof(bool) == 1)
 2507     movb(dst, src);
 2508   else if(sizeof(bool) == 2)
 2509     movw(dst, src);
 2510   else if(sizeof(bool) == 4)
 2511     movl(dst, src);
 2512   else
 2513     // unsupported
 2514     ShouldNotReachHere();
 2515 }
 2516 
 2517 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2518   if(sizeof(bool) == 1)
 2519     movb(dst, (int) boolconst);
 2520   else if(sizeof(bool) == 2)
 2521     movw(dst, (int) boolconst);
 2522   else if(sizeof(bool) == 4)
 2523     movl(dst, (int) boolconst);
 2524   else
 2525     // unsupported
 2526     ShouldNotReachHere();
 2527 }
 2528 
 2529 void MacroAssembler::movbool(Address dst, Register src) {
 2530   if(sizeof(bool) == 1)
 2531     movb(dst, src);
 2532   else if(sizeof(bool) == 2)
 2533     movw(dst, src);
 2534   else if(sizeof(bool) == 4)
 2535     movl(dst, src);
 2536   else
 2537     // unsupported
 2538     ShouldNotReachHere();
 2539 }
 2540 
 2541 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2542   assert(rscratch != noreg || always_reachable(src), "missing");
 2543 
 2544   if (reachable(src)) {
 2545     movdl(dst, as_Address(src));
 2546   } else {
 2547     lea(rscratch, src);
 2548     movdl(dst, Address(rscratch, 0));
 2549   }
 2550 }
 2551 
 2552 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2553   assert(rscratch != noreg || always_reachable(src), "missing");
 2554 
 2555   if (reachable(src)) {
 2556     movq(dst, as_Address(src));
 2557   } else {
 2558     lea(rscratch, src);
 2559     movq(dst, Address(rscratch, 0));
 2560   }
 2561 }
 2562 
 2563 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2564   assert(rscratch != noreg || always_reachable(src), "missing");
 2565 
 2566   if (reachable(src)) {
 2567     if (UseXmmLoadAndClearUpper) {
 2568       movsd (dst, as_Address(src));
 2569     } else {
 2570       movlpd(dst, as_Address(src));
 2571     }
 2572   } else {
 2573     lea(rscratch, src);
 2574     if (UseXmmLoadAndClearUpper) {
 2575       movsd (dst, Address(rscratch, 0));
 2576     } else {
 2577       movlpd(dst, Address(rscratch, 0));
 2578     }
 2579   }
 2580 }
 2581 
 2582 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2583   assert(rscratch != noreg || always_reachable(src), "missing");
 2584 
 2585   if (reachable(src)) {
 2586     movss(dst, as_Address(src));
 2587   } else {
 2588     lea(rscratch, src);
 2589     movss(dst, Address(rscratch, 0));
 2590   }
 2591 }
 2592 
 2593 void MacroAssembler::movptr(Register dst, Register src) {
 2594   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2595 }
 2596 
 2597 void MacroAssembler::movptr(Register dst, Address src) {
 2598   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2599 }
 2600 
 2601 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2602 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2603 #ifdef _LP64
 2604   if (is_uimm32(src)) {
 2605     movl(dst, checked_cast<uint32_t>(src));
 2606   } else if (is_simm32(src)) {
 2607     movq(dst, checked_cast<int32_t>(src));
 2608   } else {
 2609     mov64(dst, src);
 2610   }
 2611 #else
 2612   movl(dst, src);
 2613 #endif
 2614 }
 2615 
 2616 void MacroAssembler::movptr(Address dst, Register src) {
 2617   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2618 }
 2619 
 2620 void MacroAssembler::movptr(Address dst, int32_t src) {
 2621   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2622 }
 2623 
 2624 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2625   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2626   Assembler::movdqu(dst, src);
 2627 }
 2628 
 2629 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2630   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2631   Assembler::movdqu(dst, src);
 2632 }
 2633 
 2634 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2635   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2636   Assembler::movdqu(dst, src);
 2637 }
 2638 
 2639 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2640   assert(rscratch != noreg || always_reachable(src), "missing");
 2641 
 2642   if (reachable(src)) {
 2643     movdqu(dst, as_Address(src));
 2644   } else {
 2645     lea(rscratch, src);
 2646     movdqu(dst, Address(rscratch, 0));
 2647   }
 2648 }
 2649 
 2650 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2651   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2652   Assembler::vmovdqu(dst, src);
 2653 }
 2654 
 2655 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2656   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2657   Assembler::vmovdqu(dst, src);
 2658 }
 2659 
 2660 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2661   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2662   Assembler::vmovdqu(dst, src);
 2663 }
 2664 
 2665 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2666   assert(rscratch != noreg || always_reachable(src), "missing");
 2667 
 2668   if (reachable(src)) {
 2669     vmovdqu(dst, as_Address(src));
 2670   }
 2671   else {
 2672     lea(rscratch, src);
 2673     vmovdqu(dst, Address(rscratch, 0));
 2674   }
 2675 }
 2676 
 2677 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2678   assert(rscratch != noreg || always_reachable(src), "missing");
 2679 
 2680   if (vector_len == AVX_512bit) {
 2681     evmovdquq(dst, src, AVX_512bit, rscratch);
 2682   } else if (vector_len == AVX_256bit) {
 2683     vmovdqu(dst, src, rscratch);
 2684   } else {
 2685     movdqu(dst, src, rscratch);
 2686   }
 2687 }
 2688 
 2689 void MacroAssembler::kmov(KRegister dst, Address src) {
 2690   if (VM_Version::supports_avx512bw()) {
 2691     kmovql(dst, src);
 2692   } else {
 2693     assert(VM_Version::supports_evex(), "");
 2694     kmovwl(dst, src);
 2695   }
 2696 }
 2697 
 2698 void MacroAssembler::kmov(Address dst, KRegister src) {
 2699   if (VM_Version::supports_avx512bw()) {
 2700     kmovql(dst, src);
 2701   } else {
 2702     assert(VM_Version::supports_evex(), "");
 2703     kmovwl(dst, src);
 2704   }
 2705 }
 2706 
 2707 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2708   if (VM_Version::supports_avx512bw()) {
 2709     kmovql(dst, src);
 2710   } else {
 2711     assert(VM_Version::supports_evex(), "");
 2712     kmovwl(dst, src);
 2713   }
 2714 }
 2715 
 2716 void MacroAssembler::kmov(Register dst, KRegister src) {
 2717   if (VM_Version::supports_avx512bw()) {
 2718     kmovql(dst, src);
 2719   } else {
 2720     assert(VM_Version::supports_evex(), "");
 2721     kmovwl(dst, src);
 2722   }
 2723 }
 2724 
 2725 void MacroAssembler::kmov(KRegister dst, Register src) {
 2726   if (VM_Version::supports_avx512bw()) {
 2727     kmovql(dst, src);
 2728   } else {
 2729     assert(VM_Version::supports_evex(), "");
 2730     kmovwl(dst, src);
 2731   }
 2732 }
 2733 
 2734 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2735   assert(rscratch != noreg || always_reachable(src), "missing");
 2736 
 2737   if (reachable(src)) {
 2738     kmovql(dst, as_Address(src));
 2739   } else {
 2740     lea(rscratch, src);
 2741     kmovql(dst, Address(rscratch, 0));
 2742   }
 2743 }
 2744 
 2745 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2746   assert(rscratch != noreg || always_reachable(src), "missing");
 2747 
 2748   if (reachable(src)) {
 2749     kmovwl(dst, as_Address(src));
 2750   } else {
 2751     lea(rscratch, src);
 2752     kmovwl(dst, Address(rscratch, 0));
 2753   }
 2754 }
 2755 
 2756 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2757                                int vector_len, Register rscratch) {
 2758   assert(rscratch != noreg || always_reachable(src), "missing");
 2759 
 2760   if (reachable(src)) {
 2761     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2762   } else {
 2763     lea(rscratch, src);
 2764     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2765   }
 2766 }
 2767 
 2768 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2769                                int vector_len, Register rscratch) {
 2770   assert(rscratch != noreg || always_reachable(src), "missing");
 2771 
 2772   if (reachable(src)) {
 2773     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2774   } else {
 2775     lea(rscratch, src);
 2776     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2777   }
 2778 }
 2779 
 2780 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2781   assert(rscratch != noreg || always_reachable(src), "missing");
 2782 
 2783   if (reachable(src)) {
 2784     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2785   } else {
 2786     lea(rscratch, src);
 2787     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2788   }
 2789 }
 2790 
 2791 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2792   assert(rscratch != noreg || always_reachable(src), "missing");
 2793 
 2794   if (reachable(src)) {
 2795     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2796   } else {
 2797     lea(rscratch, src);
 2798     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2799   }
 2800 }
 2801 
 2802 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2803   assert(rscratch != noreg || always_reachable(src), "missing");
 2804 
 2805   if (reachable(src)) {
 2806     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2807   } else {
 2808     lea(rscratch, src);
 2809     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2810   }
 2811 }
 2812 
 2813 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2814   assert(rscratch != noreg || always_reachable(src), "missing");
 2815 
 2816   if (reachable(src)) {
 2817     Assembler::movdqa(dst, as_Address(src));
 2818   } else {
 2819     lea(rscratch, src);
 2820     Assembler::movdqa(dst, Address(rscratch, 0));
 2821   }
 2822 }
 2823 
 2824 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2825   assert(rscratch != noreg || always_reachable(src), "missing");
 2826 
 2827   if (reachable(src)) {
 2828     Assembler::movsd(dst, as_Address(src));
 2829   } else {
 2830     lea(rscratch, src);
 2831     Assembler::movsd(dst, Address(rscratch, 0));
 2832   }
 2833 }
 2834 
 2835 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2836   assert(rscratch != noreg || always_reachable(src), "missing");
 2837 
 2838   if (reachable(src)) {
 2839     Assembler::movss(dst, as_Address(src));
 2840   } else {
 2841     lea(rscratch, src);
 2842     Assembler::movss(dst, Address(rscratch, 0));
 2843   }
 2844 }
 2845 
 2846 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2847   assert(rscratch != noreg || always_reachable(src), "missing");
 2848 
 2849   if (reachable(src)) {
 2850     Assembler::movddup(dst, as_Address(src));
 2851   } else {
 2852     lea(rscratch, src);
 2853     Assembler::movddup(dst, Address(rscratch, 0));
 2854   }
 2855 }
 2856 
 2857 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2858   assert(rscratch != noreg || always_reachable(src), "missing");
 2859 
 2860   if (reachable(src)) {
 2861     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2862   } else {
 2863     lea(rscratch, src);
 2864     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2865   }
 2866 }
 2867 
 2868 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2869   assert(rscratch != noreg || always_reachable(src), "missing");
 2870 
 2871   if (reachable(src)) {
 2872     Assembler::mulsd(dst, as_Address(src));
 2873   } else {
 2874     lea(rscratch, src);
 2875     Assembler::mulsd(dst, Address(rscratch, 0));
 2876   }
 2877 }
 2878 
 2879 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2880   assert(rscratch != noreg || always_reachable(src), "missing");
 2881 
 2882   if (reachable(src)) {
 2883     Assembler::mulss(dst, as_Address(src));
 2884   } else {
 2885     lea(rscratch, src);
 2886     Assembler::mulss(dst, Address(rscratch, 0));
 2887   }
 2888 }
 2889 
 2890 void MacroAssembler::null_check(Register reg, int offset) {
 2891   if (needs_explicit_null_check(offset)) {
 2892     // provoke OS null exception if reg is null by
 2893     // accessing M[reg] w/o changing any (non-CC) registers
 2894     // NOTE: cmpl is plenty here to provoke a segv
 2895     cmpptr(rax, Address(reg, 0));
 2896     // Note: should probably use testl(rax, Address(reg, 0));
 2897     //       may be shorter code (however, this version of
 2898     //       testl needs to be implemented first)
 2899   } else {
 2900     // nothing to do, (later) access of M[reg + offset]
 2901     // will provoke OS null exception if reg is null
 2902   }
 2903 }
 2904 
 2905 void MacroAssembler::os_breakpoint() {
 2906   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2907   // (e.g., MSVC can't call ps() otherwise)
 2908   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2909 }
 2910 
 2911 void MacroAssembler::unimplemented(const char* what) {
 2912   const char* buf = nullptr;
 2913   {
 2914     ResourceMark rm;
 2915     stringStream ss;
 2916     ss.print("unimplemented: %s", what);
 2917     buf = code_string(ss.as_string());
 2918   }
 2919   stop(buf);
 2920 }
 2921 
 2922 #ifdef _LP64
 2923 #define XSTATE_BV 0x200
 2924 #endif
 2925 
 2926 void MacroAssembler::pop_CPU_state() {
 2927   pop_FPU_state();
 2928   pop_IU_state();
 2929 }
 2930 
 2931 void MacroAssembler::pop_FPU_state() {
 2932 #ifndef _LP64
 2933   frstor(Address(rsp, 0));
 2934 #else
 2935   fxrstor(Address(rsp, 0));
 2936 #endif
 2937   addptr(rsp, FPUStateSizeInWords * wordSize);
 2938 }
 2939 
 2940 void MacroAssembler::pop_IU_state() {
 2941   popa();
 2942   LP64_ONLY(addq(rsp, 8));
 2943   popf();
 2944 }
 2945 
 2946 // Save Integer and Float state
 2947 // Warning: Stack must be 16 byte aligned (64bit)
 2948 void MacroAssembler::push_CPU_state() {
 2949   push_IU_state();
 2950   push_FPU_state();
 2951 }
 2952 
 2953 void MacroAssembler::push_FPU_state() {
 2954   subptr(rsp, FPUStateSizeInWords * wordSize);
 2955 #ifndef _LP64
 2956   fnsave(Address(rsp, 0));
 2957   fwait();
 2958 #else
 2959   fxsave(Address(rsp, 0));
 2960 #endif // LP64
 2961 }
 2962 
 2963 void MacroAssembler::push_IU_state() {
 2964   // Push flags first because pusha kills them
 2965   pushf();
 2966   // Make sure rsp stays 16-byte aligned
 2967   LP64_ONLY(subq(rsp, 8));
 2968   pusha();
 2969 }
 2970 
 2971 void MacroAssembler::push_cont_fastpath() {
 2972   if (!Continuations::enabled()) return;
 2973 
 2974 #ifndef _LP64
 2975   Register rthread = rax;
 2976   Register rrealsp = rbx;
 2977   push(rthread);
 2978   push(rrealsp);
 2979 
 2980   get_thread(rthread);
 2981 
 2982   // The code below wants the original RSP.
 2983   // Move it back after the pushes above.
 2984   movptr(rrealsp, rsp);
 2985   addptr(rrealsp, 2*wordSize);
 2986 #else
 2987   Register rthread = r15_thread;
 2988   Register rrealsp = rsp;
 2989 #endif
 2990 
 2991   Label done;
 2992   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 2993   jccb(Assembler::belowEqual, done);
 2994   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 2995   bind(done);
 2996 
 2997 #ifndef _LP64
 2998   pop(rrealsp);
 2999   pop(rthread);
 3000 #endif
 3001 }
 3002 
 3003 void MacroAssembler::pop_cont_fastpath() {
 3004   if (!Continuations::enabled()) return;
 3005 
 3006 #ifndef _LP64
 3007   Register rthread = rax;
 3008   Register rrealsp = rbx;
 3009   push(rthread);
 3010   push(rrealsp);
 3011 
 3012   get_thread(rthread);
 3013 
 3014   // The code below wants the original RSP.
 3015   // Move it back after the pushes above.
 3016   movptr(rrealsp, rsp);
 3017   addptr(rrealsp, 2*wordSize);
 3018 #else
 3019   Register rthread = r15_thread;
 3020   Register rrealsp = rsp;
 3021 #endif
 3022 
 3023   Label done;
 3024   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3025   jccb(Assembler::below, done);
 3026   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3027   bind(done);
 3028 
 3029 #ifndef _LP64
 3030   pop(rrealsp);
 3031   pop(rthread);
 3032 #endif
 3033 }
 3034 
 3035 void MacroAssembler::inc_held_monitor_count() {
 3036 #ifndef _LP64
 3037   Register thread = rax;
 3038   push(thread);
 3039   get_thread(thread);
 3040   incrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3041   pop(thread);
 3042 #else // LP64
 3043   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3044 #endif
 3045 }
 3046 
 3047 void MacroAssembler::dec_held_monitor_count() {
 3048 #ifndef _LP64
 3049   Register thread = rax;
 3050   push(thread);
 3051   get_thread(thread);
 3052   decrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3053   pop(thread);
 3054 #else // LP64
 3055   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3056 #endif
 3057 }
 3058 
 3059 #ifdef ASSERT
 3060 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3061 #ifdef _LP64
 3062   Label no_cont;
 3063   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3064   testl(cont, cont);
 3065   jcc(Assembler::zero, no_cont);
 3066   stop(name);
 3067   bind(no_cont);
 3068 #else
 3069   Unimplemented();
 3070 #endif
 3071 }
 3072 #endif
 3073 
 3074 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3075   if (!java_thread->is_valid()) {
 3076     java_thread = rdi;
 3077     get_thread(java_thread);
 3078   }
 3079   // we must set sp to zero to clear frame
 3080   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3081   // must clear fp, so that compiled frames are not confused; it is
 3082   // possible that we need it only for debugging
 3083   if (clear_fp) {
 3084     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3085   }
 3086   // Always clear the pc because it could have been set by make_walkable()
 3087   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3088   vzeroupper();
 3089 }
 3090 
 3091 void MacroAssembler::restore_rax(Register tmp) {
 3092   if (tmp == noreg) pop(rax);
 3093   else if (tmp != rax) mov(rax, tmp);
 3094 }
 3095 
 3096 void MacroAssembler::round_to(Register reg, int modulus) {
 3097   addptr(reg, modulus - 1);
 3098   andptr(reg, -modulus);
 3099 }
 3100 
 3101 void MacroAssembler::save_rax(Register tmp) {
 3102   if (tmp == noreg) push(rax);
 3103   else if (tmp != rax) mov(tmp, rax);
 3104 }
 3105 
 3106 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3107   if (at_return) {
 3108     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3109     // we may safely use rsp instead to perform the stack watermark check.
 3110     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3111     jcc(Assembler::above, slow_path);
 3112     return;
 3113   }
 3114   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3115   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3116 }
 3117 
 3118 // Calls to C land
 3119 //
 3120 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3121 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3122 // has to be reset to 0. This is required to allow proper stack traversal.
 3123 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3124                                          Register last_java_sp,
 3125                                          Register last_java_fp,
 3126                                          address  last_java_pc,
 3127                                          Register rscratch) {
 3128   vzeroupper();
 3129   // determine java_thread register
 3130   if (!java_thread->is_valid()) {
 3131     java_thread = rdi;
 3132     get_thread(java_thread);
 3133   }
 3134   // determine last_java_sp register
 3135   if (!last_java_sp->is_valid()) {
 3136     last_java_sp = rsp;
 3137   }
 3138   // last_java_fp is optional
 3139   if (last_java_fp->is_valid()) {
 3140     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3141   }
 3142   // last_java_pc is optional
 3143   if (last_java_pc != nullptr) {
 3144     Address java_pc(java_thread,
 3145                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3146     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3147   }
 3148   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3149 }
 3150 
 3151 void MacroAssembler::shlptr(Register dst, int imm8) {
 3152   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3153 }
 3154 
 3155 void MacroAssembler::shrptr(Register dst, int imm8) {
 3156   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3157 }
 3158 
 3159 void MacroAssembler::sign_extend_byte(Register reg) {
 3160   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3161     movsbl(reg, reg); // movsxb
 3162   } else {
 3163     shll(reg, 24);
 3164     sarl(reg, 24);
 3165   }
 3166 }
 3167 
 3168 void MacroAssembler::sign_extend_short(Register reg) {
 3169   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3170     movswl(reg, reg); // movsxw
 3171   } else {
 3172     shll(reg, 16);
 3173     sarl(reg, 16);
 3174   }
 3175 }
 3176 
 3177 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3178   if (imm32 >= 0 && is8bit(imm32)) {
 3179     testb(dst, imm32);
 3180   } else {
 3181     Assembler::testl(dst, imm32);
 3182   }
 3183 }
 3184 
 3185 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3186   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3187     testb(dst, imm32);
 3188   } else {
 3189     Assembler::testl(dst, imm32);
 3190   }
 3191 }
 3192 
 3193 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3194   assert(always_reachable(src), "Address should be reachable");
 3195   testl(dst, as_Address(src));
 3196 }
 3197 
 3198 #ifdef _LP64
 3199 
 3200 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3201   if (imm32 >= 0) {
 3202     testl(dst, imm32);
 3203   } else {
 3204     Assembler::testq(dst, imm32);
 3205   }
 3206 }
 3207 
 3208 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3209   if (imm32 >= 0) {
 3210     testl(dst, imm32);
 3211   } else {
 3212     Assembler::testq(dst, imm32);
 3213   }
 3214 }
 3215 
 3216 #endif
 3217 
 3218 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3219   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3220   Assembler::pcmpeqb(dst, src);
 3221 }
 3222 
 3223 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3224   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3225   Assembler::pcmpeqw(dst, src);
 3226 }
 3227 
 3228 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3229   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3230   Assembler::pcmpestri(dst, src, imm8);
 3231 }
 3232 
 3233 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3234   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3235   Assembler::pcmpestri(dst, src, imm8);
 3236 }
 3237 
 3238 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3239   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3240   Assembler::pmovzxbw(dst, src);
 3241 }
 3242 
 3243 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3244   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3245   Assembler::pmovzxbw(dst, src);
 3246 }
 3247 
 3248 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3249   assert((src->encoding() < 16),"XMM register should be 0-15");
 3250   Assembler::pmovmskb(dst, src);
 3251 }
 3252 
 3253 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3254   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3255   Assembler::ptest(dst, src);
 3256 }
 3257 
 3258 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3259   assert(rscratch != noreg || always_reachable(src), "missing");
 3260 
 3261   if (reachable(src)) {
 3262     Assembler::sqrtss(dst, as_Address(src));
 3263   } else {
 3264     lea(rscratch, src);
 3265     Assembler::sqrtss(dst, Address(rscratch, 0));
 3266   }
 3267 }
 3268 
 3269 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3270   assert(rscratch != noreg || always_reachable(src), "missing");
 3271 
 3272   if (reachable(src)) {
 3273     Assembler::subsd(dst, as_Address(src));
 3274   } else {
 3275     lea(rscratch, src);
 3276     Assembler::subsd(dst, Address(rscratch, 0));
 3277   }
 3278 }
 3279 
 3280 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3281   assert(rscratch != noreg || always_reachable(src), "missing");
 3282 
 3283   if (reachable(src)) {
 3284     Assembler::roundsd(dst, as_Address(src), rmode);
 3285   } else {
 3286     lea(rscratch, src);
 3287     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3288   }
 3289 }
 3290 
 3291 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3292   assert(rscratch != noreg || always_reachable(src), "missing");
 3293 
 3294   if (reachable(src)) {
 3295     Assembler::subss(dst, as_Address(src));
 3296   } else {
 3297     lea(rscratch, src);
 3298     Assembler::subss(dst, Address(rscratch, 0));
 3299   }
 3300 }
 3301 
 3302 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3303   assert(rscratch != noreg || always_reachable(src), "missing");
 3304 
 3305   if (reachable(src)) {
 3306     Assembler::ucomisd(dst, as_Address(src));
 3307   } else {
 3308     lea(rscratch, src);
 3309     Assembler::ucomisd(dst, Address(rscratch, 0));
 3310   }
 3311 }
 3312 
 3313 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3314   assert(rscratch != noreg || always_reachable(src), "missing");
 3315 
 3316   if (reachable(src)) {
 3317     Assembler::ucomiss(dst, as_Address(src));
 3318   } else {
 3319     lea(rscratch, src);
 3320     Assembler::ucomiss(dst, Address(rscratch, 0));
 3321   }
 3322 }
 3323 
 3324 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3325   assert(rscratch != noreg || always_reachable(src), "missing");
 3326 
 3327   // Used in sign-bit flipping with aligned address.
 3328   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3329   if (reachable(src)) {
 3330     Assembler::xorpd(dst, as_Address(src));
 3331   } else {
 3332     lea(rscratch, src);
 3333     Assembler::xorpd(dst, Address(rscratch, 0));
 3334   }
 3335 }
 3336 
 3337 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3338   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3339     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3340   }
 3341   else {
 3342     Assembler::xorpd(dst, src);
 3343   }
 3344 }
 3345 
 3346 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3347   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3348     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3349   } else {
 3350     Assembler::xorps(dst, src);
 3351   }
 3352 }
 3353 
 3354 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3355   assert(rscratch != noreg || always_reachable(src), "missing");
 3356 
 3357   // Used in sign-bit flipping with aligned address.
 3358   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3359   if (reachable(src)) {
 3360     Assembler::xorps(dst, as_Address(src));
 3361   } else {
 3362     lea(rscratch, src);
 3363     Assembler::xorps(dst, Address(rscratch, 0));
 3364   }
 3365 }
 3366 
 3367 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3368   assert(rscratch != noreg || always_reachable(src), "missing");
 3369 
 3370   // Used in sign-bit flipping with aligned address.
 3371   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3372   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3373   if (reachable(src)) {
 3374     Assembler::pshufb(dst, as_Address(src));
 3375   } else {
 3376     lea(rscratch, src);
 3377     Assembler::pshufb(dst, Address(rscratch, 0));
 3378   }
 3379 }
 3380 
 3381 // AVX 3-operands instructions
 3382 
 3383 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3384   assert(rscratch != noreg || always_reachable(src), "missing");
 3385 
 3386   if (reachable(src)) {
 3387     vaddsd(dst, nds, as_Address(src));
 3388   } else {
 3389     lea(rscratch, src);
 3390     vaddsd(dst, nds, Address(rscratch, 0));
 3391   }
 3392 }
 3393 
 3394 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3395   assert(rscratch != noreg || always_reachable(src), "missing");
 3396 
 3397   if (reachable(src)) {
 3398     vaddss(dst, nds, as_Address(src));
 3399   } else {
 3400     lea(rscratch, src);
 3401     vaddss(dst, nds, Address(rscratch, 0));
 3402   }
 3403 }
 3404 
 3405 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3406   assert(UseAVX > 0, "requires some form of AVX");
 3407   assert(rscratch != noreg || always_reachable(src), "missing");
 3408 
 3409   if (reachable(src)) {
 3410     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3411   } else {
 3412     lea(rscratch, src);
 3413     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3414   }
 3415 }
 3416 
 3417 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3418   assert(UseAVX > 0, "requires some form of AVX");
 3419   assert(rscratch != noreg || always_reachable(src), "missing");
 3420 
 3421   if (reachable(src)) {
 3422     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3423   } else {
 3424     lea(rscratch, src);
 3425     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3426   }
 3427 }
 3428 
 3429 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3430   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3431   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3432 
 3433   vandps(dst, nds, negate_field, vector_len, rscratch);
 3434 }
 3435 
 3436 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3437   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3438   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3439 
 3440   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3441 }
 3442 
 3443 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3444   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3445   Assembler::vpaddb(dst, nds, src, vector_len);
 3446 }
 3447 
 3448 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3449   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3450   Assembler::vpaddb(dst, nds, src, vector_len);
 3451 }
 3452 
 3453 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3454   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3455   Assembler::vpaddw(dst, nds, src, vector_len);
 3456 }
 3457 
 3458 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3459   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3460   Assembler::vpaddw(dst, nds, src, vector_len);
 3461 }
 3462 
 3463 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3464   assert(rscratch != noreg || always_reachable(src), "missing");
 3465 
 3466   if (reachable(src)) {
 3467     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3468   } else {
 3469     lea(rscratch, src);
 3470     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3471   }
 3472 }
 3473 
 3474 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3475   assert(rscratch != noreg || always_reachable(src), "missing");
 3476 
 3477   if (reachable(src)) {
 3478     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3479   } else {
 3480     lea(rscratch, src);
 3481     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3482   }
 3483 }
 3484 
 3485 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3486   assert(rscratch != noreg || always_reachable(src), "missing");
 3487 
 3488   if (reachable(src)) {
 3489     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3490   } else {
 3491     lea(rscratch, src);
 3492     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3493   }
 3494 }
 3495 
 3496 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3497   assert(rscratch != noreg || always_reachable(src), "missing");
 3498 
 3499   if (reachable(src)) {
 3500     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3501   } else {
 3502     lea(rscratch, src);
 3503     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3504   }
 3505 }
 3506 
 3507 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3508   assert(rscratch != noreg || always_reachable(src), "missing");
 3509 
 3510   if (reachable(src)) {
 3511     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3512   } else {
 3513     lea(rscratch, src);
 3514     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3515   }
 3516 }
 3517 
 3518 // Vector float blend
 3519 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3520 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3521   // WARN: Allow dst == (src1|src2), mask == scratch
 3522   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3523   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3524   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3525   if (blend_emulation && scratch_available && dst_available) {
 3526     if (compute_mask) {
 3527       vpsrad(scratch, mask, 32, vector_len);
 3528       mask = scratch;
 3529     }
 3530     if (dst == src1) {
 3531       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3532       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3533     } else {
 3534       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3535       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3536     }
 3537     vpor(dst, dst, scratch, vector_len);
 3538   } else {
 3539     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3540   }
 3541 }
 3542 
 3543 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3544 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3545   // WARN: Allow dst == (src1|src2), mask == scratch
 3546   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3547   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3548   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3549   if (blend_emulation && scratch_available && dst_available) {
 3550     if (compute_mask) {
 3551       vpxor(scratch, scratch, scratch, vector_len);
 3552       vpcmpgtq(scratch, scratch, mask, vector_len);
 3553       mask = scratch;
 3554     }
 3555     if (dst == src1) {
 3556       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3557       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3558     } else {
 3559       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3560       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3561     }
 3562     vpor(dst, dst, scratch, vector_len);
 3563   } else {
 3564     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3565   }
 3566 }
 3567 
 3568 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3569   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3570   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3571 }
 3572 
 3573 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 3574   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3575   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
 3576 }
 3577 
 3578 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3579   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3580   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3581 }
 3582 
 3583 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3584   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3585   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3586 }
 3587 
 3588 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3589   assert(rscratch != noreg || always_reachable(src), "missing");
 3590 
 3591   if (reachable(src)) {
 3592     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3593   } else {
 3594     lea(rscratch, src);
 3595     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3596   }
 3597 }
 3598 
 3599 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3600                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3601   assert(rscratch != noreg || always_reachable(src), "missing");
 3602 
 3603   if (reachable(src)) {
 3604     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3605   } else {
 3606     lea(rscratch, src);
 3607     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3608   }
 3609 }
 3610 
 3611 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3612                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3613   assert(rscratch != noreg || always_reachable(src), "missing");
 3614 
 3615   if (reachable(src)) {
 3616     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3617   } else {
 3618     lea(rscratch, src);
 3619     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3620   }
 3621 }
 3622 
 3623 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3624                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3625   assert(rscratch != noreg || always_reachable(src), "missing");
 3626 
 3627   if (reachable(src)) {
 3628     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3629   } else {
 3630     lea(rscratch, src);
 3631     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3632   }
 3633 }
 3634 
 3635 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3636                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3637   assert(rscratch != noreg || always_reachable(src), "missing");
 3638 
 3639   if (reachable(src)) {
 3640     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3641   } else {
 3642     lea(rscratch, src);
 3643     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3644   }
 3645 }
 3646 
 3647 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3648   if (width == Assembler::Q) {
 3649     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3650   } else {
 3651     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3652   }
 3653 }
 3654 
 3655 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3656   int eq_cond_enc = 0x29;
 3657   int gt_cond_enc = 0x37;
 3658   if (width != Assembler::Q) {
 3659     eq_cond_enc = 0x74 + width;
 3660     gt_cond_enc = 0x64 + width;
 3661   }
 3662   switch (cond) {
 3663   case eq:
 3664     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3665     break;
 3666   case neq:
 3667     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3668     vallones(xtmp, vector_len);
 3669     vpxor(dst, xtmp, dst, vector_len);
 3670     break;
 3671   case le:
 3672     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3673     vallones(xtmp, vector_len);
 3674     vpxor(dst, xtmp, dst, vector_len);
 3675     break;
 3676   case nlt:
 3677     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3678     vallones(xtmp, vector_len);
 3679     vpxor(dst, xtmp, dst, vector_len);
 3680     break;
 3681   case lt:
 3682     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3683     break;
 3684   case nle:
 3685     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3686     break;
 3687   default:
 3688     assert(false, "Should not reach here");
 3689   }
 3690 }
 3691 
 3692 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3693   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3694   Assembler::vpmovzxbw(dst, src, vector_len);
 3695 }
 3696 
 3697 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3698   assert((src->encoding() < 16),"XMM register should be 0-15");
 3699   Assembler::vpmovmskb(dst, src, vector_len);
 3700 }
 3701 
 3702 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3703   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3704   Assembler::vpmullw(dst, nds, src, vector_len);
 3705 }
 3706 
 3707 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3708   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3709   Assembler::vpmullw(dst, nds, src, vector_len);
 3710 }
 3711 
 3712 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3713   assert((UseAVX > 0), "AVX support is needed");
 3714   assert(rscratch != noreg || always_reachable(src), "missing");
 3715 
 3716   if (reachable(src)) {
 3717     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3718   } else {
 3719     lea(rscratch, src);
 3720     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3721   }
 3722 }
 3723 
 3724 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3725   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3726   Assembler::vpsubb(dst, nds, src, vector_len);
 3727 }
 3728 
 3729 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3730   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3731   Assembler::vpsubb(dst, nds, src, vector_len);
 3732 }
 3733 
 3734 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3735   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3736   Assembler::vpsubw(dst, nds, src, vector_len);
 3737 }
 3738 
 3739 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3740   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3741   Assembler::vpsubw(dst, nds, src, vector_len);
 3742 }
 3743 
 3744 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3745   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3746   Assembler::vpsraw(dst, nds, shift, vector_len);
 3747 }
 3748 
 3749 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3750   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3751   Assembler::vpsraw(dst, nds, shift, vector_len);
 3752 }
 3753 
 3754 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3755   assert(UseAVX > 2,"");
 3756   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3757      vector_len = 2;
 3758   }
 3759   Assembler::evpsraq(dst, nds, shift, vector_len);
 3760 }
 3761 
 3762 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3763   assert(UseAVX > 2,"");
 3764   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3765      vector_len = 2;
 3766   }
 3767   Assembler::evpsraq(dst, nds, shift, vector_len);
 3768 }
 3769 
 3770 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3771   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3772   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3773 }
 3774 
 3775 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3776   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3777   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3778 }
 3779 
 3780 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3781   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3782   Assembler::vpsllw(dst, nds, shift, vector_len);
 3783 }
 3784 
 3785 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3786   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3787   Assembler::vpsllw(dst, nds, shift, vector_len);
 3788 }
 3789 
 3790 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3791   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3792   Assembler::vptest(dst, src);
 3793 }
 3794 
 3795 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3796   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3797   Assembler::punpcklbw(dst, src);
 3798 }
 3799 
 3800 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3801   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3802   Assembler::pshufd(dst, src, mode);
 3803 }
 3804 
 3805 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3806   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3807   Assembler::pshuflw(dst, src, mode);
 3808 }
 3809 
 3810 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3811   assert(rscratch != noreg || always_reachable(src), "missing");
 3812 
 3813   if (reachable(src)) {
 3814     vandpd(dst, nds, as_Address(src), vector_len);
 3815   } else {
 3816     lea(rscratch, src);
 3817     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3818   }
 3819 }
 3820 
 3821 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3822   assert(rscratch != noreg || always_reachable(src), "missing");
 3823 
 3824   if (reachable(src)) {
 3825     vandps(dst, nds, as_Address(src), vector_len);
 3826   } else {
 3827     lea(rscratch, src);
 3828     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3829   }
 3830 }
 3831 
 3832 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3833                             bool merge, int vector_len, Register rscratch) {
 3834   assert(rscratch != noreg || always_reachable(src), "missing");
 3835 
 3836   if (reachable(src)) {
 3837     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3838   } else {
 3839     lea(rscratch, src);
 3840     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3841   }
 3842 }
 3843 
 3844 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3845   assert(rscratch != noreg || always_reachable(src), "missing");
 3846 
 3847   if (reachable(src)) {
 3848     vdivsd(dst, nds, as_Address(src));
 3849   } else {
 3850     lea(rscratch, src);
 3851     vdivsd(dst, nds, Address(rscratch, 0));
 3852   }
 3853 }
 3854 
 3855 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3856   assert(rscratch != noreg || always_reachable(src), "missing");
 3857 
 3858   if (reachable(src)) {
 3859     vdivss(dst, nds, as_Address(src));
 3860   } else {
 3861     lea(rscratch, src);
 3862     vdivss(dst, nds, Address(rscratch, 0));
 3863   }
 3864 }
 3865 
 3866 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3867   assert(rscratch != noreg || always_reachable(src), "missing");
 3868 
 3869   if (reachable(src)) {
 3870     vmulsd(dst, nds, as_Address(src));
 3871   } else {
 3872     lea(rscratch, src);
 3873     vmulsd(dst, nds, Address(rscratch, 0));
 3874   }
 3875 }
 3876 
 3877 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3878   assert(rscratch != noreg || always_reachable(src), "missing");
 3879 
 3880   if (reachable(src)) {
 3881     vmulss(dst, nds, as_Address(src));
 3882   } else {
 3883     lea(rscratch, src);
 3884     vmulss(dst, nds, Address(rscratch, 0));
 3885   }
 3886 }
 3887 
 3888 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3889   assert(rscratch != noreg || always_reachable(src), "missing");
 3890 
 3891   if (reachable(src)) {
 3892     vsubsd(dst, nds, as_Address(src));
 3893   } else {
 3894     lea(rscratch, src);
 3895     vsubsd(dst, nds, Address(rscratch, 0));
 3896   }
 3897 }
 3898 
 3899 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3900   assert(rscratch != noreg || always_reachable(src), "missing");
 3901 
 3902   if (reachable(src)) {
 3903     vsubss(dst, nds, as_Address(src));
 3904   } else {
 3905     lea(rscratch, src);
 3906     vsubss(dst, nds, Address(rscratch, 0));
 3907   }
 3908 }
 3909 
 3910 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3911   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3912   assert(rscratch != noreg || always_reachable(src), "missing");
 3913 
 3914   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3915 }
 3916 
 3917 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3918   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3919   assert(rscratch != noreg || always_reachable(src), "missing");
 3920 
 3921   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3922 }
 3923 
 3924 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3925   assert(rscratch != noreg || always_reachable(src), "missing");
 3926 
 3927   if (reachable(src)) {
 3928     vxorpd(dst, nds, as_Address(src), vector_len);
 3929   } else {
 3930     lea(rscratch, src);
 3931     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 3932   }
 3933 }
 3934 
 3935 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3936   assert(rscratch != noreg || always_reachable(src), "missing");
 3937 
 3938   if (reachable(src)) {
 3939     vxorps(dst, nds, as_Address(src), vector_len);
 3940   } else {
 3941     lea(rscratch, src);
 3942     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 3943   }
 3944 }
 3945 
 3946 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3947   assert(rscratch != noreg || always_reachable(src), "missing");
 3948 
 3949   if (UseAVX > 1 || (vector_len < 1)) {
 3950     if (reachable(src)) {
 3951       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 3952     } else {
 3953       lea(rscratch, src);
 3954       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 3955     }
 3956   } else {
 3957     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 3958   }
 3959 }
 3960 
 3961 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3962   assert(rscratch != noreg || always_reachable(src), "missing");
 3963 
 3964   if (reachable(src)) {
 3965     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 3966   } else {
 3967     lea(rscratch, src);
 3968     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 3969   }
 3970 }
 3971 
 3972 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 3973   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 3974   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 3975   // The inverted mask is sign-extended
 3976   andptr(possibly_non_local, inverted_mask);
 3977 }
 3978 
 3979 void MacroAssembler::resolve_jobject(Register value,
 3980                                      Register thread,
 3981                                      Register tmp) {
 3982   assert_different_registers(value, thread, tmp);
 3983   Label done, tagged, weak_tagged;
 3984   testptr(value, value);
 3985   jcc(Assembler::zero, done);           // Use null as-is.
 3986   testptr(value, JNIHandles::tag_mask); // Test for tag.
 3987   jcc(Assembler::notZero, tagged);
 3988 
 3989   // Resolve local handle
 3990   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 3991   verify_oop(value);
 3992   jmp(done);
 3993 
 3994   bind(tagged);
 3995   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 3996   jcc(Assembler::notZero, weak_tagged);
 3997 
 3998   // Resolve global handle
 3999   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4000   verify_oop(value);
 4001   jmp(done);
 4002 
 4003   bind(weak_tagged);
 4004   // Resolve jweak.
 4005   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4006                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4007   verify_oop(value);
 4008 
 4009   bind(done);
 4010 }
 4011 
 4012 void MacroAssembler::resolve_global_jobject(Register value,
 4013                                             Register thread,
 4014                                             Register tmp) {
 4015   assert_different_registers(value, thread, tmp);
 4016   Label done;
 4017 
 4018   testptr(value, value);
 4019   jcc(Assembler::zero, done);           // Use null as-is.
 4020 
 4021 #ifdef ASSERT
 4022   {
 4023     Label valid_global_tag;
 4024     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4025     jcc(Assembler::notZero, valid_global_tag);
 4026     stop("non global jobject using resolve_global_jobject");
 4027     bind(valid_global_tag);
 4028   }
 4029 #endif
 4030 
 4031   // Resolve global handle
 4032   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4033   verify_oop(value);
 4034 
 4035   bind(done);
 4036 }
 4037 
 4038 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4039   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4040 }
 4041 
 4042 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4043 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4044   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4045 }
 4046 
 4047 void MacroAssembler::subptr(Register dst, Register src) {
 4048   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4049 }
 4050 
 4051 // C++ bool manipulation
 4052 void MacroAssembler::testbool(Register dst) {
 4053   if(sizeof(bool) == 1)
 4054     testb(dst, 0xff);
 4055   else if(sizeof(bool) == 2) {
 4056     // testw implementation needed for two byte bools
 4057     ShouldNotReachHere();
 4058   } else if(sizeof(bool) == 4)
 4059     testl(dst, dst);
 4060   else
 4061     // unsupported
 4062     ShouldNotReachHere();
 4063 }
 4064 
 4065 void MacroAssembler::testptr(Register dst, Register src) {
 4066   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4067 }
 4068 
 4069 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4070 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4071                                    Register var_size_in_bytes,
 4072                                    int con_size_in_bytes,
 4073                                    Register t1,
 4074                                    Register t2,
 4075                                    Label& slow_case) {
 4076   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4077   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4078 }
 4079 
 4080 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4081   RegSet regs;
 4082 #ifdef _LP64
 4083   regs += RegSet::of(rax, rcx, rdx);
 4084 #ifndef WINDOWS
 4085   regs += RegSet::of(rsi, rdi);
 4086 #endif
 4087   regs += RegSet::range(r8, r11);
 4088 #else
 4089   regs += RegSet::of(rax, rcx, rdx);
 4090 #endif
 4091   return regs;
 4092 }
 4093 
 4094 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4095   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4096 #if defined(WINDOWS) && defined(_LP64)
 4097   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4098   if (num_xmm_registers > 16) {
 4099      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4100   }
 4101   return result;
 4102 #else
 4103   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4104 #endif
 4105 }
 4106 
 4107 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4108 
 4109 #ifndef _LP64
 4110 static bool use_x87_registers() { return UseSSE < 2; }
 4111 #endif
 4112 static bool use_xmm_registers() { return UseSSE >= 1; }
 4113 
 4114 // C1 only ever uses the first double/float of the XMM register.
 4115 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4116 
 4117 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4118   if (UseSSE == 1) {
 4119     masm->movflt(Address(rsp, offset), reg);
 4120   } else {
 4121     masm->movdbl(Address(rsp, offset), reg);
 4122   }
 4123 }
 4124 
 4125 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4126   if (UseSSE == 1) {
 4127     masm->movflt(reg, Address(rsp, offset));
 4128   } else {
 4129     masm->movdbl(reg, Address(rsp, offset));
 4130   }
 4131 }
 4132 
 4133 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4134                                   bool save_fpu, int& gp_area_size,
 4135                                   int& fp_area_size, int& xmm_area_size) {
 4136 
 4137   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4138                          StackAlignmentInBytes);
 4139 #ifdef _LP64
 4140   fp_area_size = 0;
 4141 #else
 4142   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4143 #endif
 4144   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4145 
 4146   return gp_area_size + fp_area_size + xmm_area_size;
 4147 }
 4148 
 4149 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4150   block_comment("push_call_clobbered_registers start");
 4151   // Regular registers
 4152   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4153 
 4154   int gp_area_size;
 4155   int fp_area_size;
 4156   int xmm_area_size;
 4157   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4158                                                gp_area_size, fp_area_size, xmm_area_size);
 4159   subptr(rsp, total_save_size);
 4160 
 4161   push_set(gp_registers_to_push, 0);
 4162 
 4163 #ifndef _LP64
 4164   if (save_fpu && use_x87_registers()) {
 4165     fnsave(Address(rsp, gp_area_size));
 4166     fwait();
 4167   }
 4168 #endif
 4169   if (save_fpu && use_xmm_registers()) {
 4170     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4171   }
 4172 
 4173   block_comment("push_call_clobbered_registers end");
 4174 }
 4175 
 4176 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4177   block_comment("pop_call_clobbered_registers start");
 4178 
 4179   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4180 
 4181   int gp_area_size;
 4182   int fp_area_size;
 4183   int xmm_area_size;
 4184   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4185                                                gp_area_size, fp_area_size, xmm_area_size);
 4186 
 4187   if (restore_fpu && use_xmm_registers()) {
 4188     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4189   }
 4190 #ifndef _LP64
 4191   if (restore_fpu && use_x87_registers()) {
 4192     frstor(Address(rsp, gp_area_size));
 4193   }
 4194 #endif
 4195 
 4196   pop_set(gp_registers_to_pop, 0);
 4197 
 4198   addptr(rsp, total_save_size);
 4199 
 4200   vzeroupper();
 4201 
 4202   block_comment("pop_call_clobbered_registers end");
 4203 }
 4204 
 4205 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4206   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4207   int spill_offset = offset;
 4208 
 4209   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4210     save_xmm_register(this, spill_offset, *it);
 4211     spill_offset += xmm_save_size();
 4212   }
 4213 }
 4214 
 4215 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4216   int restore_size = set.size() * xmm_save_size();
 4217   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4218 
 4219   int restore_offset = offset + restore_size - xmm_save_size();
 4220 
 4221   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4222     restore_xmm_register(this, restore_offset, *it);
 4223     restore_offset -= xmm_save_size();
 4224   }
 4225 }
 4226 
 4227 void MacroAssembler::push_set(RegSet set, int offset) {
 4228   int spill_offset;
 4229   if (offset == -1) {
 4230     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4231     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4232     subptr(rsp, aligned_size);
 4233     spill_offset = 0;
 4234   } else {
 4235     spill_offset = offset;
 4236   }
 4237 
 4238   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4239     movptr(Address(rsp, spill_offset), *it);
 4240     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4241   }
 4242 }
 4243 
 4244 void MacroAssembler::pop_set(RegSet set, int offset) {
 4245 
 4246   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4247   int restore_size = set.size() * gp_reg_size;
 4248   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4249 
 4250   int restore_offset;
 4251   if (offset == -1) {
 4252     restore_offset = restore_size - gp_reg_size;
 4253   } else {
 4254     restore_offset = offset + restore_size - gp_reg_size;
 4255   }
 4256   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4257     movptr(*it, Address(rsp, restore_offset));
 4258     restore_offset -= gp_reg_size;
 4259   }
 4260 
 4261   if (offset == -1) {
 4262     addptr(rsp, aligned_size);
 4263   }
 4264 }
 4265 
 4266 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4267 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4268   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4269   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4270   Label done;
 4271 
 4272   testptr(length_in_bytes, length_in_bytes);
 4273   jcc(Assembler::zero, done);
 4274 
 4275   // initialize topmost word, divide index by 2, check if odd and test if zero
 4276   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4277 #ifdef ASSERT
 4278   {
 4279     Label L;
 4280     testptr(length_in_bytes, BytesPerWord - 1);
 4281     jcc(Assembler::zero, L);
 4282     stop("length must be a multiple of BytesPerWord");
 4283     bind(L);
 4284   }
 4285 #endif
 4286   Register index = length_in_bytes;
 4287   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4288   if (UseIncDec) {
 4289     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4290   } else {
 4291     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4292     shrptr(index, 1);
 4293   }
 4294 #ifndef _LP64
 4295   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4296   {
 4297     Label even;
 4298     // note: if index was a multiple of 8, then it cannot
 4299     //       be 0 now otherwise it must have been 0 before
 4300     //       => if it is even, we don't need to check for 0 again
 4301     jcc(Assembler::carryClear, even);
 4302     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4303     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4304     // index could be 0 now, must check again
 4305     jcc(Assembler::zero, done);
 4306     bind(even);
 4307   }
 4308 #endif // !_LP64
 4309   // initialize remaining object fields: index is a multiple of 2 now
 4310   {
 4311     Label loop;
 4312     bind(loop);
 4313     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4314     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4315     decrement(index);
 4316     jcc(Assembler::notZero, loop);
 4317   }
 4318 
 4319   bind(done);
 4320 }
 4321 
 4322 // Look up the method for a megamorphic invokeinterface call.
 4323 // The target method is determined by <intf_klass, itable_index>.
 4324 // The receiver klass is in recv_klass.
 4325 // On success, the result will be in method_result, and execution falls through.
 4326 // On failure, execution transfers to the given label.
 4327 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4328                                              Register intf_klass,
 4329                                              RegisterOrConstant itable_index,
 4330                                              Register method_result,
 4331                                              Register scan_temp,
 4332                                              Label& L_no_such_interface,
 4333                                              bool return_method) {
 4334   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4335   assert_different_registers(method_result, intf_klass, scan_temp);
 4336   assert(recv_klass != method_result || !return_method,
 4337          "recv_klass can be destroyed when method isn't needed");
 4338 
 4339   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4340          "caller must use same register for non-constant itable index as for method");
 4341 
 4342   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4343   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4344   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4345   int scan_step   = itableOffsetEntry::size() * wordSize;
 4346   int vte_size    = vtableEntry::size_in_bytes();
 4347   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4348   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4349 
 4350   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4351 
 4352   // Could store the aligned, prescaled offset in the klass.
 4353   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4354 
 4355   if (return_method) {
 4356     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4357     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4358     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4359   }
 4360 
 4361   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4362   //   if (scan->interface() == intf) {
 4363   //     result = (klass + scan->offset() + itable_index);
 4364   //   }
 4365   // }
 4366   Label search, found_method;
 4367 
 4368   for (int peel = 1; peel >= 0; peel--) {
 4369     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4370     cmpptr(intf_klass, method_result);
 4371 
 4372     if (peel) {
 4373       jccb(Assembler::equal, found_method);
 4374     } else {
 4375       jccb(Assembler::notEqual, search);
 4376       // (invert the test to fall through to found_method...)
 4377     }
 4378 
 4379     if (!peel)  break;
 4380 
 4381     bind(search);
 4382 
 4383     // Check that the previous entry is non-null.  A null entry means that
 4384     // the receiver class doesn't implement the interface, and wasn't the
 4385     // same as when the caller was compiled.
 4386     testptr(method_result, method_result);
 4387     jcc(Assembler::zero, L_no_such_interface);
 4388     addptr(scan_temp, scan_step);
 4389   }
 4390 
 4391   bind(found_method);
 4392 
 4393   if (return_method) {
 4394     // Got a hit.
 4395     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4396     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4397   }
 4398 }
 4399 
 4400 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4401 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4402 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4403 // The target method is determined by <holder_klass, itable_index>.
 4404 // The receiver klass is in recv_klass.
 4405 // On success, the result will be in method_result, and execution falls through.
 4406 // On failure, execution transfers to the given label.
 4407 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4408                                                   Register holder_klass,
 4409                                                   Register resolved_klass,
 4410                                                   Register method_result,
 4411                                                   Register scan_temp,
 4412                                                   Register temp_reg2,
 4413                                                   Register receiver,
 4414                                                   int itable_index,
 4415                                                   Label& L_no_such_interface) {
 4416   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4417   Register temp_itbl_klass = method_result;
 4418   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4419 
 4420   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4421   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4422   int scan_step = itableOffsetEntry::size() * wordSize;
 4423   int vte_size = vtableEntry::size_in_bytes();
 4424   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4425   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4426   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4427   assert(vte_size == wordSize, "adjust times_vte_scale");
 4428 
 4429   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4430 
 4431   // temp_itbl_klass = recv_klass.itable[0]
 4432   // scan_temp = &recv_klass.itable[0] + step
 4433   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4434   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4435   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4436   xorptr(temp_reg, temp_reg);
 4437 
 4438   // Initial checks:
 4439   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4440   //   - if (itable[0] == 0), no such interface
 4441   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4442   cmpptr(holder_klass, resolved_klass);
 4443   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4444   testptr(temp_itbl_klass, temp_itbl_klass);
 4445   jccb(Assembler::zero, L_no_such_interface);
 4446   cmpptr(holder_klass, temp_itbl_klass);
 4447   jccb(Assembler::equal, L_holder_found);
 4448 
 4449   // Loop: Look for holder_klass record in itable
 4450   //   do {
 4451   //     tmp = itable[index];
 4452   //     index += step;
 4453   //     if (tmp == holder_klass) {
 4454   //       goto L_holder_found; // Found!
 4455   //     }
 4456   //   } while (tmp != 0);
 4457   //   goto L_no_such_interface // Not found.
 4458   Label L_scan_holder;
 4459   bind(L_scan_holder);
 4460     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4461     addptr(scan_temp, scan_step);
 4462     cmpptr(holder_klass, temp_itbl_klass);
 4463     jccb(Assembler::equal, L_holder_found);
 4464     testptr(temp_itbl_klass, temp_itbl_klass);
 4465     jccb(Assembler::notZero, L_scan_holder);
 4466 
 4467   jmpb(L_no_such_interface);
 4468 
 4469   // Loop: Look for resolved_class record in itable
 4470   //   do {
 4471   //     tmp = itable[index];
 4472   //     index += step;
 4473   //     if (tmp == holder_klass) {
 4474   //        // Also check if we have met a holder klass
 4475   //        holder_tmp = itable[index-step-ioffset];
 4476   //     }
 4477   //     if (tmp == resolved_klass) {
 4478   //        goto L_resolved_found;  // Found!
 4479   //     }
 4480   //   } while (tmp != 0);
 4481   //   goto L_no_such_interface // Not found.
 4482   //
 4483   Label L_loop_scan_resolved;
 4484   bind(L_loop_scan_resolved);
 4485     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4486     addptr(scan_temp, scan_step);
 4487     bind(L_loop_scan_resolved_entry);
 4488     cmpptr(holder_klass, temp_itbl_klass);
 4489     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4490     cmpptr(resolved_klass, temp_itbl_klass);
 4491     jccb(Assembler::equal, L_resolved_found);
 4492     testptr(temp_itbl_klass, temp_itbl_klass);
 4493     jccb(Assembler::notZero, L_loop_scan_resolved);
 4494 
 4495   jmpb(L_no_such_interface);
 4496 
 4497   Label L_ready;
 4498 
 4499   // See if we already have a holder klass. If not, go and scan for it.
 4500   bind(L_resolved_found);
 4501   testptr(temp_reg, temp_reg);
 4502   jccb(Assembler::zero, L_scan_holder);
 4503   jmpb(L_ready);
 4504 
 4505   bind(L_holder_found);
 4506   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4507 
 4508   // Finally, temp_reg contains holder_klass vtable offset
 4509   bind(L_ready);
 4510   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4511   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4512     load_klass(scan_temp, receiver, noreg);
 4513     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4514   } else {
 4515     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4516   }
 4517 }
 4518 
 4519 
 4520 // virtual method calling
 4521 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4522                                            RegisterOrConstant vtable_index,
 4523                                            Register method_result) {
 4524   const ByteSize base = Klass::vtable_start_offset();
 4525   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4526   Address vtable_entry_addr(recv_klass,
 4527                             vtable_index, Address::times_ptr,
 4528                             base + vtableEntry::method_offset());
 4529   movptr(method_result, vtable_entry_addr);
 4530 }
 4531 
 4532 
 4533 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4534                            Register super_klass,
 4535                            Register temp_reg,
 4536                            Label& L_success) {
 4537   Label L_failure;
 4538   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4539   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4540   bind(L_failure);
 4541 }
 4542 
 4543 
 4544 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4545                                                    Register super_klass,
 4546                                                    Register temp_reg,
 4547                                                    Label* L_success,
 4548                                                    Label* L_failure,
 4549                                                    Label* L_slow_path,
 4550                                         RegisterOrConstant super_check_offset) {
 4551   assert_different_registers(sub_klass, super_klass, temp_reg);
 4552   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4553   if (super_check_offset.is_register()) {
 4554     assert_different_registers(sub_klass, super_klass,
 4555                                super_check_offset.as_register());
 4556   } else if (must_load_sco) {
 4557     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4558   }
 4559 
 4560   Label L_fallthrough;
 4561   int label_nulls = 0;
 4562   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4563   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4564   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4565   assert(label_nulls <= 1, "at most one null in the batch");
 4566 
 4567   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4568   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4569   Address super_check_offset_addr(super_klass, sco_offset);
 4570 
 4571   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4572   // range of a jccb.  If this routine grows larger, reconsider at
 4573   // least some of these.
 4574 #define local_jcc(assembler_cond, label)                                \
 4575   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4576   else                             jcc( assembler_cond, label) /*omit semi*/
 4577 
 4578   // Hacked jmp, which may only be used just before L_fallthrough.
 4579 #define final_jmp(label)                                                \
 4580   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4581   else                            jmp(label)                /*omit semi*/
 4582 
 4583   // If the pointers are equal, we are done (e.g., String[] elements).
 4584   // This self-check enables sharing of secondary supertype arrays among
 4585   // non-primary types such as array-of-interface.  Otherwise, each such
 4586   // type would need its own customized SSA.
 4587   // We move this check to the front of the fast path because many
 4588   // type checks are in fact trivially successful in this manner,
 4589   // so we get a nicely predicted branch right at the start of the check.
 4590   cmpptr(sub_klass, super_klass);
 4591   local_jcc(Assembler::equal, *L_success);
 4592 
 4593   // Check the supertype display:
 4594   if (must_load_sco) {
 4595     // Positive movl does right thing on LP64.
 4596     movl(temp_reg, super_check_offset_addr);
 4597     super_check_offset = RegisterOrConstant(temp_reg);
 4598   }
 4599   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4600   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4601 
 4602   // This check has worked decisively for primary supers.
 4603   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4604   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4605   // This works in the same check above because of a tricky aliasing
 4606   // between the super_cache and the primary super display elements.
 4607   // (The 'super_check_addr' can address either, as the case requires.)
 4608   // Note that the cache is updated below if it does not help us find
 4609   // what we need immediately.
 4610   // So if it was a primary super, we can just fail immediately.
 4611   // Otherwise, it's the slow path for us (no success at this point).
 4612 
 4613   if (super_check_offset.is_register()) {
 4614     local_jcc(Assembler::equal, *L_success);
 4615     cmpl(super_check_offset.as_register(), sc_offset);
 4616     if (L_failure == &L_fallthrough) {
 4617       local_jcc(Assembler::equal, *L_slow_path);
 4618     } else {
 4619       local_jcc(Assembler::notEqual, *L_failure);
 4620       final_jmp(*L_slow_path);
 4621     }
 4622   } else if (super_check_offset.as_constant() == sc_offset) {
 4623     // Need a slow path; fast failure is impossible.
 4624     if (L_slow_path == &L_fallthrough) {
 4625       local_jcc(Assembler::equal, *L_success);
 4626     } else {
 4627       local_jcc(Assembler::notEqual, *L_slow_path);
 4628       final_jmp(*L_success);
 4629     }
 4630   } else {
 4631     // No slow path; it's a fast decision.
 4632     if (L_failure == &L_fallthrough) {
 4633       local_jcc(Assembler::equal, *L_success);
 4634     } else {
 4635       local_jcc(Assembler::notEqual, *L_failure);
 4636       final_jmp(*L_success);
 4637     }
 4638   }
 4639 
 4640   bind(L_fallthrough);
 4641 
 4642 #undef local_jcc
 4643 #undef final_jmp
 4644 }
 4645 
 4646 
 4647 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4648                                                    Register super_klass,
 4649                                                    Register temp_reg,
 4650                                                    Register temp2_reg,
 4651                                                    Label* L_success,
 4652                                                    Label* L_failure,
 4653                                                    bool set_cond_codes) {
 4654   assert_different_registers(sub_klass, super_klass, temp_reg);
 4655   if (temp2_reg != noreg)
 4656     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4657 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4658 
 4659   Label L_fallthrough;
 4660   int label_nulls = 0;
 4661   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4662   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4663   assert(label_nulls <= 1, "at most one null in the batch");
 4664 
 4665   // a couple of useful fields in sub_klass:
 4666   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4667   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4668   Address secondary_supers_addr(sub_klass, ss_offset);
 4669   Address super_cache_addr(     sub_klass, sc_offset);
 4670 
 4671   // Do a linear scan of the secondary super-klass chain.
 4672   // This code is rarely used, so simplicity is a virtue here.
 4673   // The repne_scan instruction uses fixed registers, which we must spill.
 4674   // Don't worry too much about pre-existing connections with the input regs.
 4675 
 4676   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4677   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4678 
 4679   // Get super_klass value into rax (even if it was in rdi or rcx).
 4680   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4681   if (super_klass != rax) {
 4682     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4683     mov(rax, super_klass);
 4684   }
 4685   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4686   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4687 
 4688 #ifndef PRODUCT
 4689   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4690   ExternalAddress pst_counter_addr((address) pst_counter);
 4691   NOT_LP64(  incrementl(pst_counter_addr) );
 4692   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4693   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4694 #endif //PRODUCT
 4695 
 4696   // We will consult the secondary-super array.
 4697   movptr(rdi, secondary_supers_addr);
 4698   // Load the array length.  (Positive movl does right thing on LP64.)
 4699   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4700   // Skip to start of data.
 4701   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4702 
 4703   // Scan RCX words at [RDI] for an occurrence of RAX.
 4704   // Set NZ/Z based on last compare.
 4705   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4706   // not change flags (only scas instruction which is repeated sets flags).
 4707   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4708 
 4709     testptr(rax,rax); // Set Z = 0
 4710     repne_scan();
 4711 
 4712   // Unspill the temp. registers:
 4713   if (pushed_rdi)  pop(rdi);
 4714   if (pushed_rcx)  pop(rcx);
 4715   if (pushed_rax)  pop(rax);
 4716 
 4717   if (set_cond_codes) {
 4718     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4719     assert(!pushed_rdi, "rdi must be left non-null");
 4720     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4721   }
 4722 
 4723   if (L_failure == &L_fallthrough)
 4724         jccb(Assembler::notEqual, *L_failure);
 4725   else  jcc(Assembler::notEqual, *L_failure);
 4726 
 4727   // Success.  Cache the super we found and proceed in triumph.
 4728   movptr(super_cache_addr, super_klass);
 4729 
 4730   if (L_success != &L_fallthrough) {
 4731     jmp(*L_success);
 4732   }
 4733 
 4734 #undef IS_A_TEMP
 4735 
 4736   bind(L_fallthrough);
 4737 }
 4738 
 4739 #ifdef _LP64
 4740 
 4741 // population_count variant for running without the POPCNT
 4742 // instruction, which was introduced with SSE4.2 in 2008.
 4743 void MacroAssembler::population_count(Register dst, Register src,
 4744                                       Register scratch1, Register scratch2) {
 4745   assert_different_registers(src, scratch1, scratch2);
 4746   if (UsePopCountInstruction) {
 4747     Assembler::popcntq(dst, src);
 4748   } else {
 4749     assert_different_registers(src, scratch1, scratch2);
 4750     assert_different_registers(dst, scratch1, scratch2);
 4751     Label loop, done;
 4752 
 4753     mov(scratch1, src);
 4754     // dst = 0;
 4755     // while(scratch1 != 0) {
 4756     //   dst++;
 4757     //   scratch1 &= (scratch1 - 1);
 4758     // }
 4759     xorl(dst, dst);
 4760     testq(scratch1, scratch1);
 4761     jccb(Assembler::equal, done);
 4762     {
 4763       bind(loop);
 4764       incq(dst);
 4765       movq(scratch2, scratch1);
 4766       decq(scratch2);
 4767       andq(scratch1, scratch2);
 4768       jccb(Assembler::notEqual, loop);
 4769     }
 4770     bind(done);
 4771   }
 4772 }
 4773 
 4774 // Ensure that the inline code and the stub are using the same registers.
 4775 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
 4776 do {                                                                 \
 4777   assert(r_super_klass  == rax, "mismatch");                         \
 4778   assert(r_array_base   == rbx, "mismatch");                         \
 4779   assert(r_array_length == rcx, "mismatch");                         \
 4780   assert(r_array_index  == rdx, "mismatch");                         \
 4781   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
 4782   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
 4783   assert(result         == rdi || result      == noreg, "mismatch"); \
 4784 } while(0)
 4785 
 4786 void MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass,
 4787                                                    Register r_super_klass,
 4788                                                    Register temp1,
 4789                                                    Register temp2,
 4790                                                    Register temp3,
 4791                                                    Register temp4,
 4792                                                    Register result,
 4793                                                    u1 super_klass_slot) {
 4794   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 4795 
 4796   Label L_fallthrough, L_success, L_failure;
 4797 
 4798   BLOCK_COMMENT("lookup_secondary_supers_table {");
 4799 
 4800   const Register
 4801     r_array_index  = temp1,
 4802     r_array_length = temp2,
 4803     r_array_base   = temp3,
 4804     r_bitmap       = temp4;
 4805 
 4806   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 4807 
 4808   xorq(result, result); // = 0
 4809 
 4810   movq(r_bitmap, Address(r_sub_klass, Klass::bitmap_offset()));
 4811   movq(r_array_index, r_bitmap);
 4812 
 4813   // First check the bitmap to see if super_klass might be present. If
 4814   // the bit is zero, we are certain that super_klass is not one of
 4815   // the secondary supers.
 4816   u1 bit = super_klass_slot;
 4817   {
 4818     // NB: If the count in a x86 shift instruction is 0, the flags are
 4819     // not affected, so we do a testq instead.
 4820     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
 4821     if (shift_count != 0) {
 4822       salq(r_array_index, shift_count);
 4823     } else {
 4824       testq(r_array_index, r_array_index);
 4825     }
 4826   }
 4827   // We test the MSB of r_array_index, i.e. its sign bit
 4828   jcc(Assembler::positive, L_failure);
 4829 
 4830   // Get the first array index that can contain super_klass into r_array_index.
 4831   if (bit != 0) {
 4832     population_count(r_array_index, r_array_index, temp2, temp3);
 4833   } else {
 4834     movl(r_array_index, 1);
 4835   }
 4836   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 4837 
 4838   // We will consult the secondary-super array.
 4839   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 4840 
 4841   // We're asserting that the first word in an Array<Klass*> is the
 4842   // length, and the second word is the first word of the data. If
 4843   // that ever changes, r_array_base will have to be adjusted here.
 4844   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 4845   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 4846 
 4847   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 4848   jccb(Assembler::equal, L_success);
 4849 
 4850   // Is there another entry to check? Consult the bitmap.
 4851   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
 4852   jccb(Assembler::carryClear, L_failure);
 4853 
 4854   // Linear probe. Rotate the bitmap so that the next bit to test is
 4855   // in Bit 1.
 4856   if (bit != 0) {
 4857     rorq(r_bitmap, bit);
 4858   }
 4859 
 4860   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 4861   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 4862   // Kills: r_array_length.
 4863   // Returns: result.
 4864   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
 4865   // Result (0/1) is in rdi
 4866   jmpb(L_fallthrough);
 4867 
 4868   bind(L_failure);
 4869   incq(result); // 0 => 1
 4870 
 4871   bind(L_success);
 4872   // result = 0;
 4873 
 4874   bind(L_fallthrough);
 4875   BLOCK_COMMENT("} lookup_secondary_supers_table");
 4876 
 4877   if (VerifySecondarySupers) {
 4878     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 4879                                   temp1, temp2, temp3);
 4880   }
 4881 }
 4882 
 4883 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
 4884                                  Label* L_success, Label* L_failure) {
 4885   Label L_loop, L_fallthrough;
 4886   {
 4887     int label_nulls = 0;
 4888     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
 4889     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
 4890     assert(label_nulls <= 1, "at most one null in the batch");
 4891   }
 4892   bind(L_loop);
 4893   cmpq(value, Address(addr, count, Address::times_8));
 4894   jcc(Assembler::equal, *L_success);
 4895   addl(count, 1);
 4896   cmpl(count, limit);
 4897   jcc(Assembler::less, L_loop);
 4898 
 4899   if (&L_fallthrough != L_failure) {
 4900     jmp(*L_failure);
 4901   }
 4902   bind(L_fallthrough);
 4903 }
 4904 
 4905 // Called by code generated by check_klass_subtype_slow_path
 4906 // above. This is called when there is a collision in the hashed
 4907 // lookup in the secondary supers array.
 4908 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
 4909                                                              Register r_array_base,
 4910                                                              Register r_array_index,
 4911                                                              Register r_bitmap,
 4912                                                              Register temp1,
 4913                                                              Register temp2,
 4914                                                              Label* L_success,
 4915                                                              Label* L_failure) {
 4916   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
 4917 
 4918   const Register
 4919     r_array_length = temp1,
 4920     r_sub_klass    = noreg,
 4921     result         = noreg;
 4922 
 4923   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 4924 
 4925   Label L_fallthrough;
 4926   int label_nulls = 0;
 4927   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4928   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4929   assert(label_nulls <= 1, "at most one null in the batch");
 4930 
 4931   // Load the array length.
 4932   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 4933   // And adjust the array base to point to the data.
 4934   // NB! Effectively increments current slot index by 1.
 4935   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
 4936   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 4937 
 4938   // Linear probe
 4939   Label L_huge;
 4940 
 4941   // The bitmap is full to bursting.
 4942   // Implicit invariant: BITMAP_FULL implies (length > 0)
 4943   assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), "");
 4944   cmpq(r_bitmap, (int32_t)-1); // sign-extends immediate to 64-bit value
 4945   jcc(Assembler::equal, L_huge);
 4946 
 4947   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
 4948   // current slot (at secondary_supers[r_array_index]) has not yet
 4949   // been inspected, and r_array_index may be out of bounds if we
 4950   // wrapped around the end of the array.
 4951 
 4952   { // This is conventional linear probing, but instead of terminating
 4953     // when a null entry is found in the table, we maintain a bitmap
 4954     // in which a 0 indicates missing entries.
 4955     // The check above guarantees there are 0s in the bitmap, so the loop
 4956     // eventually terminates.
 4957 
 4958     xorl(temp2, temp2); // = 0;
 4959 
 4960     Label L_again;
 4961     bind(L_again);
 4962 
 4963     // Check for array wraparound.
 4964     cmpl(r_array_index, r_array_length);
 4965     cmovl(Assembler::greaterEqual, r_array_index, temp2);
 4966 
 4967     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 4968     jcc(Assembler::equal, *L_success);
 4969 
 4970     // If the next bit in bitmap is zero, we're done.
 4971     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
 4972     jcc(Assembler::carryClear, *L_failure);
 4973 
 4974     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
 4975     addl(r_array_index, 1);
 4976 
 4977     jmp(L_again);
 4978   }
 4979 
 4980   { // Degenerate case: more than 64 secondary supers.
 4981     // FIXME: We could do something smarter here, maybe a vectorized
 4982     // comparison or a binary search, but is that worth any added
 4983     // complexity?
 4984     bind(L_huge);
 4985     xorl(r_array_index, r_array_index); // = 0
 4986     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
 4987                 L_success,
 4988                 (&L_fallthrough != L_failure ? L_failure : nullptr));
 4989 
 4990     bind(L_fallthrough);
 4991   }
 4992 }
 4993 
 4994 struct VerifyHelperArguments {
 4995   Klass* _super;
 4996   Klass* _sub;
 4997   intptr_t _linear_result;
 4998   intptr_t _table_result;
 4999 };
 5000 
 5001 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
 5002   Klass::on_secondary_supers_verification_failure(args->_super,
 5003                                                   args->_sub,
 5004                                                   args->_linear_result,
 5005                                                   args->_table_result,
 5006                                                   msg);
 5007 }
 5008 
 5009 // Make sure that the hashed lookup and a linear scan agree.
 5010 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
 5011                                                    Register r_super_klass,
 5012                                                    Register result,
 5013                                                    Register temp1,
 5014                                                    Register temp2,
 5015                                                    Register temp3) {
 5016   const Register
 5017       r_array_index  = temp1,
 5018       r_array_length = temp2,
 5019       r_array_base   = temp3,
 5020       r_bitmap       = noreg;
 5021 
 5022   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 5023 
 5024   BLOCK_COMMENT("verify_secondary_supers_table {");
 5025 
 5026   Label L_success, L_failure, L_check, L_done;
 5027 
 5028   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5029   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5030   // And adjust the array base to point to the data.
 5031   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5032 
 5033   testl(r_array_length, r_array_length); // array_length == 0?
 5034   jcc(Assembler::zero, L_failure);
 5035 
 5036   movl(r_array_index, 0);
 5037   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
 5038   // fall through to L_failure
 5039 
 5040   const Register linear_result = r_array_index; // reuse temp1
 5041 
 5042   bind(L_failure); // not present
 5043   movl(linear_result, 1);
 5044   jmp(L_check);
 5045 
 5046   bind(L_success); // present
 5047   movl(linear_result, 0);
 5048 
 5049   bind(L_check);
 5050   cmpl(linear_result, result);
 5051   jcc(Assembler::equal, L_done);
 5052 
 5053   { // To avoid calling convention issues, build a record on the stack
 5054     // and pass the pointer to that instead.
 5055     push(result);
 5056     push(linear_result);
 5057     push(r_sub_klass);
 5058     push(r_super_klass);
 5059     movptr(c_rarg1, rsp);
 5060     movptr(c_rarg0, (uintptr_t) "mismatch");
 5061     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
 5062     should_not_reach_here();
 5063   }
 5064   bind(L_done);
 5065 
 5066   BLOCK_COMMENT("} verify_secondary_supers_table");
 5067 }
 5068 
 5069 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
 5070 
 5071 #endif // LP64
 5072 
 5073 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 5074   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5075 
 5076   Label L_fallthrough;
 5077   if (L_fast_path == nullptr) {
 5078     L_fast_path = &L_fallthrough;
 5079   } else if (L_slow_path == nullptr) {
 5080     L_slow_path = &L_fallthrough;
 5081   }
 5082 
 5083   // Fast path check: class is fully initialized
 5084   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5085   jcc(Assembler::equal, *L_fast_path);
 5086 
 5087   // Fast path check: current thread is initializer thread
 5088   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 5089   if (L_slow_path == &L_fallthrough) {
 5090     jcc(Assembler::equal, *L_fast_path);
 5091     bind(*L_slow_path);
 5092   } else if (L_fast_path == &L_fallthrough) {
 5093     jcc(Assembler::notEqual, *L_slow_path);
 5094     bind(*L_fast_path);
 5095   } else {
 5096     Unimplemented();
 5097   }
 5098 }
 5099 
 5100 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5101   if (VM_Version::supports_cmov()) {
 5102     cmovl(cc, dst, src);
 5103   } else {
 5104     Label L;
 5105     jccb(negate_condition(cc), L);
 5106     movl(dst, src);
 5107     bind(L);
 5108   }
 5109 }
 5110 
 5111 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5112   if (VM_Version::supports_cmov()) {
 5113     cmovl(cc, dst, src);
 5114   } else {
 5115     Label L;
 5116     jccb(negate_condition(cc), L);
 5117     movl(dst, src);
 5118     bind(L);
 5119   }
 5120 }
 5121 
 5122 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5123   if (!VerifyOops) return;
 5124 
 5125   BLOCK_COMMENT("verify_oop {");
 5126 #ifdef _LP64
 5127   push(rscratch1);
 5128 #endif
 5129   push(rax);                          // save rax
 5130   push(reg);                          // pass register argument
 5131 
 5132   // Pass register number to verify_oop_subroutine
 5133   const char* b = nullptr;
 5134   {
 5135     ResourceMark rm;
 5136     stringStream ss;
 5137     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5138     b = code_string(ss.as_string());
 5139   }
 5140   ExternalAddress buffer((address) b);
 5141   pushptr(buffer.addr(), rscratch1);
 5142 
 5143   // call indirectly to solve generation ordering problem
 5144   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5145   call(rax);
 5146   // Caller pops the arguments (oop, message) and restores rax, r10
 5147   BLOCK_COMMENT("} verify_oop");
 5148 }
 5149 
 5150 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5151   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5152     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5153     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5154     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5155   } else if (VM_Version::supports_avx()) {
 5156     vpcmpeqd(dst, dst, dst, vector_len);
 5157   } else {
 5158     assert(VM_Version::supports_sse2(), "");
 5159     pcmpeqd(dst, dst);
 5160   }
 5161 }
 5162 
 5163 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5164                                          int extra_slot_offset) {
 5165   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5166   int stackElementSize = Interpreter::stackElementSize;
 5167   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5168 #ifdef ASSERT
 5169   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5170   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5171 #endif
 5172   Register             scale_reg    = noreg;
 5173   Address::ScaleFactor scale_factor = Address::no_scale;
 5174   if (arg_slot.is_constant()) {
 5175     offset += arg_slot.as_constant() * stackElementSize;
 5176   } else {
 5177     scale_reg    = arg_slot.as_register();
 5178     scale_factor = Address::times(stackElementSize);
 5179   }
 5180   offset += wordSize;           // return PC is on stack
 5181   return Address(rsp, scale_reg, scale_factor, offset);
 5182 }
 5183 
 5184 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5185   if (!VerifyOops) return;
 5186 
 5187 #ifdef _LP64
 5188   push(rscratch1);
 5189 #endif
 5190   push(rax); // save rax,
 5191   // addr may contain rsp so we will have to adjust it based on the push
 5192   // we just did (and on 64 bit we do two pushes)
 5193   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5194   // stores rax into addr which is backwards of what was intended.
 5195   if (addr.uses(rsp)) {
 5196     lea(rax, addr);
 5197     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5198   } else {
 5199     pushptr(addr);
 5200   }
 5201 
 5202   // Pass register number to verify_oop_subroutine
 5203   const char* b = nullptr;
 5204   {
 5205     ResourceMark rm;
 5206     stringStream ss;
 5207     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5208     b = code_string(ss.as_string());
 5209   }
 5210   ExternalAddress buffer((address) b);
 5211   pushptr(buffer.addr(), rscratch1);
 5212 
 5213   // call indirectly to solve generation ordering problem
 5214   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5215   call(rax);
 5216   // Caller pops the arguments (addr, message) and restores rax, r10.
 5217 }
 5218 
 5219 void MacroAssembler::verify_tlab() {
 5220 #ifdef ASSERT
 5221   if (UseTLAB && VerifyOops) {
 5222     Label next, ok;
 5223     Register t1 = rsi;
 5224     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5225 
 5226     push(t1);
 5227     NOT_LP64(push(thread_reg));
 5228     NOT_LP64(get_thread(thread_reg));
 5229 
 5230     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5231     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5232     jcc(Assembler::aboveEqual, next);
 5233     STOP("assert(top >= start)");
 5234     should_not_reach_here();
 5235 
 5236     bind(next);
 5237     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5238     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5239     jcc(Assembler::aboveEqual, ok);
 5240     STOP("assert(top <= end)");
 5241     should_not_reach_here();
 5242 
 5243     bind(ok);
 5244     NOT_LP64(pop(thread_reg));
 5245     pop(t1);
 5246   }
 5247 #endif
 5248 }
 5249 
 5250 class ControlWord {
 5251  public:
 5252   int32_t _value;
 5253 
 5254   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5255   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5256   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5257   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5258   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5259   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5260   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5261   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5262 
 5263   void print() const {
 5264     // rounding control
 5265     const char* rc;
 5266     switch (rounding_control()) {
 5267       case 0: rc = "round near"; break;
 5268       case 1: rc = "round down"; break;
 5269       case 2: rc = "round up  "; break;
 5270       case 3: rc = "chop      "; break;
 5271       default:
 5272         rc = nullptr; // silence compiler warnings
 5273         fatal("Unknown rounding control: %d", rounding_control());
 5274     };
 5275     // precision control
 5276     const char* pc;
 5277     switch (precision_control()) {
 5278       case 0: pc = "24 bits "; break;
 5279       case 1: pc = "reserved"; break;
 5280       case 2: pc = "53 bits "; break;
 5281       case 3: pc = "64 bits "; break;
 5282       default:
 5283         pc = nullptr; // silence compiler warnings
 5284         fatal("Unknown precision control: %d", precision_control());
 5285     };
 5286     // flags
 5287     char f[9];
 5288     f[0] = ' ';
 5289     f[1] = ' ';
 5290     f[2] = (precision   ()) ? 'P' : 'p';
 5291     f[3] = (underflow   ()) ? 'U' : 'u';
 5292     f[4] = (overflow    ()) ? 'O' : 'o';
 5293     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5294     f[6] = (denormalized()) ? 'D' : 'd';
 5295     f[7] = (invalid     ()) ? 'I' : 'i';
 5296     f[8] = '\x0';
 5297     // output
 5298     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5299   }
 5300 
 5301 };
 5302 
 5303 class StatusWord {
 5304  public:
 5305   int32_t _value;
 5306 
 5307   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5308   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5309   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5310   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5311   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5312   int  top() const                     { return  (_value >> 11) & 7      ; }
 5313   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5314   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5315   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5316   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5317   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5318   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5319   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5320   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5321 
 5322   void print() const {
 5323     // condition codes
 5324     char c[5];
 5325     c[0] = (C3()) ? '3' : '-';
 5326     c[1] = (C2()) ? '2' : '-';
 5327     c[2] = (C1()) ? '1' : '-';
 5328     c[3] = (C0()) ? '0' : '-';
 5329     c[4] = '\x0';
 5330     // flags
 5331     char f[9];
 5332     f[0] = (error_status()) ? 'E' : '-';
 5333     f[1] = (stack_fault ()) ? 'S' : '-';
 5334     f[2] = (precision   ()) ? 'P' : '-';
 5335     f[3] = (underflow   ()) ? 'U' : '-';
 5336     f[4] = (overflow    ()) ? 'O' : '-';
 5337     f[5] = (zero_divide ()) ? 'Z' : '-';
 5338     f[6] = (denormalized()) ? 'D' : '-';
 5339     f[7] = (invalid     ()) ? 'I' : '-';
 5340     f[8] = '\x0';
 5341     // output
 5342     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5343   }
 5344 
 5345 };
 5346 
 5347 class TagWord {
 5348  public:
 5349   int32_t _value;
 5350 
 5351   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5352 
 5353   void print() const {
 5354     printf("%04x", _value & 0xFFFF);
 5355   }
 5356 
 5357 };
 5358 
 5359 class FPU_Register {
 5360  public:
 5361   int32_t _m0;
 5362   int32_t _m1;
 5363   int16_t _ex;
 5364 
 5365   bool is_indefinite() const           {
 5366     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5367   }
 5368 
 5369   void print() const {
 5370     char  sign = (_ex < 0) ? '-' : '+';
 5371     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5372     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5373   };
 5374 
 5375 };
 5376 
 5377 class FPU_State {
 5378  public:
 5379   enum {
 5380     register_size       = 10,
 5381     number_of_registers =  8,
 5382     register_mask       =  7
 5383   };
 5384 
 5385   ControlWord  _control_word;
 5386   StatusWord   _status_word;
 5387   TagWord      _tag_word;
 5388   int32_t      _error_offset;
 5389   int32_t      _error_selector;
 5390   int32_t      _data_offset;
 5391   int32_t      _data_selector;
 5392   int8_t       _register[register_size * number_of_registers];
 5393 
 5394   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5395   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5396 
 5397   const char* tag_as_string(int tag) const {
 5398     switch (tag) {
 5399       case 0: return "valid";
 5400       case 1: return "zero";
 5401       case 2: return "special";
 5402       case 3: return "empty";
 5403     }
 5404     ShouldNotReachHere();
 5405     return nullptr;
 5406   }
 5407 
 5408   void print() const {
 5409     // print computation registers
 5410     { int t = _status_word.top();
 5411       for (int i = 0; i < number_of_registers; i++) {
 5412         int j = (i - t) & register_mask;
 5413         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5414         st(j)->print();
 5415         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5416       }
 5417     }
 5418     printf("\n");
 5419     // print control registers
 5420     printf("ctrl = "); _control_word.print(); printf("\n");
 5421     printf("stat = "); _status_word .print(); printf("\n");
 5422     printf("tags = "); _tag_word    .print(); printf("\n");
 5423   }
 5424 
 5425 };
 5426 
 5427 class Flag_Register {
 5428  public:
 5429   int32_t _value;
 5430 
 5431   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5432   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5433   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5434   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5435   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5436   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5437   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5438 
 5439   void print() const {
 5440     // flags
 5441     char f[8];
 5442     f[0] = (overflow       ()) ? 'O' : '-';
 5443     f[1] = (direction      ()) ? 'D' : '-';
 5444     f[2] = (sign           ()) ? 'S' : '-';
 5445     f[3] = (zero           ()) ? 'Z' : '-';
 5446     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5447     f[5] = (parity         ()) ? 'P' : '-';
 5448     f[6] = (carry          ()) ? 'C' : '-';
 5449     f[7] = '\x0';
 5450     // output
 5451     printf("%08x  flags = %s", _value, f);
 5452   }
 5453 
 5454 };
 5455 
 5456 class IU_Register {
 5457  public:
 5458   int32_t _value;
 5459 
 5460   void print() const {
 5461     printf("%08x  %11d", _value, _value);
 5462   }
 5463 
 5464 };
 5465 
 5466 class IU_State {
 5467  public:
 5468   Flag_Register _eflags;
 5469   IU_Register   _rdi;
 5470   IU_Register   _rsi;
 5471   IU_Register   _rbp;
 5472   IU_Register   _rsp;
 5473   IU_Register   _rbx;
 5474   IU_Register   _rdx;
 5475   IU_Register   _rcx;
 5476   IU_Register   _rax;
 5477 
 5478   void print() const {
 5479     // computation registers
 5480     printf("rax,  = "); _rax.print(); printf("\n");
 5481     printf("rbx,  = "); _rbx.print(); printf("\n");
 5482     printf("rcx  = "); _rcx.print(); printf("\n");
 5483     printf("rdx  = "); _rdx.print(); printf("\n");
 5484     printf("rdi  = "); _rdi.print(); printf("\n");
 5485     printf("rsi  = "); _rsi.print(); printf("\n");
 5486     printf("rbp,  = "); _rbp.print(); printf("\n");
 5487     printf("rsp  = "); _rsp.print(); printf("\n");
 5488     printf("\n");
 5489     // control registers
 5490     printf("flgs = "); _eflags.print(); printf("\n");
 5491   }
 5492 };
 5493 
 5494 
 5495 class CPU_State {
 5496  public:
 5497   FPU_State _fpu_state;
 5498   IU_State  _iu_state;
 5499 
 5500   void print() const {
 5501     printf("--------------------------------------------------\n");
 5502     _iu_state .print();
 5503     printf("\n");
 5504     _fpu_state.print();
 5505     printf("--------------------------------------------------\n");
 5506   }
 5507 
 5508 };
 5509 
 5510 
 5511 static void _print_CPU_state(CPU_State* state) {
 5512   state->print();
 5513 };
 5514 
 5515 
 5516 void MacroAssembler::print_CPU_state() {
 5517   push_CPU_state();
 5518   push(rsp);                // pass CPU state
 5519   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5520   addptr(rsp, wordSize);       // discard argument
 5521   pop_CPU_state();
 5522 }
 5523 
 5524 
 5525 #ifndef _LP64
 5526 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5527   static int counter = 0;
 5528   FPU_State* fs = &state->_fpu_state;
 5529   counter++;
 5530   // For leaf calls, only verify that the top few elements remain empty.
 5531   // We only need 1 empty at the top for C2 code.
 5532   if( stack_depth < 0 ) {
 5533     if( fs->tag_for_st(7) != 3 ) {
 5534       printf("FPR7 not empty\n");
 5535       state->print();
 5536       assert(false, "error");
 5537       return false;
 5538     }
 5539     return true;                // All other stack states do not matter
 5540   }
 5541 
 5542   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5543          "bad FPU control word");
 5544 
 5545   // compute stack depth
 5546   int i = 0;
 5547   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5548   int d = i;
 5549   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5550   // verify findings
 5551   if (i != FPU_State::number_of_registers) {
 5552     // stack not contiguous
 5553     printf("%s: stack not contiguous at ST%d\n", s, i);
 5554     state->print();
 5555     assert(false, "error");
 5556     return false;
 5557   }
 5558   // check if computed stack depth corresponds to expected stack depth
 5559   if (stack_depth < 0) {
 5560     // expected stack depth is -stack_depth or less
 5561     if (d > -stack_depth) {
 5562       // too many elements on the stack
 5563       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5564       state->print();
 5565       assert(false, "error");
 5566       return false;
 5567     }
 5568   } else {
 5569     // expected stack depth is stack_depth
 5570     if (d != stack_depth) {
 5571       // wrong stack depth
 5572       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5573       state->print();
 5574       assert(false, "error");
 5575       return false;
 5576     }
 5577   }
 5578   // everything is cool
 5579   return true;
 5580 }
 5581 
 5582 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5583   if (!VerifyFPU) return;
 5584   push_CPU_state();
 5585   push(rsp);                // pass CPU state
 5586   ExternalAddress msg((address) s);
 5587   // pass message string s
 5588   pushptr(msg.addr(), noreg);
 5589   push(stack_depth);        // pass stack depth
 5590   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5591   addptr(rsp, 3 * wordSize);   // discard arguments
 5592   // check for error
 5593   { Label L;
 5594     testl(rax, rax);
 5595     jcc(Assembler::notZero, L);
 5596     int3();                  // break if error condition
 5597     bind(L);
 5598   }
 5599   pop_CPU_state();
 5600 }
 5601 #endif // _LP64
 5602 
 5603 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5604   // Either restore the MXCSR register after returning from the JNI Call
 5605   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5606   if (VM_Version::supports_sse()) {
 5607     if (RestoreMXCSROnJNICalls) {
 5608       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5609     } else if (CheckJNICalls) {
 5610       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5611     }
 5612   }
 5613   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5614   vzeroupper();
 5615 
 5616 #ifndef _LP64
 5617   // Either restore the x87 floating pointer control word after returning
 5618   // from the JNI call or verify that it wasn't changed.
 5619   if (CheckJNICalls) {
 5620     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5621   }
 5622 #endif // _LP64
 5623 }
 5624 
 5625 // ((OopHandle)result).resolve();
 5626 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5627   assert_different_registers(result, tmp);
 5628 
 5629   // Only 64 bit platforms support GCs that require a tmp register
 5630   // Only IN_HEAP loads require a thread_tmp register
 5631   // OopHandle::resolve is an indirection like jobject.
 5632   access_load_at(T_OBJECT, IN_NATIVE,
 5633                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5634 }
 5635 
 5636 // ((WeakHandle)result).resolve();
 5637 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5638   assert_different_registers(rresult, rtmp);
 5639   Label resolved;
 5640 
 5641   // A null weak handle resolves to null.
 5642   cmpptr(rresult, 0);
 5643   jcc(Assembler::equal, resolved);
 5644 
 5645   // Only 64 bit platforms support GCs that require a tmp register
 5646   // Only IN_HEAP loads require a thread_tmp register
 5647   // WeakHandle::resolve is an indirection like jweak.
 5648   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5649                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5650   bind(resolved);
 5651 }
 5652 
 5653 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5654   // get mirror
 5655   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5656   load_method_holder(mirror, method);
 5657   movptr(mirror, Address(mirror, mirror_offset));
 5658   resolve_oop_handle(mirror, tmp);
 5659 }
 5660 
 5661 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5662   load_method_holder(rresult, rmethod);
 5663   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5664 }
 5665 
 5666 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5667   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5668   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5669   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5670 }
 5671 
 5672 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5673   assert_different_registers(src, tmp);
 5674   assert_different_registers(dst, tmp);
 5675 #ifdef _LP64
 5676   if (UseCompressedClassPointers) {
 5677     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5678     decode_klass_not_null(dst, tmp);
 5679   } else
 5680 #endif
 5681     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5682 }
 5683 
 5684 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5685   assert_different_registers(src, tmp);
 5686   assert_different_registers(dst, tmp);
 5687 #ifdef _LP64
 5688   if (UseCompressedClassPointers) {
 5689     encode_klass_not_null(src, tmp);
 5690     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5691   } else
 5692 #endif
 5693     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5694 }
 5695 
 5696 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5697                                     Register tmp1, Register thread_tmp) {
 5698   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5699   decorators = AccessInternal::decorator_fixup(decorators, type);
 5700   bool as_raw = (decorators & AS_RAW) != 0;
 5701   if (as_raw) {
 5702     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5703   } else {
 5704     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5705   }
 5706 }
 5707 
 5708 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5709                                      Register tmp1, Register tmp2, Register tmp3) {
 5710   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5711   decorators = AccessInternal::decorator_fixup(decorators, type);
 5712   bool as_raw = (decorators & AS_RAW) != 0;
 5713   if (as_raw) {
 5714     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5715   } else {
 5716     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5717   }
 5718 }
 5719 
 5720 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5721                                    Register thread_tmp, DecoratorSet decorators) {
 5722   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5723 }
 5724 
 5725 // Doesn't do verification, generates fixed size code
 5726 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5727                                             Register thread_tmp, DecoratorSet decorators) {
 5728   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5729 }
 5730 
 5731 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5732                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5733   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5734 }
 5735 
 5736 // Used for storing nulls.
 5737 void MacroAssembler::store_heap_oop_null(Address dst) {
 5738   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5739 }
 5740 
 5741 #ifdef _LP64
 5742 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 5743   if (UseCompressedClassPointers) {
 5744     // Store to klass gap in destination
 5745     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 5746   }
 5747 }
 5748 
 5749 #ifdef ASSERT
 5750 void MacroAssembler::verify_heapbase(const char* msg) {
 5751   assert (UseCompressedOops, "should be compressed");
 5752   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5753   if (CheckCompressedOops) {
 5754     Label ok;
 5755     ExternalAddress src2(CompressedOops::ptrs_base_addr());
 5756     const bool is_src2_reachable = reachable(src2);
 5757     if (!is_src2_reachable) {
 5758       push(rscratch1);  // cmpptr trashes rscratch1
 5759     }
 5760     cmpptr(r12_heapbase, src2, rscratch1);
 5761     jcc(Assembler::equal, ok);
 5762     STOP(msg);
 5763     bind(ok);
 5764     if (!is_src2_reachable) {
 5765       pop(rscratch1);
 5766     }
 5767   }
 5768 }
 5769 #endif
 5770 
 5771 // Algorithm must match oop.inline.hpp encode_heap_oop.
 5772 void MacroAssembler::encode_heap_oop(Register r) {
 5773 #ifdef ASSERT
 5774   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 5775 #endif
 5776   verify_oop_msg(r, "broken oop in encode_heap_oop");
 5777   if (CompressedOops::base() == nullptr) {
 5778     if (CompressedOops::shift() != 0) {
 5779       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5780       shrq(r, LogMinObjAlignmentInBytes);
 5781     }
 5782     return;
 5783   }
 5784   testq(r, r);
 5785   cmovq(Assembler::equal, r, r12_heapbase);
 5786   subq(r, r12_heapbase);
 5787   shrq(r, LogMinObjAlignmentInBytes);
 5788 }
 5789 
 5790 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 5791 #ifdef ASSERT
 5792   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 5793   if (CheckCompressedOops) {
 5794     Label ok;
 5795     testq(r, r);
 5796     jcc(Assembler::notEqual, ok);
 5797     STOP("null oop passed to encode_heap_oop_not_null");
 5798     bind(ok);
 5799   }
 5800 #endif
 5801   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 5802   if (CompressedOops::base() != nullptr) {
 5803     subq(r, r12_heapbase);
 5804   }
 5805   if (CompressedOops::shift() != 0) {
 5806     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5807     shrq(r, LogMinObjAlignmentInBytes);
 5808   }
 5809 }
 5810 
 5811 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 5812 #ifdef ASSERT
 5813   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 5814   if (CheckCompressedOops) {
 5815     Label ok;
 5816     testq(src, src);
 5817     jcc(Assembler::notEqual, ok);
 5818     STOP("null oop passed to encode_heap_oop_not_null2");
 5819     bind(ok);
 5820   }
 5821 #endif
 5822   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 5823   if (dst != src) {
 5824     movq(dst, src);
 5825   }
 5826   if (CompressedOops::base() != nullptr) {
 5827     subq(dst, r12_heapbase);
 5828   }
 5829   if (CompressedOops::shift() != 0) {
 5830     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5831     shrq(dst, LogMinObjAlignmentInBytes);
 5832   }
 5833 }
 5834 
 5835 void  MacroAssembler::decode_heap_oop(Register r) {
 5836 #ifdef ASSERT
 5837   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 5838 #endif
 5839   if (CompressedOops::base() == nullptr) {
 5840     if (CompressedOops::shift() != 0) {
 5841       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5842       shlq(r, LogMinObjAlignmentInBytes);
 5843     }
 5844   } else {
 5845     Label done;
 5846     shlq(r, LogMinObjAlignmentInBytes);
 5847     jccb(Assembler::equal, done);
 5848     addq(r, r12_heapbase);
 5849     bind(done);
 5850   }
 5851   verify_oop_msg(r, "broken oop in decode_heap_oop");
 5852 }
 5853 
 5854 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 5855   // Note: it will change flags
 5856   assert (UseCompressedOops, "should only be used for compressed headers");
 5857   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5858   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5859   // vtableStubs also counts instructions in pd_code_size_limit.
 5860   // Also do not verify_oop as this is called by verify_oop.
 5861   if (CompressedOops::shift() != 0) {
 5862     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5863     shlq(r, LogMinObjAlignmentInBytes);
 5864     if (CompressedOops::base() != nullptr) {
 5865       addq(r, r12_heapbase);
 5866     }
 5867   } else {
 5868     assert (CompressedOops::base() == nullptr, "sanity");
 5869   }
 5870 }
 5871 
 5872 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 5873   // Note: it will change flags
 5874   assert (UseCompressedOops, "should only be used for compressed headers");
 5875   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5876   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5877   // vtableStubs also counts instructions in pd_code_size_limit.
 5878   // Also do not verify_oop as this is called by verify_oop.
 5879   if (CompressedOops::shift() != 0) {
 5880     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5881     if (LogMinObjAlignmentInBytes == Address::times_8) {
 5882       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 5883     } else {
 5884       if (dst != src) {
 5885         movq(dst, src);
 5886       }
 5887       shlq(dst, LogMinObjAlignmentInBytes);
 5888       if (CompressedOops::base() != nullptr) {
 5889         addq(dst, r12_heapbase);
 5890       }
 5891     }
 5892   } else {
 5893     assert (CompressedOops::base() == nullptr, "sanity");
 5894     if (dst != src) {
 5895       movq(dst, src);
 5896     }
 5897   }
 5898 }
 5899 
 5900 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 5901   assert_different_registers(r, tmp);
 5902   if (CompressedKlassPointers::base() != nullptr) {
 5903     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5904     subq(r, tmp);
 5905   }
 5906   if (CompressedKlassPointers::shift() != 0) {
 5907     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5908     shrq(r, LogKlassAlignmentInBytes);
 5909   }
 5910 }
 5911 
 5912 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 5913   assert_different_registers(src, dst);
 5914   if (CompressedKlassPointers::base() != nullptr) {
 5915     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 5916     addq(dst, src);
 5917   } else {
 5918     movptr(dst, src);
 5919   }
 5920   if (CompressedKlassPointers::shift() != 0) {
 5921     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5922     shrq(dst, LogKlassAlignmentInBytes);
 5923   }
 5924 }
 5925 
 5926 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 5927   assert_different_registers(r, tmp);
 5928   // Note: it will change flags
 5929   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 5930   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5931   // vtableStubs also counts instructions in pd_code_size_limit.
 5932   // Also do not verify_oop as this is called by verify_oop.
 5933   if (CompressedKlassPointers::shift() != 0) {
 5934     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5935     shlq(r, LogKlassAlignmentInBytes);
 5936   }
 5937   if (CompressedKlassPointers::base() != nullptr) {
 5938     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5939     addq(r, tmp);
 5940   }
 5941 }
 5942 
 5943 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 5944   assert_different_registers(src, dst);
 5945   // Note: it will change flags
 5946   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5947   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5948   // vtableStubs also counts instructions in pd_code_size_limit.
 5949   // Also do not verify_oop as this is called by verify_oop.
 5950 
 5951   if (CompressedKlassPointers::base() == nullptr &&
 5952       CompressedKlassPointers::shift() == 0) {
 5953     // The best case scenario is that there is no base or shift. Then it is already
 5954     // a pointer that needs nothing but a register rename.
 5955     movl(dst, src);
 5956   } else {
 5957     if (CompressedKlassPointers::base() != nullptr) {
 5958       mov64(dst, (int64_t)CompressedKlassPointers::base());
 5959     } else {
 5960       xorq(dst, dst);
 5961     }
 5962     if (CompressedKlassPointers::shift() != 0) {
 5963       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5964       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 5965       leaq(dst, Address(dst, src, Address::times_8, 0));
 5966     } else {
 5967       addq(dst, src);
 5968     }
 5969   }
 5970 }
 5971 
 5972 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 5973   assert (UseCompressedOops, "should only be used for compressed headers");
 5974   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5975   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5976   int oop_index = oop_recorder()->find_index(obj);
 5977   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5978   mov_narrow_oop(dst, oop_index, rspec);
 5979 }
 5980 
 5981 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 5982   assert (UseCompressedOops, "should only be used for compressed headers");
 5983   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5984   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5985   int oop_index = oop_recorder()->find_index(obj);
 5986   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5987   mov_narrow_oop(dst, oop_index, rspec);
 5988 }
 5989 
 5990 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 5991   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5992   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5993   int klass_index = oop_recorder()->find_index(k);
 5994   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5995   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5996 }
 5997 
 5998 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 5999   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6000   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6001   int klass_index = oop_recorder()->find_index(k);
 6002   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6003   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6004 }
 6005 
 6006 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6007   assert (UseCompressedOops, "should only be used for compressed headers");
 6008   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6009   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6010   int oop_index = oop_recorder()->find_index(obj);
 6011   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6012   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6013 }
 6014 
 6015 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6016   assert (UseCompressedOops, "should only be used for compressed headers");
 6017   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6018   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6019   int oop_index = oop_recorder()->find_index(obj);
 6020   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6021   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6022 }
 6023 
 6024 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6025   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6026   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6027   int klass_index = oop_recorder()->find_index(k);
 6028   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6029   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6030 }
 6031 
 6032 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6033   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6034   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6035   int klass_index = oop_recorder()->find_index(k);
 6036   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6037   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6038 }
 6039 
 6040 void MacroAssembler::reinit_heapbase() {
 6041   if (UseCompressedOops) {
 6042     if (Universe::heap() != nullptr) {
 6043       if (CompressedOops::base() == nullptr) {
 6044         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6045       } else {
 6046         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 6047       }
 6048     } else {
 6049       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 6050     }
 6051   }
 6052 }
 6053 
 6054 #endif // _LP64
 6055 
 6056 #if COMPILER2_OR_JVMCI
 6057 
 6058 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6059 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6060   // cnt - number of qwords (8-byte words).
 6061   // base - start address, qword aligned.
 6062   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6063   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6064   if (use64byteVector) {
 6065     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 6066   } else if (MaxVectorSize >= 32) {
 6067     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
 6068   } else {
 6069     pxor(xtmp, xtmp);
 6070   }
 6071   jmp(L_zero_64_bytes);
 6072 
 6073   BIND(L_loop);
 6074   if (MaxVectorSize >= 32) {
 6075     fill64(base, 0, xtmp, use64byteVector);
 6076   } else {
 6077     movdqu(Address(base,  0), xtmp);
 6078     movdqu(Address(base, 16), xtmp);
 6079     movdqu(Address(base, 32), xtmp);
 6080     movdqu(Address(base, 48), xtmp);
 6081   }
 6082   addptr(base, 64);
 6083 
 6084   BIND(L_zero_64_bytes);
 6085   subptr(cnt, 8);
 6086   jccb(Assembler::greaterEqual, L_loop);
 6087 
 6088   // Copy trailing 64 bytes
 6089   if (use64byteVector) {
 6090     addptr(cnt, 8);
 6091     jccb(Assembler::equal, L_end);
 6092     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 6093     jmp(L_end);
 6094   } else {
 6095     addptr(cnt, 4);
 6096     jccb(Assembler::less, L_tail);
 6097     if (MaxVectorSize >= 32) {
 6098       vmovdqu(Address(base, 0), xtmp);
 6099     } else {
 6100       movdqu(Address(base,  0), xtmp);
 6101       movdqu(Address(base, 16), xtmp);
 6102     }
 6103   }
 6104   addptr(base, 32);
 6105   subptr(cnt, 4);
 6106 
 6107   BIND(L_tail);
 6108   addptr(cnt, 4);
 6109   jccb(Assembler::lessEqual, L_end);
 6110   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6111     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 6112   } else {
 6113     decrement(cnt);
 6114 
 6115     BIND(L_sloop);
 6116     movq(Address(base, 0), xtmp);
 6117     addptr(base, 8);
 6118     decrement(cnt);
 6119     jccb(Assembler::greaterEqual, L_sloop);
 6120   }
 6121   BIND(L_end);
 6122 }
 6123 
 6124 // Clearing constant sized memory using YMM/ZMM registers.
 6125 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6126   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6127   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6128 
 6129   int vector64_count = (cnt & (~0x7)) >> 3;
 6130   cnt = cnt & 0x7;
 6131   const int fill64_per_loop = 4;
 6132   const int max_unrolled_fill64 = 8;
 6133 
 6134   // 64 byte initialization loop.
 6135   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6136   int start64 = 0;
 6137   if (vector64_count > max_unrolled_fill64) {
 6138     Label LOOP;
 6139     Register index = rtmp;
 6140 
 6141     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6142 
 6143     movl(index, 0);
 6144     BIND(LOOP);
 6145     for (int i = 0; i < fill64_per_loop; i++) {
 6146       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6147     }
 6148     addl(index, fill64_per_loop * 64);
 6149     cmpl(index, start64 * 64);
 6150     jccb(Assembler::less, LOOP);
 6151   }
 6152   for (int i = start64; i < vector64_count; i++) {
 6153     fill64(base, i * 64, xtmp, use64byteVector);
 6154   }
 6155 
 6156   // Clear remaining 64 byte tail.
 6157   int disp = vector64_count * 64;
 6158   if (cnt) {
 6159     switch (cnt) {
 6160       case 1:
 6161         movq(Address(base, disp), xtmp);
 6162         break;
 6163       case 2:
 6164         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6165         break;
 6166       case 3:
 6167         movl(rtmp, 0x7);
 6168         kmovwl(mask, rtmp);
 6169         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6170         break;
 6171       case 4:
 6172         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6173         break;
 6174       case 5:
 6175         if (use64byteVector) {
 6176           movl(rtmp, 0x1F);
 6177           kmovwl(mask, rtmp);
 6178           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6179         } else {
 6180           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6181           movq(Address(base, disp + 32), xtmp);
 6182         }
 6183         break;
 6184       case 6:
 6185         if (use64byteVector) {
 6186           movl(rtmp, 0x3F);
 6187           kmovwl(mask, rtmp);
 6188           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6189         } else {
 6190           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6191           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6192         }
 6193         break;
 6194       case 7:
 6195         if (use64byteVector) {
 6196           movl(rtmp, 0x7F);
 6197           kmovwl(mask, rtmp);
 6198           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6199         } else {
 6200           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6201           movl(rtmp, 0x7);
 6202           kmovwl(mask, rtmp);
 6203           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6204         }
 6205         break;
 6206       default:
 6207         fatal("Unexpected length : %d\n",cnt);
 6208         break;
 6209     }
 6210   }
 6211 }
 6212 
 6213 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 6214                                bool is_large, KRegister mask) {
 6215   // cnt      - number of qwords (8-byte words).
 6216   // base     - start address, qword aligned.
 6217   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6218   assert(base==rdi, "base register must be edi for rep stos");
 6219   assert(tmp==rax,   "tmp register must be eax for rep stos");
 6220   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6221   assert(InitArrayShortSize % BytesPerLong == 0,
 6222     "InitArrayShortSize should be the multiple of BytesPerLong");
 6223 
 6224   Label DONE;
 6225   if (!is_large || !UseXMMForObjInit) {
 6226     xorptr(tmp, tmp);
 6227   }
 6228 
 6229   if (!is_large) {
 6230     Label LOOP, LONG;
 6231     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6232     jccb(Assembler::greater, LONG);
 6233 
 6234     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6235 
 6236     decrement(cnt);
 6237     jccb(Assembler::negative, DONE); // Zero length
 6238 
 6239     // Use individual pointer-sized stores for small counts:
 6240     BIND(LOOP);
 6241     movptr(Address(base, cnt, Address::times_ptr), tmp);
 6242     decrement(cnt);
 6243     jccb(Assembler::greaterEqual, LOOP);
 6244     jmpb(DONE);
 6245 
 6246     BIND(LONG);
 6247   }
 6248 
 6249   // Use longer rep-prefixed ops for non-small counts:
 6250   if (UseFastStosb) {
 6251     shlptr(cnt, 3); // convert to number of bytes
 6252     rep_stosb();
 6253   } else if (UseXMMForObjInit) {
 6254     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 6255   } else {
 6256     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6257     rep_stos();
 6258   }
 6259 
 6260   BIND(DONE);
 6261 }
 6262 
 6263 #endif //COMPILER2_OR_JVMCI
 6264 
 6265 
 6266 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6267                                    Register to, Register value, Register count,
 6268                                    Register rtmp, XMMRegister xtmp) {
 6269   ShortBranchVerifier sbv(this);
 6270   assert_different_registers(to, value, count, rtmp);
 6271   Label L_exit;
 6272   Label L_fill_2_bytes, L_fill_4_bytes;
 6273 
 6274 #if defined(COMPILER2) && defined(_LP64)
 6275   if(MaxVectorSize >=32 &&
 6276      VM_Version::supports_avx512vlbw() &&
 6277      VM_Version::supports_bmi2()) {
 6278     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6279     return;
 6280   }
 6281 #endif
 6282 
 6283   int shift = -1;
 6284   switch (t) {
 6285     case T_BYTE:
 6286       shift = 2;
 6287       break;
 6288     case T_SHORT:
 6289       shift = 1;
 6290       break;
 6291     case T_INT:
 6292       shift = 0;
 6293       break;
 6294     default: ShouldNotReachHere();
 6295   }
 6296 
 6297   if (t == T_BYTE) {
 6298     andl(value, 0xff);
 6299     movl(rtmp, value);
 6300     shll(rtmp, 8);
 6301     orl(value, rtmp);
 6302   }
 6303   if (t == T_SHORT) {
 6304     andl(value, 0xffff);
 6305   }
 6306   if (t == T_BYTE || t == T_SHORT) {
 6307     movl(rtmp, value);
 6308     shll(rtmp, 16);
 6309     orl(value, rtmp);
 6310   }
 6311 
 6312   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6313   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6314   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6315     Label L_skip_align2;
 6316     // align source address at 4 bytes address boundary
 6317     if (t == T_BYTE) {
 6318       Label L_skip_align1;
 6319       // One byte misalignment happens only for byte arrays
 6320       testptr(to, 1);
 6321       jccb(Assembler::zero, L_skip_align1);
 6322       movb(Address(to, 0), value);
 6323       increment(to);
 6324       decrement(count);
 6325       BIND(L_skip_align1);
 6326     }
 6327     // Two bytes misalignment happens only for byte and short (char) arrays
 6328     testptr(to, 2);
 6329     jccb(Assembler::zero, L_skip_align2);
 6330     movw(Address(to, 0), value);
 6331     addptr(to, 2);
 6332     subptr(count, 1<<(shift-1));
 6333     BIND(L_skip_align2);
 6334   }
 6335   if (UseSSE < 2) {
 6336     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6337     // Fill 32-byte chunks
 6338     subptr(count, 8 << shift);
 6339     jcc(Assembler::less, L_check_fill_8_bytes);
 6340     align(16);
 6341 
 6342     BIND(L_fill_32_bytes_loop);
 6343 
 6344     for (int i = 0; i < 32; i += 4) {
 6345       movl(Address(to, i), value);
 6346     }
 6347 
 6348     addptr(to, 32);
 6349     subptr(count, 8 << shift);
 6350     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6351     BIND(L_check_fill_8_bytes);
 6352     addptr(count, 8 << shift);
 6353     jccb(Assembler::zero, L_exit);
 6354     jmpb(L_fill_8_bytes);
 6355 
 6356     //
 6357     // length is too short, just fill qwords
 6358     //
 6359     BIND(L_fill_8_bytes_loop);
 6360     movl(Address(to, 0), value);
 6361     movl(Address(to, 4), value);
 6362     addptr(to, 8);
 6363     BIND(L_fill_8_bytes);
 6364     subptr(count, 1 << (shift + 1));
 6365     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6366     // fall through to fill 4 bytes
 6367   } else {
 6368     Label L_fill_32_bytes;
 6369     if (!UseUnalignedLoadStores) {
 6370       // align to 8 bytes, we know we are 4 byte aligned to start
 6371       testptr(to, 4);
 6372       jccb(Assembler::zero, L_fill_32_bytes);
 6373       movl(Address(to, 0), value);
 6374       addptr(to, 4);
 6375       subptr(count, 1<<shift);
 6376     }
 6377     BIND(L_fill_32_bytes);
 6378     {
 6379       assert( UseSSE >= 2, "supported cpu only" );
 6380       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6381       movdl(xtmp, value);
 6382       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6383         Label L_check_fill_32_bytes;
 6384         if (UseAVX > 2) {
 6385           // Fill 64-byte chunks
 6386           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6387 
 6388           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6389           cmpptr(count, VM_Version::avx3_threshold());
 6390           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6391 
 6392           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6393 
 6394           subptr(count, 16 << shift);
 6395           jccb(Assembler::less, L_check_fill_32_bytes);
 6396           align(16);
 6397 
 6398           BIND(L_fill_64_bytes_loop_avx3);
 6399           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6400           addptr(to, 64);
 6401           subptr(count, 16 << shift);
 6402           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6403           jmpb(L_check_fill_32_bytes);
 6404 
 6405           BIND(L_check_fill_64_bytes_avx2);
 6406         }
 6407         // Fill 64-byte chunks
 6408         Label L_fill_64_bytes_loop;
 6409         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6410 
 6411         subptr(count, 16 << shift);
 6412         jcc(Assembler::less, L_check_fill_32_bytes);
 6413         align(16);
 6414 
 6415         BIND(L_fill_64_bytes_loop);
 6416         vmovdqu(Address(to, 0), xtmp);
 6417         vmovdqu(Address(to, 32), xtmp);
 6418         addptr(to, 64);
 6419         subptr(count, 16 << shift);
 6420         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6421 
 6422         BIND(L_check_fill_32_bytes);
 6423         addptr(count, 8 << shift);
 6424         jccb(Assembler::less, L_check_fill_8_bytes);
 6425         vmovdqu(Address(to, 0), xtmp);
 6426         addptr(to, 32);
 6427         subptr(count, 8 << shift);
 6428 
 6429         BIND(L_check_fill_8_bytes);
 6430         // clean upper bits of YMM registers
 6431         movdl(xtmp, value);
 6432         pshufd(xtmp, xtmp, 0);
 6433       } else {
 6434         // Fill 32-byte chunks
 6435         pshufd(xtmp, xtmp, 0);
 6436 
 6437         subptr(count, 8 << shift);
 6438         jcc(Assembler::less, L_check_fill_8_bytes);
 6439         align(16);
 6440 
 6441         BIND(L_fill_32_bytes_loop);
 6442 
 6443         if (UseUnalignedLoadStores) {
 6444           movdqu(Address(to, 0), xtmp);
 6445           movdqu(Address(to, 16), xtmp);
 6446         } else {
 6447           movq(Address(to, 0), xtmp);
 6448           movq(Address(to, 8), xtmp);
 6449           movq(Address(to, 16), xtmp);
 6450           movq(Address(to, 24), xtmp);
 6451         }
 6452 
 6453         addptr(to, 32);
 6454         subptr(count, 8 << shift);
 6455         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6456 
 6457         BIND(L_check_fill_8_bytes);
 6458       }
 6459       addptr(count, 8 << shift);
 6460       jccb(Assembler::zero, L_exit);
 6461       jmpb(L_fill_8_bytes);
 6462 
 6463       //
 6464       // length is too short, just fill qwords
 6465       //
 6466       BIND(L_fill_8_bytes_loop);
 6467       movq(Address(to, 0), xtmp);
 6468       addptr(to, 8);
 6469       BIND(L_fill_8_bytes);
 6470       subptr(count, 1 << (shift + 1));
 6471       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6472     }
 6473   }
 6474   // fill trailing 4 bytes
 6475   BIND(L_fill_4_bytes);
 6476   testl(count, 1<<shift);
 6477   jccb(Assembler::zero, L_fill_2_bytes);
 6478   movl(Address(to, 0), value);
 6479   if (t == T_BYTE || t == T_SHORT) {
 6480     Label L_fill_byte;
 6481     addptr(to, 4);
 6482     BIND(L_fill_2_bytes);
 6483     // fill trailing 2 bytes
 6484     testl(count, 1<<(shift-1));
 6485     jccb(Assembler::zero, L_fill_byte);
 6486     movw(Address(to, 0), value);
 6487     if (t == T_BYTE) {
 6488       addptr(to, 2);
 6489       BIND(L_fill_byte);
 6490       // fill trailing byte
 6491       testl(count, 1);
 6492       jccb(Assembler::zero, L_exit);
 6493       movb(Address(to, 0), value);
 6494     } else {
 6495       BIND(L_fill_byte);
 6496     }
 6497   } else {
 6498     BIND(L_fill_2_bytes);
 6499   }
 6500   BIND(L_exit);
 6501 }
 6502 
 6503 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6504   switch(type) {
 6505     case T_BYTE:
 6506     case T_BOOLEAN:
 6507       evpbroadcastb(dst, src, vector_len);
 6508       break;
 6509     case T_SHORT:
 6510     case T_CHAR:
 6511       evpbroadcastw(dst, src, vector_len);
 6512       break;
 6513     case T_INT:
 6514     case T_FLOAT:
 6515       evpbroadcastd(dst, src, vector_len);
 6516       break;
 6517     case T_LONG:
 6518     case T_DOUBLE:
 6519       evpbroadcastq(dst, src, vector_len);
 6520       break;
 6521     default:
 6522       fatal("Unhandled type : %s", type2name(type));
 6523       break;
 6524   }
 6525 }
 6526 
 6527 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6528    //@IntrinsicCandidate
 6529    //private static int implEncodeISOArray(byte[] sa, int sp,
 6530    //byte[] da, int dp, int len) {
 6531    //  int i = 0;
 6532    //  for (; i < len; i++) {
 6533    //    char c = StringUTF16.getChar(sa, sp++);
 6534    //    if (c > '\u00FF')
 6535    //      break;
 6536    //    da[dp++] = (byte)c;
 6537    //  }
 6538    //  return i;
 6539    //}
 6540    //
 6541    //@IntrinsicCandidate
 6542    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6543    //    byte[] da, int dp, int len) {
 6544    //  int i = 0;
 6545    //  for (; i < len; i++) {
 6546    //    char c = sa[sp++];
 6547    //    if (c >= '\u0080')
 6548    //      break;
 6549    //    da[dp++] = (byte)c;
 6550    //  }
 6551    //  return i;
 6552    //}
 6553 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6554   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6555   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6556   Register tmp5, Register result, bool ascii) {
 6557 
 6558   // rsi: src
 6559   // rdi: dst
 6560   // rdx: len
 6561   // rcx: tmp5
 6562   // rax: result
 6563   ShortBranchVerifier sbv(this);
 6564   assert_different_registers(src, dst, len, tmp5, result);
 6565   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6566 
 6567   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6568   int short_mask = ascii ? 0xff80 : 0xff00;
 6569 
 6570   // set result
 6571   xorl(result, result);
 6572   // check for zero length
 6573   testl(len, len);
 6574   jcc(Assembler::zero, L_done);
 6575 
 6576   movl(result, len);
 6577 
 6578   // Setup pointers
 6579   lea(src, Address(src, len, Address::times_2)); // char[]
 6580   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6581   negptr(len);
 6582 
 6583   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6584     Label L_copy_8_chars, L_copy_8_chars_exit;
 6585     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6586 
 6587     if (UseAVX >= 2) {
 6588       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6589       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6590       movdl(tmp1Reg, tmp5);
 6591       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 6592       jmp(L_chars_32_check);
 6593 
 6594       bind(L_copy_32_chars);
 6595       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 6596       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 6597       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6598       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6599       jccb(Assembler::notZero, L_copy_32_chars_exit);
 6600       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6601       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 6602       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 6603 
 6604       bind(L_chars_32_check);
 6605       addptr(len, 32);
 6606       jcc(Assembler::lessEqual, L_copy_32_chars);
 6607 
 6608       bind(L_copy_32_chars_exit);
 6609       subptr(len, 16);
 6610       jccb(Assembler::greater, L_copy_16_chars_exit);
 6611 
 6612     } else if (UseSSE42Intrinsics) {
 6613       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6614       movdl(tmp1Reg, tmp5);
 6615       pshufd(tmp1Reg, tmp1Reg, 0);
 6616       jmpb(L_chars_16_check);
 6617     }
 6618 
 6619     bind(L_copy_16_chars);
 6620     if (UseAVX >= 2) {
 6621       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 6622       vptest(tmp2Reg, tmp1Reg);
 6623       jcc(Assembler::notZero, L_copy_16_chars_exit);
 6624       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 6625       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 6626     } else {
 6627       if (UseAVX > 0) {
 6628         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6629         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6630         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 6631       } else {
 6632         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6633         por(tmp2Reg, tmp3Reg);
 6634         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6635         por(tmp2Reg, tmp4Reg);
 6636       }
 6637       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6638       jccb(Assembler::notZero, L_copy_16_chars_exit);
 6639       packuswb(tmp3Reg, tmp4Reg);
 6640     }
 6641     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 6642 
 6643     bind(L_chars_16_check);
 6644     addptr(len, 16);
 6645     jcc(Assembler::lessEqual, L_copy_16_chars);
 6646 
 6647     bind(L_copy_16_chars_exit);
 6648     if (UseAVX >= 2) {
 6649       // clean upper bits of YMM registers
 6650       vpxor(tmp2Reg, tmp2Reg);
 6651       vpxor(tmp3Reg, tmp3Reg);
 6652       vpxor(tmp4Reg, tmp4Reg);
 6653       movdl(tmp1Reg, tmp5);
 6654       pshufd(tmp1Reg, tmp1Reg, 0);
 6655     }
 6656     subptr(len, 8);
 6657     jccb(Assembler::greater, L_copy_8_chars_exit);
 6658 
 6659     bind(L_copy_8_chars);
 6660     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 6661     ptest(tmp3Reg, tmp1Reg);
 6662     jccb(Assembler::notZero, L_copy_8_chars_exit);
 6663     packuswb(tmp3Reg, tmp1Reg);
 6664     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 6665     addptr(len, 8);
 6666     jccb(Assembler::lessEqual, L_copy_8_chars);
 6667 
 6668     bind(L_copy_8_chars_exit);
 6669     subptr(len, 8);
 6670     jccb(Assembler::zero, L_done);
 6671   }
 6672 
 6673   bind(L_copy_1_char);
 6674   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 6675   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 6676   jccb(Assembler::notZero, L_copy_1_char_exit);
 6677   movb(Address(dst, len, Address::times_1, 0), tmp5);
 6678   addptr(len, 1);
 6679   jccb(Assembler::less, L_copy_1_char);
 6680 
 6681   bind(L_copy_1_char_exit);
 6682   addptr(result, len); // len is negative count of not processed elements
 6683 
 6684   bind(L_done);
 6685 }
 6686 
 6687 #ifdef _LP64
 6688 /**
 6689  * Helper for multiply_to_len().
 6690  */
 6691 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 6692   addq(dest_lo, src1);
 6693   adcq(dest_hi, 0);
 6694   addq(dest_lo, src2);
 6695   adcq(dest_hi, 0);
 6696 }
 6697 
 6698 /**
 6699  * Multiply 64 bit by 64 bit first loop.
 6700  */
 6701 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 6702                                            Register y, Register y_idx, Register z,
 6703                                            Register carry, Register product,
 6704                                            Register idx, Register kdx) {
 6705   //
 6706   //  jlong carry, x[], y[], z[];
 6707   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 6708   //    huge_128 product = y[idx] * x[xstart] + carry;
 6709   //    z[kdx] = (jlong)product;
 6710   //    carry  = (jlong)(product >>> 64);
 6711   //  }
 6712   //  z[xstart] = carry;
 6713   //
 6714 
 6715   Label L_first_loop, L_first_loop_exit;
 6716   Label L_one_x, L_one_y, L_multiply;
 6717 
 6718   decrementl(xstart);
 6719   jcc(Assembler::negative, L_one_x);
 6720 
 6721   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 6722   rorq(x_xstart, 32); // convert big-endian to little-endian
 6723 
 6724   bind(L_first_loop);
 6725   decrementl(idx);
 6726   jcc(Assembler::negative, L_first_loop_exit);
 6727   decrementl(idx);
 6728   jcc(Assembler::negative, L_one_y);
 6729   movq(y_idx, Address(y, idx, Address::times_4,  0));
 6730   rorq(y_idx, 32); // convert big-endian to little-endian
 6731   bind(L_multiply);
 6732   movq(product, x_xstart);
 6733   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 6734   addq(product, carry);
 6735   adcq(rdx, 0);
 6736   subl(kdx, 2);
 6737   movl(Address(z, kdx, Address::times_4,  4), product);
 6738   shrq(product, 32);
 6739   movl(Address(z, kdx, Address::times_4,  0), product);
 6740   movq(carry, rdx);
 6741   jmp(L_first_loop);
 6742 
 6743   bind(L_one_y);
 6744   movl(y_idx, Address(y,  0));
 6745   jmp(L_multiply);
 6746 
 6747   bind(L_one_x);
 6748   movl(x_xstart, Address(x,  0));
 6749   jmp(L_first_loop);
 6750 
 6751   bind(L_first_loop_exit);
 6752 }
 6753 
 6754 /**
 6755  * Multiply 64 bit by 64 bit and add 128 bit.
 6756  */
 6757 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 6758                                             Register yz_idx, Register idx,
 6759                                             Register carry, Register product, int offset) {
 6760   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 6761   //     z[kdx] = (jlong)product;
 6762 
 6763   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 6764   rorq(yz_idx, 32); // convert big-endian to little-endian
 6765   movq(product, x_xstart);
 6766   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 6767   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 6768   rorq(yz_idx, 32); // convert big-endian to little-endian
 6769 
 6770   add2_with_carry(rdx, product, carry, yz_idx);
 6771 
 6772   movl(Address(z, idx, Address::times_4,  offset+4), product);
 6773   shrq(product, 32);
 6774   movl(Address(z, idx, Address::times_4,  offset), product);
 6775 
 6776 }
 6777 
 6778 /**
 6779  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 6780  */
 6781 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 6782                                              Register yz_idx, Register idx, Register jdx,
 6783                                              Register carry, Register product,
 6784                                              Register carry2) {
 6785   //   jlong carry, x[], y[], z[];
 6786   //   int kdx = ystart+1;
 6787   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6788   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 6789   //     z[kdx+idx+1] = (jlong)product;
 6790   //     jlong carry2  = (jlong)(product >>> 64);
 6791   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 6792   //     z[kdx+idx] = (jlong)product;
 6793   //     carry  = (jlong)(product >>> 64);
 6794   //   }
 6795   //   idx += 2;
 6796   //   if (idx > 0) {
 6797   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 6798   //     z[kdx+idx] = (jlong)product;
 6799   //     carry  = (jlong)(product >>> 64);
 6800   //   }
 6801   //
 6802 
 6803   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6804 
 6805   movl(jdx, idx);
 6806   andl(jdx, 0xFFFFFFFC);
 6807   shrl(jdx, 2);
 6808 
 6809   bind(L_third_loop);
 6810   subl(jdx, 1);
 6811   jcc(Assembler::negative, L_third_loop_exit);
 6812   subl(idx, 4);
 6813 
 6814   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 6815   movq(carry2, rdx);
 6816 
 6817   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 6818   movq(carry, rdx);
 6819   jmp(L_third_loop);
 6820 
 6821   bind (L_third_loop_exit);
 6822 
 6823   andl (idx, 0x3);
 6824   jcc(Assembler::zero, L_post_third_loop_done);
 6825 
 6826   Label L_check_1;
 6827   subl(idx, 2);
 6828   jcc(Assembler::negative, L_check_1);
 6829 
 6830   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 6831   movq(carry, rdx);
 6832 
 6833   bind (L_check_1);
 6834   addl (idx, 0x2);
 6835   andl (idx, 0x1);
 6836   subl(idx, 1);
 6837   jcc(Assembler::negative, L_post_third_loop_done);
 6838 
 6839   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 6840   movq(product, x_xstart);
 6841   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 6842   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 6843 
 6844   add2_with_carry(rdx, product, yz_idx, carry);
 6845 
 6846   movl(Address(z, idx, Address::times_4,  0), product);
 6847   shrq(product, 32);
 6848 
 6849   shlq(rdx, 32);
 6850   orq(product, rdx);
 6851   movq(carry, product);
 6852 
 6853   bind(L_post_third_loop_done);
 6854 }
 6855 
 6856 /**
 6857  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 6858  *
 6859  */
 6860 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 6861                                                   Register carry, Register carry2,
 6862                                                   Register idx, Register jdx,
 6863                                                   Register yz_idx1, Register yz_idx2,
 6864                                                   Register tmp, Register tmp3, Register tmp4) {
 6865   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 6866 
 6867   //   jlong carry, x[], y[], z[];
 6868   //   int kdx = ystart+1;
 6869   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6870   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 6871   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 6872   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 6873   //     carry  = (jlong)(tmp4 >>> 64);
 6874   //     z[kdx+idx+1] = (jlong)tmp3;
 6875   //     z[kdx+idx] = (jlong)tmp4;
 6876   //   }
 6877   //   idx += 2;
 6878   //   if (idx > 0) {
 6879   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 6880   //     z[kdx+idx] = (jlong)yz_idx1;
 6881   //     carry  = (jlong)(yz_idx1 >>> 64);
 6882   //   }
 6883   //
 6884 
 6885   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6886 
 6887   movl(jdx, idx);
 6888   andl(jdx, 0xFFFFFFFC);
 6889   shrl(jdx, 2);
 6890 
 6891   bind(L_third_loop);
 6892   subl(jdx, 1);
 6893   jcc(Assembler::negative, L_third_loop_exit);
 6894   subl(idx, 4);
 6895 
 6896   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 6897   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 6898   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 6899   rorxq(yz_idx2, yz_idx2, 32);
 6900 
 6901   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 6902   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 6903 
 6904   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 6905   rorxq(yz_idx1, yz_idx1, 32);
 6906   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6907   rorxq(yz_idx2, yz_idx2, 32);
 6908 
 6909   if (VM_Version::supports_adx()) {
 6910     adcxq(tmp3, carry);
 6911     adoxq(tmp3, yz_idx1);
 6912 
 6913     adcxq(tmp4, tmp);
 6914     adoxq(tmp4, yz_idx2);
 6915 
 6916     movl(carry, 0); // does not affect flags
 6917     adcxq(carry2, carry);
 6918     adoxq(carry2, carry);
 6919   } else {
 6920     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 6921     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 6922   }
 6923   movq(carry, carry2);
 6924 
 6925   movl(Address(z, idx, Address::times_4, 12), tmp3);
 6926   shrq(tmp3, 32);
 6927   movl(Address(z, idx, Address::times_4,  8), tmp3);
 6928 
 6929   movl(Address(z, idx, Address::times_4,  4), tmp4);
 6930   shrq(tmp4, 32);
 6931   movl(Address(z, idx, Address::times_4,  0), tmp4);
 6932 
 6933   jmp(L_third_loop);
 6934 
 6935   bind (L_third_loop_exit);
 6936 
 6937   andl (idx, 0x3);
 6938   jcc(Assembler::zero, L_post_third_loop_done);
 6939 
 6940   Label L_check_1;
 6941   subl(idx, 2);
 6942   jcc(Assembler::negative, L_check_1);
 6943 
 6944   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 6945   rorxq(yz_idx1, yz_idx1, 32);
 6946   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 6947   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6948   rorxq(yz_idx2, yz_idx2, 32);
 6949 
 6950   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 6951 
 6952   movl(Address(z, idx, Address::times_4,  4), tmp3);
 6953   shrq(tmp3, 32);
 6954   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6955   movq(carry, tmp4);
 6956 
 6957   bind (L_check_1);
 6958   addl (idx, 0x2);
 6959   andl (idx, 0x1);
 6960   subl(idx, 1);
 6961   jcc(Assembler::negative, L_post_third_loop_done);
 6962   movl(tmp4, Address(y, idx, Address::times_4,  0));
 6963   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 6964   movl(tmp4, Address(z, idx, Address::times_4,  0));
 6965 
 6966   add2_with_carry(carry2, tmp3, tmp4, carry);
 6967 
 6968   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6969   shrq(tmp3, 32);
 6970 
 6971   shlq(carry2, 32);
 6972   orq(tmp3, carry2);
 6973   movq(carry, tmp3);
 6974 
 6975   bind(L_post_third_loop_done);
 6976 }
 6977 
 6978 /**
 6979  * Code for BigInteger::multiplyToLen() intrinsic.
 6980  *
 6981  * rdi: x
 6982  * rax: xlen
 6983  * rsi: y
 6984  * rcx: ylen
 6985  * r8:  z
 6986  * r11: tmp0
 6987  * r12: tmp1
 6988  * r13: tmp2
 6989  * r14: tmp3
 6990  * r15: tmp4
 6991  * rbx: tmp5
 6992  *
 6993  */
 6994 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
 6995                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 6996   ShortBranchVerifier sbv(this);
 6997   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 6998 
 6999   push(tmp0);
 7000   push(tmp1);
 7001   push(tmp2);
 7002   push(tmp3);
 7003   push(tmp4);
 7004   push(tmp5);
 7005 
 7006   push(xlen);
 7007 
 7008   const Register idx = tmp1;
 7009   const Register kdx = tmp2;
 7010   const Register xstart = tmp3;
 7011 
 7012   const Register y_idx = tmp4;
 7013   const Register carry = tmp5;
 7014   const Register product  = xlen;
 7015   const Register x_xstart = tmp0;
 7016 
 7017   // First Loop.
 7018   //
 7019   //  final static long LONG_MASK = 0xffffffffL;
 7020   //  int xstart = xlen - 1;
 7021   //  int ystart = ylen - 1;
 7022   //  long carry = 0;
 7023   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7024   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7025   //    z[kdx] = (int)product;
 7026   //    carry = product >>> 32;
 7027   //  }
 7028   //  z[xstart] = (int)carry;
 7029   //
 7030 
 7031   movl(idx, ylen);               // idx = ylen;
 7032   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
 7033   xorq(carry, carry);            // carry = 0;
 7034 
 7035   Label L_done;
 7036 
 7037   movl(xstart, xlen);
 7038   decrementl(xstart);
 7039   jcc(Assembler::negative, L_done);
 7040 
 7041   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7042 
 7043   Label L_second_loop;
 7044   testl(kdx, kdx);
 7045   jcc(Assembler::zero, L_second_loop);
 7046 
 7047   Label L_carry;
 7048   subl(kdx, 1);
 7049   jcc(Assembler::zero, L_carry);
 7050 
 7051   movl(Address(z, kdx, Address::times_4,  0), carry);
 7052   shrq(carry, 32);
 7053   subl(kdx, 1);
 7054 
 7055   bind(L_carry);
 7056   movl(Address(z, kdx, Address::times_4,  0), carry);
 7057 
 7058   // Second and third (nested) loops.
 7059   //
 7060   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7061   //   carry = 0;
 7062   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7063   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7064   //                    (z[k] & LONG_MASK) + carry;
 7065   //     z[k] = (int)product;
 7066   //     carry = product >>> 32;
 7067   //   }
 7068   //   z[i] = (int)carry;
 7069   // }
 7070   //
 7071   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7072 
 7073   const Register jdx = tmp1;
 7074 
 7075   bind(L_second_loop);
 7076   xorl(carry, carry);    // carry = 0;
 7077   movl(jdx, ylen);       // j = ystart+1
 7078 
 7079   subl(xstart, 1);       // i = xstart-1;
 7080   jcc(Assembler::negative, L_done);
 7081 
 7082   push (z);
 7083 
 7084   Label L_last_x;
 7085   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7086   subl(xstart, 1);       // i = xstart-1;
 7087   jcc(Assembler::negative, L_last_x);
 7088 
 7089   if (UseBMI2Instructions) {
 7090     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7091     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7092   } else {
 7093     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7094     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7095   }
 7096 
 7097   Label L_third_loop_prologue;
 7098   bind(L_third_loop_prologue);
 7099 
 7100   push (x);
 7101   push (xstart);
 7102   push (ylen);
 7103 
 7104 
 7105   if (UseBMI2Instructions) {
 7106     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7107   } else { // !UseBMI2Instructions
 7108     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7109   }
 7110 
 7111   pop(ylen);
 7112   pop(xlen);
 7113   pop(x);
 7114   pop(z);
 7115 
 7116   movl(tmp3, xlen);
 7117   addl(tmp3, 1);
 7118   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7119   subl(tmp3, 1);
 7120   jccb(Assembler::negative, L_done);
 7121 
 7122   shrq(carry, 32);
 7123   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7124   jmp(L_second_loop);
 7125 
 7126   // Next infrequent code is moved outside loops.
 7127   bind(L_last_x);
 7128   if (UseBMI2Instructions) {
 7129     movl(rdx, Address(x,  0));
 7130   } else {
 7131     movl(x_xstart, Address(x,  0));
 7132   }
 7133   jmp(L_third_loop_prologue);
 7134 
 7135   bind(L_done);
 7136 
 7137   pop(xlen);
 7138 
 7139   pop(tmp5);
 7140   pop(tmp4);
 7141   pop(tmp3);
 7142   pop(tmp2);
 7143   pop(tmp1);
 7144   pop(tmp0);
 7145 }
 7146 
 7147 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7148   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7149   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7150   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7151   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7152   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7153   Label SAME_TILL_END, DONE;
 7154   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7155 
 7156   //scale is in rcx in both Win64 and Unix
 7157   ShortBranchVerifier sbv(this);
 7158 
 7159   shlq(length);
 7160   xorq(result, result);
 7161 
 7162   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7163       VM_Version::supports_avx512vlbw()) {
 7164     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7165 
 7166     cmpq(length, 64);
 7167     jcc(Assembler::less, VECTOR32_TAIL);
 7168 
 7169     movq(tmp1, length);
 7170     andq(tmp1, 0x3F);      // tail count
 7171     andq(length, ~(0x3F)); //vector count
 7172 
 7173     bind(VECTOR64_LOOP);
 7174     // AVX512 code to compare 64 byte vectors.
 7175     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7176     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7177     kortestql(k7, k7);
 7178     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7179     addq(result, 64);
 7180     subq(length, 64);
 7181     jccb(Assembler::notZero, VECTOR64_LOOP);
 7182 
 7183     //bind(VECTOR64_TAIL);
 7184     testq(tmp1, tmp1);
 7185     jcc(Assembler::zero, SAME_TILL_END);
 7186 
 7187     //bind(VECTOR64_TAIL);
 7188     // AVX512 code to compare up to 63 byte vectors.
 7189     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7190     shlxq(tmp2, tmp2, tmp1);
 7191     notq(tmp2);
 7192     kmovql(k3, tmp2);
 7193 
 7194     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7195     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7196 
 7197     ktestql(k7, k3);
 7198     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7199 
 7200     bind(VECTOR64_NOT_EQUAL);
 7201     kmovql(tmp1, k7);
 7202     notq(tmp1);
 7203     tzcntq(tmp1, tmp1);
 7204     addq(result, tmp1);
 7205     shrq(result);
 7206     jmp(DONE);
 7207     bind(VECTOR32_TAIL);
 7208   }
 7209 
 7210   cmpq(length, 8);
 7211   jcc(Assembler::equal, VECTOR8_LOOP);
 7212   jcc(Assembler::less, VECTOR4_TAIL);
 7213 
 7214   if (UseAVX >= 2) {
 7215     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7216 
 7217     cmpq(length, 16);
 7218     jcc(Assembler::equal, VECTOR16_LOOP);
 7219     jcc(Assembler::less, VECTOR8_LOOP);
 7220 
 7221     cmpq(length, 32);
 7222     jccb(Assembler::less, VECTOR16_TAIL);
 7223 
 7224     subq(length, 32);
 7225     bind(VECTOR32_LOOP);
 7226     vmovdqu(rymm0, Address(obja, result));
 7227     vmovdqu(rymm1, Address(objb, result));
 7228     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7229     vptest(rymm2, rymm2);
 7230     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7231     addq(result, 32);
 7232     subq(length, 32);
 7233     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7234     addq(length, 32);
 7235     jcc(Assembler::equal, SAME_TILL_END);
 7236     //falling through if less than 32 bytes left //close the branch here.
 7237 
 7238     bind(VECTOR16_TAIL);
 7239     cmpq(length, 16);
 7240     jccb(Assembler::less, VECTOR8_TAIL);
 7241     bind(VECTOR16_LOOP);
 7242     movdqu(rymm0, Address(obja, result));
 7243     movdqu(rymm1, Address(objb, result));
 7244     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7245     ptest(rymm2, rymm2);
 7246     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7247     addq(result, 16);
 7248     subq(length, 16);
 7249     jcc(Assembler::equal, SAME_TILL_END);
 7250     //falling through if less than 16 bytes left
 7251   } else {//regular intrinsics
 7252 
 7253     cmpq(length, 16);
 7254     jccb(Assembler::less, VECTOR8_TAIL);
 7255 
 7256     subq(length, 16);
 7257     bind(VECTOR16_LOOP);
 7258     movdqu(rymm0, Address(obja, result));
 7259     movdqu(rymm1, Address(objb, result));
 7260     pxor(rymm0, rymm1);
 7261     ptest(rymm0, rymm0);
 7262     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7263     addq(result, 16);
 7264     subq(length, 16);
 7265     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7266     addq(length, 16);
 7267     jcc(Assembler::equal, SAME_TILL_END);
 7268     //falling through if less than 16 bytes left
 7269   }
 7270 
 7271   bind(VECTOR8_TAIL);
 7272   cmpq(length, 8);
 7273   jccb(Assembler::less, VECTOR4_TAIL);
 7274   bind(VECTOR8_LOOP);
 7275   movq(tmp1, Address(obja, result));
 7276   movq(tmp2, Address(objb, result));
 7277   xorq(tmp1, tmp2);
 7278   testq(tmp1, tmp1);
 7279   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7280   addq(result, 8);
 7281   subq(length, 8);
 7282   jcc(Assembler::equal, SAME_TILL_END);
 7283   //falling through if less than 8 bytes left
 7284 
 7285   bind(VECTOR4_TAIL);
 7286   cmpq(length, 4);
 7287   jccb(Assembler::less, BYTES_TAIL);
 7288   bind(VECTOR4_LOOP);
 7289   movl(tmp1, Address(obja, result));
 7290   xorl(tmp1, Address(objb, result));
 7291   testl(tmp1, tmp1);
 7292   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7293   addq(result, 4);
 7294   subq(length, 4);
 7295   jcc(Assembler::equal, SAME_TILL_END);
 7296   //falling through if less than 4 bytes left
 7297 
 7298   bind(BYTES_TAIL);
 7299   bind(BYTES_LOOP);
 7300   load_unsigned_byte(tmp1, Address(obja, result));
 7301   load_unsigned_byte(tmp2, Address(objb, result));
 7302   xorl(tmp1, tmp2);
 7303   testl(tmp1, tmp1);
 7304   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7305   decq(length);
 7306   jcc(Assembler::zero, SAME_TILL_END);
 7307   incq(result);
 7308   load_unsigned_byte(tmp1, Address(obja, result));
 7309   load_unsigned_byte(tmp2, Address(objb, result));
 7310   xorl(tmp1, tmp2);
 7311   testl(tmp1, tmp1);
 7312   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7313   decq(length);
 7314   jcc(Assembler::zero, SAME_TILL_END);
 7315   incq(result);
 7316   load_unsigned_byte(tmp1, Address(obja, result));
 7317   load_unsigned_byte(tmp2, Address(objb, result));
 7318   xorl(tmp1, tmp2);
 7319   testl(tmp1, tmp1);
 7320   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7321   jmp(SAME_TILL_END);
 7322 
 7323   if (UseAVX >= 2) {
 7324     bind(VECTOR32_NOT_EQUAL);
 7325     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7326     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7327     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7328     vpmovmskb(tmp1, rymm0);
 7329     bsfq(tmp1, tmp1);
 7330     addq(result, tmp1);
 7331     shrq(result);
 7332     jmp(DONE);
 7333   }
 7334 
 7335   bind(VECTOR16_NOT_EQUAL);
 7336   if (UseAVX >= 2) {
 7337     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7338     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7339     pxor(rymm0, rymm2);
 7340   } else {
 7341     pcmpeqb(rymm2, rymm2);
 7342     pxor(rymm0, rymm1);
 7343     pcmpeqb(rymm0, rymm1);
 7344     pxor(rymm0, rymm2);
 7345   }
 7346   pmovmskb(tmp1, rymm0);
 7347   bsfq(tmp1, tmp1);
 7348   addq(result, tmp1);
 7349   shrq(result);
 7350   jmpb(DONE);
 7351 
 7352   bind(VECTOR8_NOT_EQUAL);
 7353   bind(VECTOR4_NOT_EQUAL);
 7354   bsfq(tmp1, tmp1);
 7355   shrq(tmp1, 3);
 7356   addq(result, tmp1);
 7357   bind(BYTES_NOT_EQUAL);
 7358   shrq(result);
 7359   jmpb(DONE);
 7360 
 7361   bind(SAME_TILL_END);
 7362   mov64(result, -1);
 7363 
 7364   bind(DONE);
 7365 }
 7366 
 7367 //Helper functions for square_to_len()
 7368 
 7369 /**
 7370  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7371  * Preserves x and z and modifies rest of the registers.
 7372  */
 7373 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7374   // Perform square and right shift by 1
 7375   // Handle odd xlen case first, then for even xlen do the following
 7376   // jlong carry = 0;
 7377   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7378   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7379   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7380   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7381   //     carry = (jlong)product;
 7382   // }
 7383 
 7384   xorq(tmp5, tmp5);     // carry
 7385   xorq(rdxReg, rdxReg);
 7386   xorl(tmp1, tmp1);     // index for x
 7387   xorl(tmp4, tmp4);     // index for z
 7388 
 7389   Label L_first_loop, L_first_loop_exit;
 7390 
 7391   testl(xlen, 1);
 7392   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7393 
 7394   // Square and right shift by 1 the odd element using 32 bit multiply
 7395   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7396   imulq(raxReg, raxReg);
 7397   shrq(raxReg, 1);
 7398   adcq(tmp5, 0);
 7399   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7400   incrementl(tmp1);
 7401   addl(tmp4, 2);
 7402 
 7403   // Square and  right shift by 1 the rest using 64 bit multiply
 7404   bind(L_first_loop);
 7405   cmpptr(tmp1, xlen);
 7406   jccb(Assembler::equal, L_first_loop_exit);
 7407 
 7408   // Square
 7409   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7410   rorq(raxReg, 32);    // convert big-endian to little-endian
 7411   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7412 
 7413   // Right shift by 1 and save carry
 7414   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7415   rcrq(rdxReg, 1);
 7416   rcrq(raxReg, 1);
 7417   adcq(tmp5, 0);
 7418 
 7419   // Store result in z
 7420   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7421   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7422 
 7423   // Update indices for x and z
 7424   addl(tmp1, 2);
 7425   addl(tmp4, 4);
 7426   jmp(L_first_loop);
 7427 
 7428   bind(L_first_loop_exit);
 7429 }
 7430 
 7431 
 7432 /**
 7433  * Perform the following multiply add operation using BMI2 instructions
 7434  * carry:sum = sum + op1*op2 + carry
 7435  * op2 should be in rdx
 7436  * op2 is preserved, all other registers are modified
 7437  */
 7438 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7439   // assert op2 is rdx
 7440   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7441   addq(sum, carry);
 7442   adcq(tmp2, 0);
 7443   addq(sum, op1);
 7444   adcq(tmp2, 0);
 7445   movq(carry, tmp2);
 7446 }
 7447 
 7448 /**
 7449  * Perform the following multiply add operation:
 7450  * carry:sum = sum + op1*op2 + carry
 7451  * Preserves op1, op2 and modifies rest of registers
 7452  */
 7453 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7454   // rdx:rax = op1 * op2
 7455   movq(raxReg, op2);
 7456   mulq(op1);
 7457 
 7458   //  rdx:rax = sum + carry + rdx:rax
 7459   addq(sum, carry);
 7460   adcq(rdxReg, 0);
 7461   addq(sum, raxReg);
 7462   adcq(rdxReg, 0);
 7463 
 7464   // carry:sum = rdx:sum
 7465   movq(carry, rdxReg);
 7466 }
 7467 
 7468 /**
 7469  * Add 64 bit long carry into z[] with carry propagation.
 7470  * Preserves z and carry register values and modifies rest of registers.
 7471  *
 7472  */
 7473 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7474   Label L_fourth_loop, L_fourth_loop_exit;
 7475 
 7476   movl(tmp1, 1);
 7477   subl(zlen, 2);
 7478   addq(Address(z, zlen, Address::times_4, 0), carry);
 7479 
 7480   bind(L_fourth_loop);
 7481   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7482   subl(zlen, 2);
 7483   jccb(Assembler::negative, L_fourth_loop_exit);
 7484   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7485   jmp(L_fourth_loop);
 7486   bind(L_fourth_loop_exit);
 7487 }
 7488 
 7489 /**
 7490  * Shift z[] left by 1 bit.
 7491  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7492  *
 7493  */
 7494 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7495 
 7496   Label L_fifth_loop, L_fifth_loop_exit;
 7497 
 7498   // Fifth loop
 7499   // Perform primitiveLeftShift(z, zlen, 1)
 7500 
 7501   const Register prev_carry = tmp1;
 7502   const Register new_carry = tmp4;
 7503   const Register value = tmp2;
 7504   const Register zidx = tmp3;
 7505 
 7506   // int zidx, carry;
 7507   // long value;
 7508   // carry = 0;
 7509   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7510   //    (carry:value)  = (z[i] << 1) | carry ;
 7511   //    z[i] = value;
 7512   // }
 7513 
 7514   movl(zidx, zlen);
 7515   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7516 
 7517   bind(L_fifth_loop);
 7518   decl(zidx);  // Use decl to preserve carry flag
 7519   decl(zidx);
 7520   jccb(Assembler::negative, L_fifth_loop_exit);
 7521 
 7522   if (UseBMI2Instructions) {
 7523      movq(value, Address(z, zidx, Address::times_4, 0));
 7524      rclq(value, 1);
 7525      rorxq(value, value, 32);
 7526      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7527   }
 7528   else {
 7529     // clear new_carry
 7530     xorl(new_carry, new_carry);
 7531 
 7532     // Shift z[i] by 1, or in previous carry and save new carry
 7533     movq(value, Address(z, zidx, Address::times_4, 0));
 7534     shlq(value, 1);
 7535     adcl(new_carry, 0);
 7536 
 7537     orq(value, prev_carry);
 7538     rorq(value, 0x20);
 7539     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7540 
 7541     // Set previous carry = new carry
 7542     movl(prev_carry, new_carry);
 7543   }
 7544   jmp(L_fifth_loop);
 7545 
 7546   bind(L_fifth_loop_exit);
 7547 }
 7548 
 7549 
 7550 /**
 7551  * Code for BigInteger::squareToLen() intrinsic
 7552  *
 7553  * rdi: x
 7554  * rsi: len
 7555  * r8:  z
 7556  * rcx: zlen
 7557  * r12: tmp1
 7558  * r13: tmp2
 7559  * r14: tmp3
 7560  * r15: tmp4
 7561  * rbx: tmp5
 7562  *
 7563  */
 7564 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7565 
 7566   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7567   push(tmp1);
 7568   push(tmp2);
 7569   push(tmp3);
 7570   push(tmp4);
 7571   push(tmp5);
 7572 
 7573   // First loop
 7574   // Store the squares, right shifted one bit (i.e., divided by 2).
 7575   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7576 
 7577   // Add in off-diagonal sums.
 7578   //
 7579   // Second, third (nested) and fourth loops.
 7580   // zlen +=2;
 7581   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7582   //    carry = 0;
 7583   //    long op2 = x[xidx:xidx+1];
 7584   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7585   //       k -= 2;
 7586   //       long op1 = x[j:j+1];
 7587   //       long sum = z[k:k+1];
 7588   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7589   //       z[k:k+1] = sum;
 7590   //    }
 7591   //    add_one_64(z, k, carry, tmp_regs);
 7592   // }
 7593 
 7594   const Register carry = tmp5;
 7595   const Register sum = tmp3;
 7596   const Register op1 = tmp4;
 7597   Register op2 = tmp2;
 7598 
 7599   push(zlen);
 7600   push(len);
 7601   addl(zlen,2);
 7602   bind(L_second_loop);
 7603   xorq(carry, carry);
 7604   subl(zlen, 4);
 7605   subl(len, 2);
 7606   push(zlen);
 7607   push(len);
 7608   cmpl(len, 0);
 7609   jccb(Assembler::lessEqual, L_second_loop_exit);
 7610 
 7611   // Multiply an array by one 64 bit long.
 7612   if (UseBMI2Instructions) {
 7613     op2 = rdxReg;
 7614     movq(op2, Address(x, len, Address::times_4,  0));
 7615     rorxq(op2, op2, 32);
 7616   }
 7617   else {
 7618     movq(op2, Address(x, len, Address::times_4,  0));
 7619     rorq(op2, 32);
 7620   }
 7621 
 7622   bind(L_third_loop);
 7623   decrementl(len);
 7624   jccb(Assembler::negative, L_third_loop_exit);
 7625   decrementl(len);
 7626   jccb(Assembler::negative, L_last_x);
 7627 
 7628   movq(op1, Address(x, len, Address::times_4,  0));
 7629   rorq(op1, 32);
 7630 
 7631   bind(L_multiply);
 7632   subl(zlen, 2);
 7633   movq(sum, Address(z, zlen, Address::times_4,  0));
 7634 
 7635   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 7636   if (UseBMI2Instructions) {
 7637     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 7638   }
 7639   else {
 7640     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7641   }
 7642 
 7643   movq(Address(z, zlen, Address::times_4, 0), sum);
 7644 
 7645   jmp(L_third_loop);
 7646   bind(L_third_loop_exit);
 7647 
 7648   // Fourth loop
 7649   // Add 64 bit long carry into z with carry propagation.
 7650   // Uses offsetted zlen.
 7651   add_one_64(z, zlen, carry, tmp1);
 7652 
 7653   pop(len);
 7654   pop(zlen);
 7655   jmp(L_second_loop);
 7656 
 7657   // Next infrequent code is moved outside loops.
 7658   bind(L_last_x);
 7659   movl(op1, Address(x, 0));
 7660   jmp(L_multiply);
 7661 
 7662   bind(L_second_loop_exit);
 7663   pop(len);
 7664   pop(zlen);
 7665   pop(len);
 7666   pop(zlen);
 7667 
 7668   // Fifth loop
 7669   // Shift z left 1 bit.
 7670   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 7671 
 7672   // z[zlen-1] |= x[len-1] & 1;
 7673   movl(tmp3, Address(x, len, Address::times_4, -4));
 7674   andl(tmp3, 1);
 7675   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 7676 
 7677   pop(tmp5);
 7678   pop(tmp4);
 7679   pop(tmp3);
 7680   pop(tmp2);
 7681   pop(tmp1);
 7682 }
 7683 
 7684 /**
 7685  * Helper function for mul_add()
 7686  * Multiply the in[] by int k and add to out[] starting at offset offs using
 7687  * 128 bit by 32 bit multiply and return the carry in tmp5.
 7688  * Only quad int aligned length of in[] is operated on in this function.
 7689  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 7690  * This function preserves out, in and k registers.
 7691  * len and offset point to the appropriate index in "in" & "out" correspondingly
 7692  * tmp5 has the carry.
 7693  * other registers are temporary and are modified.
 7694  *
 7695  */
 7696 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 7697   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 7698   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7699 
 7700   Label L_first_loop, L_first_loop_exit;
 7701 
 7702   movl(tmp1, len);
 7703   shrl(tmp1, 2);
 7704 
 7705   bind(L_first_loop);
 7706   subl(tmp1, 1);
 7707   jccb(Assembler::negative, L_first_loop_exit);
 7708 
 7709   subl(len, 4);
 7710   subl(offset, 4);
 7711 
 7712   Register op2 = tmp2;
 7713   const Register sum = tmp3;
 7714   const Register op1 = tmp4;
 7715   const Register carry = tmp5;
 7716 
 7717   if (UseBMI2Instructions) {
 7718     op2 = rdxReg;
 7719   }
 7720 
 7721   movq(op1, Address(in, len, Address::times_4,  8));
 7722   rorq(op1, 32);
 7723   movq(sum, Address(out, offset, Address::times_4,  8));
 7724   rorq(sum, 32);
 7725   if (UseBMI2Instructions) {
 7726     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7727   }
 7728   else {
 7729     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7730   }
 7731   // Store back in big endian from little endian
 7732   rorq(sum, 0x20);
 7733   movq(Address(out, offset, Address::times_4,  8), sum);
 7734 
 7735   movq(op1, Address(in, len, Address::times_4,  0));
 7736   rorq(op1, 32);
 7737   movq(sum, Address(out, offset, Address::times_4,  0));
 7738   rorq(sum, 32);
 7739   if (UseBMI2Instructions) {
 7740     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7741   }
 7742   else {
 7743     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7744   }
 7745   // Store back in big endian from little endian
 7746   rorq(sum, 0x20);
 7747   movq(Address(out, offset, Address::times_4,  0), sum);
 7748 
 7749   jmp(L_first_loop);
 7750   bind(L_first_loop_exit);
 7751 }
 7752 
 7753 /**
 7754  * Code for BigInteger::mulAdd() intrinsic
 7755  *
 7756  * rdi: out
 7757  * rsi: in
 7758  * r11: offs (out.length - offset)
 7759  * rcx: len
 7760  * r8:  k
 7761  * r12: tmp1
 7762  * r13: tmp2
 7763  * r14: tmp3
 7764  * r15: tmp4
 7765  * rbx: tmp5
 7766  * Multiply the in[] by word k and add to out[], return the carry in rax
 7767  */
 7768 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 7769    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 7770    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7771 
 7772   Label L_carry, L_last_in, L_done;
 7773 
 7774 // carry = 0;
 7775 // for (int j=len-1; j >= 0; j--) {
 7776 //    long product = (in[j] & LONG_MASK) * kLong +
 7777 //                   (out[offs] & LONG_MASK) + carry;
 7778 //    out[offs--] = (int)product;
 7779 //    carry = product >>> 32;
 7780 // }
 7781 //
 7782   push(tmp1);
 7783   push(tmp2);
 7784   push(tmp3);
 7785   push(tmp4);
 7786   push(tmp5);
 7787 
 7788   Register op2 = tmp2;
 7789   const Register sum = tmp3;
 7790   const Register op1 = tmp4;
 7791   const Register carry =  tmp5;
 7792 
 7793   if (UseBMI2Instructions) {
 7794     op2 = rdxReg;
 7795     movl(op2, k);
 7796   }
 7797   else {
 7798     movl(op2, k);
 7799   }
 7800 
 7801   xorq(carry, carry);
 7802 
 7803   //First loop
 7804 
 7805   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 7806   //The carry is in tmp5
 7807   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7808 
 7809   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 7810   decrementl(len);
 7811   jccb(Assembler::negative, L_carry);
 7812   decrementl(len);
 7813   jccb(Assembler::negative, L_last_in);
 7814 
 7815   movq(op1, Address(in, len, Address::times_4,  0));
 7816   rorq(op1, 32);
 7817 
 7818   subl(offs, 2);
 7819   movq(sum, Address(out, offs, Address::times_4,  0));
 7820   rorq(sum, 32);
 7821 
 7822   if (UseBMI2Instructions) {
 7823     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7824   }
 7825   else {
 7826     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7827   }
 7828 
 7829   // Store back in big endian from little endian
 7830   rorq(sum, 0x20);
 7831   movq(Address(out, offs, Address::times_4,  0), sum);
 7832 
 7833   testl(len, len);
 7834   jccb(Assembler::zero, L_carry);
 7835 
 7836   //Multiply the last in[] entry, if any
 7837   bind(L_last_in);
 7838   movl(op1, Address(in, 0));
 7839   movl(sum, Address(out, offs, Address::times_4,  -4));
 7840 
 7841   movl(raxReg, k);
 7842   mull(op1); //tmp4 * eax -> edx:eax
 7843   addl(sum, carry);
 7844   adcl(rdxReg, 0);
 7845   addl(sum, raxReg);
 7846   adcl(rdxReg, 0);
 7847   movl(carry, rdxReg);
 7848 
 7849   movl(Address(out, offs, Address::times_4,  -4), sum);
 7850 
 7851   bind(L_carry);
 7852   //return tmp5/carry as carry in rax
 7853   movl(rax, carry);
 7854 
 7855   bind(L_done);
 7856   pop(tmp5);
 7857   pop(tmp4);
 7858   pop(tmp3);
 7859   pop(tmp2);
 7860   pop(tmp1);
 7861 }
 7862 #endif
 7863 
 7864 /**
 7865  * Emits code to update CRC-32 with a byte value according to constants in table
 7866  *
 7867  * @param [in,out]crc   Register containing the crc.
 7868  * @param [in]val       Register containing the byte to fold into the CRC.
 7869  * @param [in]table     Register containing the table of crc constants.
 7870  *
 7871  * uint32_t crc;
 7872  * val = crc_table[(val ^ crc) & 0xFF];
 7873  * crc = val ^ (crc >> 8);
 7874  *
 7875  */
 7876 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 7877   xorl(val, crc);
 7878   andl(val, 0xFF);
 7879   shrl(crc, 8); // unsigned shift
 7880   xorl(crc, Address(table, val, Address::times_4, 0));
 7881 }
 7882 
 7883 /**
 7884  * Fold 128-bit data chunk
 7885  */
 7886 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 7887   if (UseAVX > 0) {
 7888     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 7889     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 7890     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 7891     pxor(xcrc, xtmp);
 7892   } else {
 7893     movdqa(xtmp, xcrc);
 7894     pclmulhdq(xtmp, xK);   // [123:64]
 7895     pclmulldq(xcrc, xK);   // [63:0]
 7896     pxor(xcrc, xtmp);
 7897     movdqu(xtmp, Address(buf, offset));
 7898     pxor(xcrc, xtmp);
 7899   }
 7900 }
 7901 
 7902 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 7903   if (UseAVX > 0) {
 7904     vpclmulhdq(xtmp, xK, xcrc);
 7905     vpclmulldq(xcrc, xK, xcrc);
 7906     pxor(xcrc, xbuf);
 7907     pxor(xcrc, xtmp);
 7908   } else {
 7909     movdqa(xtmp, xcrc);
 7910     pclmulhdq(xtmp, xK);
 7911     pclmulldq(xcrc, xK);
 7912     pxor(xcrc, xbuf);
 7913     pxor(xcrc, xtmp);
 7914   }
 7915 }
 7916 
 7917 /**
 7918  * 8-bit folds to compute 32-bit CRC
 7919  *
 7920  * uint64_t xcrc;
 7921  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 7922  */
 7923 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 7924   movdl(tmp, xcrc);
 7925   andl(tmp, 0xFF);
 7926   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 7927   psrldq(xcrc, 1); // unsigned shift one byte
 7928   pxor(xcrc, xtmp);
 7929 }
 7930 
 7931 /**
 7932  * uint32_t crc;
 7933  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 7934  */
 7935 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 7936   movl(tmp, crc);
 7937   andl(tmp, 0xFF);
 7938   shrl(crc, 8);
 7939   xorl(crc, Address(table, tmp, Address::times_4, 0));
 7940 }
 7941 
 7942 /**
 7943  * @param crc   register containing existing CRC (32-bit)
 7944  * @param buf   register pointing to input byte buffer (byte*)
 7945  * @param len   register containing number of bytes
 7946  * @param table register that will contain address of CRC table
 7947  * @param tmp   scratch register
 7948  */
 7949 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 7950   assert_different_registers(crc, buf, len, table, tmp, rax);
 7951 
 7952   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 7953   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 7954 
 7955   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 7956   // context for the registers used, where all instructions below are using 128-bit mode
 7957   // On EVEX without VL and BW, these instructions will all be AVX.
 7958   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 7959   notl(crc); // ~crc
 7960   cmpl(len, 16);
 7961   jcc(Assembler::less, L_tail);
 7962 
 7963   // Align buffer to 16 bytes
 7964   movl(tmp, buf);
 7965   andl(tmp, 0xF);
 7966   jccb(Assembler::zero, L_aligned);
 7967   subl(tmp,  16);
 7968   addl(len, tmp);
 7969 
 7970   align(4);
 7971   BIND(L_align_loop);
 7972   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 7973   update_byte_crc32(crc, rax, table);
 7974   increment(buf);
 7975   incrementl(tmp);
 7976   jccb(Assembler::less, L_align_loop);
 7977 
 7978   BIND(L_aligned);
 7979   movl(tmp, len); // save
 7980   shrl(len, 4);
 7981   jcc(Assembler::zero, L_tail_restore);
 7982 
 7983   // Fold crc into first bytes of vector
 7984   movdqa(xmm1, Address(buf, 0));
 7985   movdl(rax, xmm1);
 7986   xorl(crc, rax);
 7987   if (VM_Version::supports_sse4_1()) {
 7988     pinsrd(xmm1, crc, 0);
 7989   } else {
 7990     pinsrw(xmm1, crc, 0);
 7991     shrl(crc, 16);
 7992     pinsrw(xmm1, crc, 1);
 7993   }
 7994   addptr(buf, 16);
 7995   subl(len, 4); // len > 0
 7996   jcc(Assembler::less, L_fold_tail);
 7997 
 7998   movdqa(xmm2, Address(buf,  0));
 7999   movdqa(xmm3, Address(buf, 16));
 8000   movdqa(xmm4, Address(buf, 32));
 8001   addptr(buf, 48);
 8002   subl(len, 3);
 8003   jcc(Assembler::lessEqual, L_fold_512b);
 8004 
 8005   // Fold total 512 bits of polynomial on each iteration,
 8006   // 128 bits per each of 4 parallel streams.
 8007   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8008 
 8009   align32();
 8010   BIND(L_fold_512b_loop);
 8011   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8012   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8013   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8014   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8015   addptr(buf, 64);
 8016   subl(len, 4);
 8017   jcc(Assembler::greater, L_fold_512b_loop);
 8018 
 8019   // Fold 512 bits to 128 bits.
 8020   BIND(L_fold_512b);
 8021   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8022   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8023   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8024   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8025 
 8026   // Fold the rest of 128 bits data chunks
 8027   BIND(L_fold_tail);
 8028   addl(len, 3);
 8029   jccb(Assembler::lessEqual, L_fold_128b);
 8030   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8031 
 8032   BIND(L_fold_tail_loop);
 8033   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8034   addptr(buf, 16);
 8035   decrementl(len);
 8036   jccb(Assembler::greater, L_fold_tail_loop);
 8037 
 8038   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8039   BIND(L_fold_128b);
 8040   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8041   if (UseAVX > 0) {
 8042     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8043     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8044     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8045   } else {
 8046     movdqa(xmm2, xmm0);
 8047     pclmulqdq(xmm2, xmm1, 0x1);
 8048     movdqa(xmm3, xmm0);
 8049     pand(xmm3, xmm2);
 8050     pclmulqdq(xmm0, xmm3, 0x1);
 8051   }
 8052   psrldq(xmm1, 8);
 8053   psrldq(xmm2, 4);
 8054   pxor(xmm0, xmm1);
 8055   pxor(xmm0, xmm2);
 8056 
 8057   // 8 8-bit folds to compute 32-bit CRC.
 8058   for (int j = 0; j < 4; j++) {
 8059     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8060   }
 8061   movdl(crc, xmm0); // mov 32 bits to general register
 8062   for (int j = 0; j < 4; j++) {
 8063     fold_8bit_crc32(crc, table, rax);
 8064   }
 8065 
 8066   BIND(L_tail_restore);
 8067   movl(len, tmp); // restore
 8068   BIND(L_tail);
 8069   andl(len, 0xf);
 8070   jccb(Assembler::zero, L_exit);
 8071 
 8072   // Fold the rest of bytes
 8073   align(4);
 8074   BIND(L_tail_loop);
 8075   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8076   update_byte_crc32(crc, rax, table);
 8077   increment(buf);
 8078   decrementl(len);
 8079   jccb(Assembler::greater, L_tail_loop);
 8080 
 8081   BIND(L_exit);
 8082   notl(crc); // ~c
 8083 }
 8084 
 8085 #ifdef _LP64
 8086 // Helper function for AVX 512 CRC32
 8087 // Fold 512-bit data chunks
 8088 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8089                                              Register pos, int offset) {
 8090   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8091   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8092   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8093   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8094   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8095 }
 8096 
 8097 // Helper function for AVX 512 CRC32
 8098 // Compute CRC32 for < 256B buffers
 8099 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8100                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8101                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8102 
 8103   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8104   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8105   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8106 
 8107   // check if there is enough buffer to be able to fold 16B at a time
 8108   cmpl(len, 32);
 8109   jcc(Assembler::less, L_less_than_32);
 8110 
 8111   // if there is, load the constants
 8112   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8113   movdl(xmm0, crc);                        // get the initial crc value
 8114   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8115   pxor(xmm7, xmm0);
 8116 
 8117   // update the buffer pointer
 8118   addl(pos, 16);
 8119   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8120   subl(len, 32);
 8121   jmp(L_16B_reduction_loop);
 8122 
 8123   bind(L_less_than_32);
 8124   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8125   movl(rax, crc);
 8126   testl(len, len);
 8127   jcc(Assembler::equal, L_cleanup);
 8128 
 8129   movdl(xmm0, crc);                        //get the initial crc value
 8130 
 8131   cmpl(len, 16);
 8132   jcc(Assembler::equal, L_exact_16_left);
 8133   jcc(Assembler::less, L_less_than_16_left);
 8134 
 8135   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8136   pxor(xmm7, xmm0);                       //xor the initial crc value
 8137   addl(pos, 16);
 8138   subl(len, 16);
 8139   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8140   jmp(L_get_last_two_xmms);
 8141 
 8142   bind(L_less_than_16_left);
 8143   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8144   pxor(xmm1, xmm1);
 8145   movptr(tmp1, rsp);
 8146   movdqu(Address(tmp1, 0 * 16), xmm1);
 8147 
 8148   cmpl(len, 4);
 8149   jcc(Assembler::less, L_only_less_than_4);
 8150 
 8151   //backup the counter value
 8152   movl(tmp2, len);
 8153   cmpl(len, 8);
 8154   jcc(Assembler::less, L_less_than_8_left);
 8155 
 8156   //load 8 Bytes
 8157   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8158   movq(Address(tmp1, 0 * 16), rax);
 8159   addptr(tmp1, 8);
 8160   subl(len, 8);
 8161   addl(pos, 8);
 8162 
 8163   bind(L_less_than_8_left);
 8164   cmpl(len, 4);
 8165   jcc(Assembler::less, L_less_than_4_left);
 8166 
 8167   //load 4 Bytes
 8168   movl(rax, Address(buf, pos, Address::times_1, 0));
 8169   movl(Address(tmp1, 0 * 16), rax);
 8170   addptr(tmp1, 4);
 8171   subl(len, 4);
 8172   addl(pos, 4);
 8173 
 8174   bind(L_less_than_4_left);
 8175   cmpl(len, 2);
 8176   jcc(Assembler::less, L_less_than_2_left);
 8177 
 8178   // load 2 Bytes
 8179   movw(rax, Address(buf, pos, Address::times_1, 0));
 8180   movl(Address(tmp1, 0 * 16), rax);
 8181   addptr(tmp1, 2);
 8182   subl(len, 2);
 8183   addl(pos, 2);
 8184 
 8185   bind(L_less_than_2_left);
 8186   cmpl(len, 1);
 8187   jcc(Assembler::less, L_zero_left);
 8188 
 8189   // load 1 Byte
 8190   movb(rax, Address(buf, pos, Address::times_1, 0));
 8191   movb(Address(tmp1, 0 * 16), rax);
 8192 
 8193   bind(L_zero_left);
 8194   movdqu(xmm7, Address(rsp, 0));
 8195   pxor(xmm7, xmm0);                       //xor the initial crc value
 8196 
 8197   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8198   movdqu(xmm0, Address(rax, tmp2));
 8199   pshufb(xmm7, xmm0);
 8200   jmp(L_128_done);
 8201 
 8202   bind(L_exact_16_left);
 8203   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8204   pxor(xmm7, xmm0);                       //xor the initial crc value
 8205   jmp(L_128_done);
 8206 
 8207   bind(L_only_less_than_4);
 8208   cmpl(len, 3);
 8209   jcc(Assembler::less, L_only_less_than_3);
 8210 
 8211   // load 3 Bytes
 8212   movb(rax, Address(buf, pos, Address::times_1, 0));
 8213   movb(Address(tmp1, 0), rax);
 8214 
 8215   movb(rax, Address(buf, pos, Address::times_1, 1));
 8216   movb(Address(tmp1, 1), rax);
 8217 
 8218   movb(rax, Address(buf, pos, Address::times_1, 2));
 8219   movb(Address(tmp1, 2), rax);
 8220 
 8221   movdqu(xmm7, Address(rsp, 0));
 8222   pxor(xmm7, xmm0);                     //xor the initial crc value
 8223 
 8224   pslldq(xmm7, 0x5);
 8225   jmp(L_barrett);
 8226   bind(L_only_less_than_3);
 8227   cmpl(len, 2);
 8228   jcc(Assembler::less, L_only_less_than_2);
 8229 
 8230   // load 2 Bytes
 8231   movb(rax, Address(buf, pos, Address::times_1, 0));
 8232   movb(Address(tmp1, 0), rax);
 8233 
 8234   movb(rax, Address(buf, pos, Address::times_1, 1));
 8235   movb(Address(tmp1, 1), rax);
 8236 
 8237   movdqu(xmm7, Address(rsp, 0));
 8238   pxor(xmm7, xmm0);                     //xor the initial crc value
 8239 
 8240   pslldq(xmm7, 0x6);
 8241   jmp(L_barrett);
 8242 
 8243   bind(L_only_less_than_2);
 8244   //load 1 Byte
 8245   movb(rax, Address(buf, pos, Address::times_1, 0));
 8246   movb(Address(tmp1, 0), rax);
 8247 
 8248   movdqu(xmm7, Address(rsp, 0));
 8249   pxor(xmm7, xmm0);                     //xor the initial crc value
 8250 
 8251   pslldq(xmm7, 0x7);
 8252 }
 8253 
 8254 /**
 8255 * Compute CRC32 using AVX512 instructions
 8256 * param crc   register containing existing CRC (32-bit)
 8257 * param buf   register pointing to input byte buffer (byte*)
 8258 * param len   register containing number of bytes
 8259 * param table address of crc or crc32c table
 8260 * param tmp1  scratch register
 8261 * param tmp2  scratch register
 8262 * return rax  result register
 8263 *
 8264 * This routine is identical for crc32c with the exception of the precomputed constant
 8265 * table which will be passed as the table argument.  The calculation steps are
 8266 * the same for both variants.
 8267 */
 8268 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8269   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8270 
 8271   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8272   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8273   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8274   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8275   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8276 
 8277   const Register pos = r12;
 8278   push(r12);
 8279   subptr(rsp, 16 * 2 + 8);
 8280 
 8281   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8282   // context for the registers used, where all instructions below are using 128-bit mode
 8283   // On EVEX without VL and BW, these instructions will all be AVX.
 8284   movl(pos, 0);
 8285 
 8286   // check if smaller than 256B
 8287   cmpl(len, 256);
 8288   jcc(Assembler::less, L_less_than_256);
 8289 
 8290   // load the initial crc value
 8291   movdl(xmm10, crc);
 8292 
 8293   // receive the initial 64B data, xor the initial crc value
 8294   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8295   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8296   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8297   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8298 
 8299   subl(len, 256);
 8300   cmpl(len, 256);
 8301   jcc(Assembler::less, L_fold_128_B_loop);
 8302 
 8303   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8304   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8305   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8306   subl(len, 256);
 8307 
 8308   bind(L_fold_256_B_loop);
 8309   addl(pos, 256);
 8310   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8311   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8312   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8313   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8314 
 8315   subl(len, 256);
 8316   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8317 
 8318   // Fold 256 into 128
 8319   addl(pos, 256);
 8320   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8321   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8322   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8323 
 8324   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8325   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8326   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8327 
 8328   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8329   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8330 
 8331   addl(len, 128);
 8332   jmp(L_fold_128_B_register);
 8333 
 8334   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8335   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8336 
 8337   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8338   bind(L_fold_128_B_loop);
 8339   addl(pos, 128);
 8340   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8341   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8342 
 8343   subl(len, 128);
 8344   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8345 
 8346   addl(pos, 128);
 8347 
 8348   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8349   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8350   bind(L_fold_128_B_register);
 8351   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8352   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8353   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8354   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8355   // save last that has no multiplicand
 8356   vextracti64x2(xmm7, xmm4, 3);
 8357 
 8358   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8359   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8360   // Needed later in reduction loop
 8361   movdqu(xmm10, Address(table, 1 * 16));
 8362   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8363   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8364 
 8365   // Swap 1,0,3,2 - 01 00 11 10
 8366   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8367   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8368   vextracti128(xmm5, xmm8, 1);
 8369   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8370 
 8371   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8372   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8373   addl(len, 128 - 16);
 8374   jcc(Assembler::less, L_final_reduction_for_128);
 8375 
 8376   bind(L_16B_reduction_loop);
 8377   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8378   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8379   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8380   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8381   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8382   addl(pos, 16);
 8383   subl(len, 16);
 8384   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8385 
 8386   bind(L_final_reduction_for_128);
 8387   addl(len, 16);
 8388   jcc(Assembler::equal, L_128_done);
 8389 
 8390   bind(L_get_last_two_xmms);
 8391   movdqu(xmm2, xmm7);
 8392   addl(pos, len);
 8393   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8394   subl(pos, len);
 8395 
 8396   // get rid of the extra data that was loaded before
 8397   // load the shift constant
 8398   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8399   movdqu(xmm0, Address(rax, len));
 8400   addl(rax, len);
 8401 
 8402   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8403   //Change mask to 512
 8404   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8405   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8406 
 8407   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8408   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8409   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8410   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8411   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8412 
 8413   bind(L_128_done);
 8414   // compute crc of a 128-bit value
 8415   movdqu(xmm10, Address(table, 3 * 16));
 8416   movdqu(xmm0, xmm7);
 8417 
 8418   // 64b fold
 8419   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8420   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8421   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8422 
 8423   // 32b fold
 8424   movdqu(xmm0, xmm7);
 8425   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8426   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8427   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8428   jmp(L_barrett);
 8429 
 8430   bind(L_less_than_256);
 8431   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8432 
 8433   //barrett reduction
 8434   bind(L_barrett);
 8435   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8436   movdqu(xmm1, xmm7);
 8437   movdqu(xmm2, xmm7);
 8438   movdqu(xmm10, Address(table, 4 * 16));
 8439 
 8440   pclmulqdq(xmm7, xmm10, 0x0);
 8441   pxor(xmm7, xmm2);
 8442   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8443   movdqu(xmm2, xmm7);
 8444   pclmulqdq(xmm7, xmm10, 0x10);
 8445   pxor(xmm7, xmm2);
 8446   pxor(xmm7, xmm1);
 8447   pextrd(crc, xmm7, 2);
 8448 
 8449   bind(L_cleanup);
 8450   addptr(rsp, 16 * 2 + 8);
 8451   pop(r12);
 8452 }
 8453 
 8454 // S. Gueron / Information Processing Letters 112 (2012) 184
 8455 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8456 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8457 // Output: the 64-bit carry-less product of B * CONST
 8458 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8459                                      Register tmp1, Register tmp2, Register tmp3) {
 8460   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8461   if (n > 0) {
 8462     addq(tmp3, n * 256 * 8);
 8463   }
 8464   //    Q1 = TABLEExt[n][B & 0xFF];
 8465   movl(tmp1, in);
 8466   andl(tmp1, 0x000000FF);
 8467   shll(tmp1, 3);
 8468   addq(tmp1, tmp3);
 8469   movq(tmp1, Address(tmp1, 0));
 8470 
 8471   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8472   movl(tmp2, in);
 8473   shrl(tmp2, 8);
 8474   andl(tmp2, 0x000000FF);
 8475   shll(tmp2, 3);
 8476   addq(tmp2, tmp3);
 8477   movq(tmp2, Address(tmp2, 0));
 8478 
 8479   shlq(tmp2, 8);
 8480   xorq(tmp1, tmp2);
 8481 
 8482   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8483   movl(tmp2, in);
 8484   shrl(tmp2, 16);
 8485   andl(tmp2, 0x000000FF);
 8486   shll(tmp2, 3);
 8487   addq(tmp2, tmp3);
 8488   movq(tmp2, Address(tmp2, 0));
 8489 
 8490   shlq(tmp2, 16);
 8491   xorq(tmp1, tmp2);
 8492 
 8493   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8494   shrl(in, 24);
 8495   andl(in, 0x000000FF);
 8496   shll(in, 3);
 8497   addq(in, tmp3);
 8498   movq(in, Address(in, 0));
 8499 
 8500   shlq(in, 24);
 8501   xorq(in, tmp1);
 8502   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8503 }
 8504 
 8505 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8506                                       Register in_out,
 8507                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8508                                       XMMRegister w_xtmp2,
 8509                                       Register tmp1,
 8510                                       Register n_tmp2, Register n_tmp3) {
 8511   if (is_pclmulqdq_supported) {
 8512     movdl(w_xtmp1, in_out); // modified blindly
 8513 
 8514     movl(tmp1, const_or_pre_comp_const_index);
 8515     movdl(w_xtmp2, tmp1);
 8516     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8517 
 8518     movdq(in_out, w_xtmp1);
 8519   } else {
 8520     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8521   }
 8522 }
 8523 
 8524 // Recombination Alternative 2: No bit-reflections
 8525 // T1 = (CRC_A * U1) << 1
 8526 // T2 = (CRC_B * U2) << 1
 8527 // C1 = T1 >> 32
 8528 // C2 = T2 >> 32
 8529 // T1 = T1 & 0xFFFFFFFF
 8530 // T2 = T2 & 0xFFFFFFFF
 8531 // T1 = CRC32(0, T1)
 8532 // T2 = CRC32(0, T2)
 8533 // C1 = C1 ^ T1
 8534 // C2 = C2 ^ T2
 8535 // CRC = C1 ^ C2 ^ CRC_C
 8536 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8537                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8538                                      Register tmp1, Register tmp2,
 8539                                      Register n_tmp3) {
 8540   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8541   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8542   shlq(in_out, 1);
 8543   movl(tmp1, in_out);
 8544   shrq(in_out, 32);
 8545   xorl(tmp2, tmp2);
 8546   crc32(tmp2, tmp1, 4);
 8547   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8548   shlq(in1, 1);
 8549   movl(tmp1, in1);
 8550   shrq(in1, 32);
 8551   xorl(tmp2, tmp2);
 8552   crc32(tmp2, tmp1, 4);
 8553   xorl(in1, tmp2);
 8554   xorl(in_out, in1);
 8555   xorl(in_out, in2);
 8556 }
 8557 
 8558 // Set N to predefined value
 8559 // Subtract from a length of a buffer
 8560 // execute in a loop:
 8561 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8562 // for i = 1 to N do
 8563 //  CRC_A = CRC32(CRC_A, A[i])
 8564 //  CRC_B = CRC32(CRC_B, B[i])
 8565 //  CRC_C = CRC32(CRC_C, C[i])
 8566 // end for
 8567 // Recombine
 8568 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8569                                        Register in_out1, Register in_out2, Register in_out3,
 8570                                        Register tmp1, Register tmp2, Register tmp3,
 8571                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8572                                        Register tmp4, Register tmp5,
 8573                                        Register n_tmp6) {
 8574   Label L_processPartitions;
 8575   Label L_processPartition;
 8576   Label L_exit;
 8577 
 8578   bind(L_processPartitions);
 8579   cmpl(in_out1, 3 * size);
 8580   jcc(Assembler::less, L_exit);
 8581     xorl(tmp1, tmp1);
 8582     xorl(tmp2, tmp2);
 8583     movq(tmp3, in_out2);
 8584     addq(tmp3, size);
 8585 
 8586     bind(L_processPartition);
 8587       crc32(in_out3, Address(in_out2, 0), 8);
 8588       crc32(tmp1, Address(in_out2, size), 8);
 8589       crc32(tmp2, Address(in_out2, size * 2), 8);
 8590       addq(in_out2, 8);
 8591       cmpq(in_out2, tmp3);
 8592       jcc(Assembler::less, L_processPartition);
 8593     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8594             w_xtmp1, w_xtmp2, w_xtmp3,
 8595             tmp4, tmp5,
 8596             n_tmp6);
 8597     addq(in_out2, 2 * size);
 8598     subl(in_out1, 3 * size);
 8599     jmp(L_processPartitions);
 8600 
 8601   bind(L_exit);
 8602 }
 8603 #else
 8604 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 8605                                      Register tmp1, Register tmp2, Register tmp3,
 8606                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 8607   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8608   if (n > 0) {
 8609     addl(tmp3, n * 256 * 8);
 8610   }
 8611   //    Q1 = TABLEExt[n][B & 0xFF];
 8612   movl(tmp1, in_out);
 8613   andl(tmp1, 0x000000FF);
 8614   shll(tmp1, 3);
 8615   addl(tmp1, tmp3);
 8616   movq(xtmp1, Address(tmp1, 0));
 8617 
 8618   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8619   movl(tmp2, in_out);
 8620   shrl(tmp2, 8);
 8621   andl(tmp2, 0x000000FF);
 8622   shll(tmp2, 3);
 8623   addl(tmp2, tmp3);
 8624   movq(xtmp2, Address(tmp2, 0));
 8625 
 8626   psllq(xtmp2, 8);
 8627   pxor(xtmp1, xtmp2);
 8628 
 8629   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8630   movl(tmp2, in_out);
 8631   shrl(tmp2, 16);
 8632   andl(tmp2, 0x000000FF);
 8633   shll(tmp2, 3);
 8634   addl(tmp2, tmp3);
 8635   movq(xtmp2, Address(tmp2, 0));
 8636 
 8637   psllq(xtmp2, 16);
 8638   pxor(xtmp1, xtmp2);
 8639 
 8640   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8641   shrl(in_out, 24);
 8642   andl(in_out, 0x000000FF);
 8643   shll(in_out, 3);
 8644   addl(in_out, tmp3);
 8645   movq(xtmp2, Address(in_out, 0));
 8646 
 8647   psllq(xtmp2, 24);
 8648   pxor(xtmp1, xtmp2); // Result in CXMM
 8649   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8650 }
 8651 
 8652 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8653                                       Register in_out,
 8654                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8655                                       XMMRegister w_xtmp2,
 8656                                       Register tmp1,
 8657                                       Register n_tmp2, Register n_tmp3) {
 8658   if (is_pclmulqdq_supported) {
 8659     movdl(w_xtmp1, in_out);
 8660 
 8661     movl(tmp1, const_or_pre_comp_const_index);
 8662     movdl(w_xtmp2, tmp1);
 8663     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8664     // Keep result in XMM since GPR is 32 bit in length
 8665   } else {
 8666     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 8667   }
 8668 }
 8669 
 8670 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8671                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8672                                      Register tmp1, Register tmp2,
 8673                                      Register n_tmp3) {
 8674   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8675   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8676 
 8677   psllq(w_xtmp1, 1);
 8678   movdl(tmp1, w_xtmp1);
 8679   psrlq(w_xtmp1, 32);
 8680   movdl(in_out, w_xtmp1);
 8681 
 8682   xorl(tmp2, tmp2);
 8683   crc32(tmp2, tmp1, 4);
 8684   xorl(in_out, tmp2);
 8685 
 8686   psllq(w_xtmp2, 1);
 8687   movdl(tmp1, w_xtmp2);
 8688   psrlq(w_xtmp2, 32);
 8689   movdl(in1, w_xtmp2);
 8690 
 8691   xorl(tmp2, tmp2);
 8692   crc32(tmp2, tmp1, 4);
 8693   xorl(in1, tmp2);
 8694   xorl(in_out, in1);
 8695   xorl(in_out, in2);
 8696 }
 8697 
 8698 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8699                                        Register in_out1, Register in_out2, Register in_out3,
 8700                                        Register tmp1, Register tmp2, Register tmp3,
 8701                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8702                                        Register tmp4, Register tmp5,
 8703                                        Register n_tmp6) {
 8704   Label L_processPartitions;
 8705   Label L_processPartition;
 8706   Label L_exit;
 8707 
 8708   bind(L_processPartitions);
 8709   cmpl(in_out1, 3 * size);
 8710   jcc(Assembler::less, L_exit);
 8711     xorl(tmp1, tmp1);
 8712     xorl(tmp2, tmp2);
 8713     movl(tmp3, in_out2);
 8714     addl(tmp3, size);
 8715 
 8716     bind(L_processPartition);
 8717       crc32(in_out3, Address(in_out2, 0), 4);
 8718       crc32(tmp1, Address(in_out2, size), 4);
 8719       crc32(tmp2, Address(in_out2, size*2), 4);
 8720       crc32(in_out3, Address(in_out2, 0+4), 4);
 8721       crc32(tmp1, Address(in_out2, size+4), 4);
 8722       crc32(tmp2, Address(in_out2, size*2+4), 4);
 8723       addl(in_out2, 8);
 8724       cmpl(in_out2, tmp3);
 8725       jcc(Assembler::less, L_processPartition);
 8726 
 8727         push(tmp3);
 8728         push(in_out1);
 8729         push(in_out2);
 8730         tmp4 = tmp3;
 8731         tmp5 = in_out1;
 8732         n_tmp6 = in_out2;
 8733 
 8734       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8735             w_xtmp1, w_xtmp2, w_xtmp3,
 8736             tmp4, tmp5,
 8737             n_tmp6);
 8738 
 8739         pop(in_out2);
 8740         pop(in_out1);
 8741         pop(tmp3);
 8742 
 8743     addl(in_out2, 2 * size);
 8744     subl(in_out1, 3 * size);
 8745     jmp(L_processPartitions);
 8746 
 8747   bind(L_exit);
 8748 }
 8749 #endif //LP64
 8750 
 8751 #ifdef _LP64
 8752 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 8753 // Input: A buffer I of L bytes.
 8754 // Output: the CRC32C value of the buffer.
 8755 // Notations:
 8756 // Write L = 24N + r, with N = floor (L/24).
 8757 // r = L mod 24 (0 <= r < 24).
 8758 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 8759 // N quadwords, and R consists of r bytes.
 8760 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 8761 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 8762 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 8763 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 8764 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8765                                           Register tmp1, Register tmp2, Register tmp3,
 8766                                           Register tmp4, Register tmp5, Register tmp6,
 8767                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8768                                           bool is_pclmulqdq_supported) {
 8769   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8770   Label L_wordByWord;
 8771   Label L_byteByByteProlog;
 8772   Label L_byteByByte;
 8773   Label L_exit;
 8774 
 8775   if (is_pclmulqdq_supported ) {
 8776     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8777     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
 8778 
 8779     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8780     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8781 
 8782     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8783     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8784     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 8785   } else {
 8786     const_or_pre_comp_const_index[0] = 1;
 8787     const_or_pre_comp_const_index[1] = 0;
 8788 
 8789     const_or_pre_comp_const_index[2] = 3;
 8790     const_or_pre_comp_const_index[3] = 2;
 8791 
 8792     const_or_pre_comp_const_index[4] = 5;
 8793     const_or_pre_comp_const_index[5] = 4;
 8794    }
 8795   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8796                     in2, in1, in_out,
 8797                     tmp1, tmp2, tmp3,
 8798                     w_xtmp1, w_xtmp2, w_xtmp3,
 8799                     tmp4, tmp5,
 8800                     tmp6);
 8801   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8802                     in2, in1, in_out,
 8803                     tmp1, tmp2, tmp3,
 8804                     w_xtmp1, w_xtmp2, w_xtmp3,
 8805                     tmp4, tmp5,
 8806                     tmp6);
 8807   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8808                     in2, in1, in_out,
 8809                     tmp1, tmp2, tmp3,
 8810                     w_xtmp1, w_xtmp2, w_xtmp3,
 8811                     tmp4, tmp5,
 8812                     tmp6);
 8813   movl(tmp1, in2);
 8814   andl(tmp1, 0x00000007);
 8815   negl(tmp1);
 8816   addl(tmp1, in2);
 8817   addq(tmp1, in1);
 8818 
 8819   cmpq(in1, tmp1);
 8820   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 8821   align(16);
 8822   BIND(L_wordByWord);
 8823     crc32(in_out, Address(in1, 0), 8);
 8824     addq(in1, 8);
 8825     cmpq(in1, tmp1);
 8826     jcc(Assembler::less, L_wordByWord);
 8827 
 8828   BIND(L_byteByByteProlog);
 8829   andl(in2, 0x00000007);
 8830   movl(tmp2, 1);
 8831 
 8832   cmpl(tmp2, in2);
 8833   jccb(Assembler::greater, L_exit);
 8834   BIND(L_byteByByte);
 8835     crc32(in_out, Address(in1, 0), 1);
 8836     incq(in1);
 8837     incl(tmp2);
 8838     cmpl(tmp2, in2);
 8839     jcc(Assembler::lessEqual, L_byteByByte);
 8840 
 8841   BIND(L_exit);
 8842 }
 8843 #else
 8844 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8845                                           Register tmp1, Register  tmp2, Register tmp3,
 8846                                           Register tmp4, Register  tmp5, Register tmp6,
 8847                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8848                                           bool is_pclmulqdq_supported) {
 8849   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8850   Label L_wordByWord;
 8851   Label L_byteByByteProlog;
 8852   Label L_byteByByte;
 8853   Label L_exit;
 8854 
 8855   if (is_pclmulqdq_supported) {
 8856     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8857     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
 8858 
 8859     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8860     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8861 
 8862     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8863     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8864   } else {
 8865     const_or_pre_comp_const_index[0] = 1;
 8866     const_or_pre_comp_const_index[1] = 0;
 8867 
 8868     const_or_pre_comp_const_index[2] = 3;
 8869     const_or_pre_comp_const_index[3] = 2;
 8870 
 8871     const_or_pre_comp_const_index[4] = 5;
 8872     const_or_pre_comp_const_index[5] = 4;
 8873   }
 8874   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8875                     in2, in1, in_out,
 8876                     tmp1, tmp2, tmp3,
 8877                     w_xtmp1, w_xtmp2, w_xtmp3,
 8878                     tmp4, tmp5,
 8879                     tmp6);
 8880   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8881                     in2, in1, in_out,
 8882                     tmp1, tmp2, tmp3,
 8883                     w_xtmp1, w_xtmp2, w_xtmp3,
 8884                     tmp4, tmp5,
 8885                     tmp6);
 8886   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8887                     in2, in1, in_out,
 8888                     tmp1, tmp2, tmp3,
 8889                     w_xtmp1, w_xtmp2, w_xtmp3,
 8890                     tmp4, tmp5,
 8891                     tmp6);
 8892   movl(tmp1, in2);
 8893   andl(tmp1, 0x00000007);
 8894   negl(tmp1);
 8895   addl(tmp1, in2);
 8896   addl(tmp1, in1);
 8897 
 8898   BIND(L_wordByWord);
 8899   cmpl(in1, tmp1);
 8900   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 8901     crc32(in_out, Address(in1,0), 4);
 8902     addl(in1, 4);
 8903     jmp(L_wordByWord);
 8904 
 8905   BIND(L_byteByByteProlog);
 8906   andl(in2, 0x00000007);
 8907   movl(tmp2, 1);
 8908 
 8909   BIND(L_byteByByte);
 8910   cmpl(tmp2, in2);
 8911   jccb(Assembler::greater, L_exit);
 8912     movb(tmp1, Address(in1, 0));
 8913     crc32(in_out, tmp1, 1);
 8914     incl(in1);
 8915     incl(tmp2);
 8916     jmp(L_byteByByte);
 8917 
 8918   BIND(L_exit);
 8919 }
 8920 #endif // LP64
 8921 #undef BIND
 8922 #undef BLOCK_COMMENT
 8923 
 8924 // Compress char[] array to byte[].
 8925 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
 8926 // Return the array length if every element in array can be encoded,
 8927 // otherwise, the index of first non-latin1 (> 0xff) character.
 8928 //   @IntrinsicCandidate
 8929 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 8930 //     for (int i = 0; i < len; i++) {
 8931 //       char c = src[srcOff];
 8932 //       if (c > 0xff) {
 8933 //           return i;  // return index of non-latin1 char
 8934 //       }
 8935 //       dst[dstOff] = (byte)c;
 8936 //       srcOff++;
 8937 //       dstOff++;
 8938 //     }
 8939 //     return len;
 8940 //   }
 8941 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 8942   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 8943   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 8944   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 8945   Label copy_chars_loop, done, reset_sp, copy_tail;
 8946 
 8947   // rsi: src
 8948   // rdi: dst
 8949   // rdx: len
 8950   // rcx: tmp5
 8951   // rax: result
 8952 
 8953   // rsi holds start addr of source char[] to be compressed
 8954   // rdi holds start addr of destination byte[]
 8955   // rdx holds length
 8956 
 8957   assert(len != result, "");
 8958 
 8959   // save length for return
 8960   movl(result, len);
 8961 
 8962   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 8963     VM_Version::supports_avx512vlbw() &&
 8964     VM_Version::supports_bmi2()) {
 8965 
 8966     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
 8967 
 8968     // alignment
 8969     Label post_alignment;
 8970 
 8971     // if length of the string is less than 32, handle it the old fashioned way
 8972     testl(len, -32);
 8973     jcc(Assembler::zero, below_threshold);
 8974 
 8975     // First check whether a character is compressible ( <= 0xFF).
 8976     // Create mask to test for Unicode chars inside zmm vector
 8977     movl(tmp5, 0x00FF);
 8978     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
 8979 
 8980     testl(len, -64);
 8981     jccb(Assembler::zero, post_alignment);
 8982 
 8983     movl(tmp5, dst);
 8984     andl(tmp5, (32 - 1));
 8985     negl(tmp5);
 8986     andl(tmp5, (32 - 1));
 8987 
 8988     // bail out when there is nothing to be done
 8989     testl(tmp5, 0xFFFFFFFF);
 8990     jccb(Assembler::zero, post_alignment);
 8991 
 8992     // ~(~0 << len), where len is the # of remaining elements to process
 8993     movl(len, 0xFFFFFFFF);
 8994     shlxl(len, len, tmp5);
 8995     notl(len);
 8996     kmovdl(mask2, len);
 8997     movl(len, result);
 8998 
 8999     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9000     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9001     ktestd(mask1, mask2);
 9002     jcc(Assembler::carryClear, copy_tail);
 9003 
 9004     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9005 
 9006     addptr(src, tmp5);
 9007     addptr(src, tmp5);
 9008     addptr(dst, tmp5);
 9009     subl(len, tmp5);
 9010 
 9011     bind(post_alignment);
 9012     // end of alignment
 9013 
 9014     movl(tmp5, len);
 9015     andl(tmp5, (32 - 1));    // tail count (in chars)
 9016     andl(len, ~(32 - 1));    // vector count (in chars)
 9017     jccb(Assembler::zero, copy_loop_tail);
 9018 
 9019     lea(src, Address(src, len, Address::times_2));
 9020     lea(dst, Address(dst, len, Address::times_1));
 9021     negptr(len);
 9022 
 9023     bind(copy_32_loop);
 9024     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9025     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9026     kortestdl(mask1, mask1);
 9027     jccb(Assembler::carryClear, reset_for_copy_tail);
 9028 
 9029     // All elements in current processed chunk are valid candidates for
 9030     // compression. Write a truncated byte elements to the memory.
 9031     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9032     addptr(len, 32);
 9033     jccb(Assembler::notZero, copy_32_loop);
 9034 
 9035     bind(copy_loop_tail);
 9036     // bail out when there is nothing to be done
 9037     testl(tmp5, 0xFFFFFFFF);
 9038     jcc(Assembler::zero, done);
 9039 
 9040     movl(len, tmp5);
 9041 
 9042     // ~(~0 << len), where len is the # of remaining elements to process
 9043     movl(tmp5, 0xFFFFFFFF);
 9044     shlxl(tmp5, tmp5, len);
 9045     notl(tmp5);
 9046 
 9047     kmovdl(mask2, tmp5);
 9048 
 9049     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9050     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9051     ktestd(mask1, mask2);
 9052     jcc(Assembler::carryClear, copy_tail);
 9053 
 9054     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9055     jmp(done);
 9056 
 9057     bind(reset_for_copy_tail);
 9058     lea(src, Address(src, tmp5, Address::times_2));
 9059     lea(dst, Address(dst, tmp5, Address::times_1));
 9060     subptr(len, tmp5);
 9061     jmp(copy_chars_loop);
 9062 
 9063     bind(below_threshold);
 9064   }
 9065 
 9066   if (UseSSE42Intrinsics) {
 9067     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
 9068 
 9069     // vectored compression
 9070     testl(len, 0xfffffff8);
 9071     jcc(Assembler::zero, copy_tail);
 9072 
 9073     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9074     movdl(tmp1Reg, tmp5);
 9075     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9076 
 9077     andl(len, 0xfffffff0);
 9078     jccb(Assembler::zero, copy_16);
 9079 
 9080     // compress 16 chars per iter
 9081     pxor(tmp4Reg, tmp4Reg);
 9082 
 9083     lea(src, Address(src, len, Address::times_2));
 9084     lea(dst, Address(dst, len, Address::times_1));
 9085     negptr(len);
 9086 
 9087     bind(copy_32_loop);
 9088     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9089     por(tmp4Reg, tmp2Reg);
 9090     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9091     por(tmp4Reg, tmp3Reg);
 9092     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9093     jccb(Assembler::notZero, reset_for_copy_tail);
 9094     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9095     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9096     addptr(len, 16);
 9097     jccb(Assembler::notZero, copy_32_loop);
 9098 
 9099     // compress next vector of 8 chars (if any)
 9100     bind(copy_16);
 9101     // len = 0
 9102     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
 9103     jccb(Assembler::zero, copy_tail_sse);
 9104 
 9105     pxor(tmp3Reg, tmp3Reg);
 9106 
 9107     movdqu(tmp2Reg, Address(src, 0));
 9108     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9109     jccb(Assembler::notZero, reset_for_copy_tail);
 9110     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9111     movq(Address(dst, 0), tmp2Reg);
 9112     addptr(src, 16);
 9113     addptr(dst, 8);
 9114     jmpb(copy_tail_sse);
 9115 
 9116     bind(reset_for_copy_tail);
 9117     movl(tmp5, result);
 9118     andl(tmp5, 0x0000000f);
 9119     lea(src, Address(src, tmp5, Address::times_2));
 9120     lea(dst, Address(dst, tmp5, Address::times_1));
 9121     subptr(len, tmp5);
 9122     jmpb(copy_chars_loop);
 9123 
 9124     bind(copy_tail_sse);
 9125     movl(len, result);
 9126     andl(len, 0x00000007);    // tail count (in chars)
 9127   }
 9128   // compress 1 char per iter
 9129   bind(copy_tail);
 9130   testl(len, len);
 9131   jccb(Assembler::zero, done);
 9132   lea(src, Address(src, len, Address::times_2));
 9133   lea(dst, Address(dst, len, Address::times_1));
 9134   negptr(len);
 9135 
 9136   bind(copy_chars_loop);
 9137   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
 9138   testl(tmp5, 0xff00);      // check if Unicode char
 9139   jccb(Assembler::notZero, reset_sp);
 9140   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
 9141   increment(len);
 9142   jccb(Assembler::notZero, copy_chars_loop);
 9143 
 9144   // add len then return (len will be zero if compress succeeded, otherwise negative)
 9145   bind(reset_sp);
 9146   addl(result, len);
 9147 
 9148   bind(done);
 9149 }
 9150 
 9151 // Inflate byte[] array to char[].
 9152 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9153 //   @IntrinsicCandidate
 9154 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9155 //     for (int i = 0; i < len; i++) {
 9156 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9157 //     }
 9158 //   }
 9159 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9160   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9161   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9162   // rsi: src
 9163   // rdi: dst
 9164   // rdx: len
 9165   // rcx: tmp2
 9166 
 9167   // rsi holds start addr of source byte[] to be inflated
 9168   // rdi holds start addr of destination char[]
 9169   // rdx holds length
 9170   assert_different_registers(src, dst, len, tmp2);
 9171   movl(tmp2, len);
 9172   if ((UseAVX > 2) && // AVX512
 9173     VM_Version::supports_avx512vlbw() &&
 9174     VM_Version::supports_bmi2()) {
 9175 
 9176     Label copy_32_loop, copy_tail;
 9177     Register tmp3_aliased = len;
 9178 
 9179     // if length of the string is less than 16, handle it in an old fashioned way
 9180     testl(len, -16);
 9181     jcc(Assembler::zero, below_threshold);
 9182 
 9183     testl(len, -1 * AVX3Threshold);
 9184     jcc(Assembler::zero, avx3_threshold);
 9185 
 9186     // In order to use only one arithmetic operation for the main loop we use
 9187     // this pre-calculation
 9188     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9189     andl(len, -32);     // vector count
 9190     jccb(Assembler::zero, copy_tail);
 9191 
 9192     lea(src, Address(src, len, Address::times_1));
 9193     lea(dst, Address(dst, len, Address::times_2));
 9194     negptr(len);
 9195 
 9196 
 9197     // inflate 32 chars per iter
 9198     bind(copy_32_loop);
 9199     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9200     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9201     addptr(len, 32);
 9202     jcc(Assembler::notZero, copy_32_loop);
 9203 
 9204     bind(copy_tail);
 9205     // bail out when there is nothing to be done
 9206     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9207     jcc(Assembler::zero, done);
 9208 
 9209     // ~(~0 << length), where length is the # of remaining elements to process
 9210     movl(tmp3_aliased, -1);
 9211     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9212     notl(tmp3_aliased);
 9213     kmovdl(mask, tmp3_aliased);
 9214     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9215     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9216 
 9217     jmp(done);
 9218     bind(avx3_threshold);
 9219   }
 9220   if (UseSSE42Intrinsics) {
 9221     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9222 
 9223     if (UseAVX > 1) {
 9224       andl(tmp2, (16 - 1));
 9225       andl(len, -16);
 9226       jccb(Assembler::zero, copy_new_tail);
 9227     } else {
 9228       andl(tmp2, 0x00000007);   // tail count (in chars)
 9229       andl(len, 0xfffffff8);    // vector count (in chars)
 9230       jccb(Assembler::zero, copy_tail);
 9231     }
 9232 
 9233     // vectored inflation
 9234     lea(src, Address(src, len, Address::times_1));
 9235     lea(dst, Address(dst, len, Address::times_2));
 9236     negptr(len);
 9237 
 9238     if (UseAVX > 1) {
 9239       bind(copy_16_loop);
 9240       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9241       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9242       addptr(len, 16);
 9243       jcc(Assembler::notZero, copy_16_loop);
 9244 
 9245       bind(below_threshold);
 9246       bind(copy_new_tail);
 9247       movl(len, tmp2);
 9248       andl(tmp2, 0x00000007);
 9249       andl(len, 0xFFFFFFF8);
 9250       jccb(Assembler::zero, copy_tail);
 9251 
 9252       pmovzxbw(tmp1, Address(src, 0));
 9253       movdqu(Address(dst, 0), tmp1);
 9254       addptr(src, 8);
 9255       addptr(dst, 2 * 8);
 9256 
 9257       jmp(copy_tail, true);
 9258     }
 9259 
 9260     // inflate 8 chars per iter
 9261     bind(copy_8_loop);
 9262     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9263     movdqu(Address(dst, len, Address::times_2), tmp1);
 9264     addptr(len, 8);
 9265     jcc(Assembler::notZero, copy_8_loop);
 9266 
 9267     bind(copy_tail);
 9268     movl(len, tmp2);
 9269 
 9270     cmpl(len, 4);
 9271     jccb(Assembler::less, copy_bytes);
 9272 
 9273     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9274     pmovzxbw(tmp1, tmp1);
 9275     movq(Address(dst, 0), tmp1);
 9276     subptr(len, 4);
 9277     addptr(src, 4);
 9278     addptr(dst, 8);
 9279 
 9280     bind(copy_bytes);
 9281   } else {
 9282     bind(below_threshold);
 9283   }
 9284 
 9285   testl(len, len);
 9286   jccb(Assembler::zero, done);
 9287   lea(src, Address(src, len, Address::times_1));
 9288   lea(dst, Address(dst, len, Address::times_2));
 9289   negptr(len);
 9290 
 9291   // inflate 1 char per iter
 9292   bind(copy_chars_loop);
 9293   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9294   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9295   increment(len);
 9296   jcc(Assembler::notZero, copy_chars_loop);
 9297 
 9298   bind(done);
 9299 }
 9300 
 9301 
 9302 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9303   switch(type) {
 9304     case T_BYTE:
 9305     case T_BOOLEAN:
 9306       evmovdqub(dst, kmask, src, merge, vector_len);
 9307       break;
 9308     case T_CHAR:
 9309     case T_SHORT:
 9310       evmovdquw(dst, kmask, src, merge, vector_len);
 9311       break;
 9312     case T_INT:
 9313     case T_FLOAT:
 9314       evmovdqul(dst, kmask, src, merge, vector_len);
 9315       break;
 9316     case T_LONG:
 9317     case T_DOUBLE:
 9318       evmovdquq(dst, kmask, src, merge, vector_len);
 9319       break;
 9320     default:
 9321       fatal("Unexpected type argument %s", type2name(type));
 9322       break;
 9323   }
 9324 }
 9325 
 9326 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9327   switch(type) {
 9328     case T_BYTE:
 9329     case T_BOOLEAN:
 9330       evmovdqub(dst, kmask, src, merge, vector_len);
 9331       break;
 9332     case T_CHAR:
 9333     case T_SHORT:
 9334       evmovdquw(dst, kmask, src, merge, vector_len);
 9335       break;
 9336     case T_INT:
 9337     case T_FLOAT:
 9338       evmovdqul(dst, kmask, src, merge, vector_len);
 9339       break;
 9340     case T_LONG:
 9341     case T_DOUBLE:
 9342       evmovdquq(dst, kmask, src, merge, vector_len);
 9343       break;
 9344     default:
 9345       fatal("Unexpected type argument %s", type2name(type));
 9346       break;
 9347   }
 9348 }
 9349 
 9350 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9351   switch(masklen) {
 9352     case 2:
 9353        knotbl(dst, src);
 9354        movl(rtmp, 3);
 9355        kmovbl(ktmp, rtmp);
 9356        kandbl(dst, ktmp, dst);
 9357        break;
 9358     case 4:
 9359        knotbl(dst, src);
 9360        movl(rtmp, 15);
 9361        kmovbl(ktmp, rtmp);
 9362        kandbl(dst, ktmp, dst);
 9363        break;
 9364     case 8:
 9365        knotbl(dst, src);
 9366        break;
 9367     case 16:
 9368        knotwl(dst, src);
 9369        break;
 9370     case 32:
 9371        knotdl(dst, src);
 9372        break;
 9373     case 64:
 9374        knotql(dst, src);
 9375        break;
 9376     default:
 9377       fatal("Unexpected vector length %d", masklen);
 9378       break;
 9379   }
 9380 }
 9381 
 9382 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9383   switch(type) {
 9384     case T_BOOLEAN:
 9385     case T_BYTE:
 9386        kandbl(dst, src1, src2);
 9387        break;
 9388     case T_CHAR:
 9389     case T_SHORT:
 9390        kandwl(dst, src1, src2);
 9391        break;
 9392     case T_INT:
 9393     case T_FLOAT:
 9394        kanddl(dst, src1, src2);
 9395        break;
 9396     case T_LONG:
 9397     case T_DOUBLE:
 9398        kandql(dst, src1, src2);
 9399        break;
 9400     default:
 9401       fatal("Unexpected type argument %s", type2name(type));
 9402       break;
 9403   }
 9404 }
 9405 
 9406 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9407   switch(type) {
 9408     case T_BOOLEAN:
 9409     case T_BYTE:
 9410        korbl(dst, src1, src2);
 9411        break;
 9412     case T_CHAR:
 9413     case T_SHORT:
 9414        korwl(dst, src1, src2);
 9415        break;
 9416     case T_INT:
 9417     case T_FLOAT:
 9418        kordl(dst, src1, src2);
 9419        break;
 9420     case T_LONG:
 9421     case T_DOUBLE:
 9422        korql(dst, src1, src2);
 9423        break;
 9424     default:
 9425       fatal("Unexpected type argument %s", type2name(type));
 9426       break;
 9427   }
 9428 }
 9429 
 9430 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9431   switch(type) {
 9432     case T_BOOLEAN:
 9433     case T_BYTE:
 9434        kxorbl(dst, src1, src2);
 9435        break;
 9436     case T_CHAR:
 9437     case T_SHORT:
 9438        kxorwl(dst, src1, src2);
 9439        break;
 9440     case T_INT:
 9441     case T_FLOAT:
 9442        kxordl(dst, src1, src2);
 9443        break;
 9444     case T_LONG:
 9445     case T_DOUBLE:
 9446        kxorql(dst, src1, src2);
 9447        break;
 9448     default:
 9449       fatal("Unexpected type argument %s", type2name(type));
 9450       break;
 9451   }
 9452 }
 9453 
 9454 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9455   switch(type) {
 9456     case T_BOOLEAN:
 9457     case T_BYTE:
 9458       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9459     case T_CHAR:
 9460     case T_SHORT:
 9461       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9462     case T_INT:
 9463     case T_FLOAT:
 9464       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9465     case T_LONG:
 9466     case T_DOUBLE:
 9467       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9468     default:
 9469       fatal("Unexpected type argument %s", type2name(type)); break;
 9470   }
 9471 }
 9472 
 9473 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9474   switch(type) {
 9475     case T_BOOLEAN:
 9476     case T_BYTE:
 9477       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9478     case T_CHAR:
 9479     case T_SHORT:
 9480       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9481     case T_INT:
 9482     case T_FLOAT:
 9483       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9484     case T_LONG:
 9485     case T_DOUBLE:
 9486       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9487     default:
 9488       fatal("Unexpected type argument %s", type2name(type)); break;
 9489   }
 9490 }
 9491 
 9492 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9493   switch(type) {
 9494     case T_BYTE:
 9495       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9496     case T_SHORT:
 9497       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9498     case T_INT:
 9499       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9500     case T_LONG:
 9501       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9502     default:
 9503       fatal("Unexpected type argument %s", type2name(type)); break;
 9504   }
 9505 }
 9506 
 9507 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9508   switch(type) {
 9509     case T_BYTE:
 9510       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9511     case T_SHORT:
 9512       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9513     case T_INT:
 9514       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9515     case T_LONG:
 9516       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9517     default:
 9518       fatal("Unexpected type argument %s", type2name(type)); break;
 9519   }
 9520 }
 9521 
 9522 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9523   switch(type) {
 9524     case T_BYTE:
 9525       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9526     case T_SHORT:
 9527       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9528     case T_INT:
 9529       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9530     case T_LONG:
 9531       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9532     default:
 9533       fatal("Unexpected type argument %s", type2name(type)); break;
 9534   }
 9535 }
 9536 
 9537 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9538   switch(type) {
 9539     case T_BYTE:
 9540       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9541     case T_SHORT:
 9542       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9543     case T_INT:
 9544       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9545     case T_LONG:
 9546       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9547     default:
 9548       fatal("Unexpected type argument %s", type2name(type)); break;
 9549   }
 9550 }
 9551 
 9552 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9553   switch(type) {
 9554     case T_INT:
 9555       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9556     case T_LONG:
 9557       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9558     default:
 9559       fatal("Unexpected type argument %s", type2name(type)); break;
 9560   }
 9561 }
 9562 
 9563 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9564   switch(type) {
 9565     case T_INT:
 9566       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9567     case T_LONG:
 9568       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9569     default:
 9570       fatal("Unexpected type argument %s", type2name(type)); break;
 9571   }
 9572 }
 9573 
 9574 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9575   switch(type) {
 9576     case T_INT:
 9577       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9578     case T_LONG:
 9579       evporq(dst, mask, nds, src, merge, vector_len); break;
 9580     default:
 9581       fatal("Unexpected type argument %s", type2name(type)); break;
 9582   }
 9583 }
 9584 
 9585 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9586   switch(type) {
 9587     case T_INT:
 9588       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9589     case T_LONG:
 9590       evporq(dst, mask, nds, src, merge, vector_len); break;
 9591     default:
 9592       fatal("Unexpected type argument %s", type2name(type)); break;
 9593   }
 9594 }
 9595 
 9596 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9597   switch(type) {
 9598     case T_INT:
 9599       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9600     case T_LONG:
 9601       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9602     default:
 9603       fatal("Unexpected type argument %s", type2name(type)); break;
 9604   }
 9605 }
 9606 
 9607 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9608   switch(type) {
 9609     case T_INT:
 9610       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9611     case T_LONG:
 9612       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9613     default:
 9614       fatal("Unexpected type argument %s", type2name(type)); break;
 9615   }
 9616 }
 9617 
 9618 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
 9619   switch(masklen) {
 9620     case 8:
 9621        kortestbl(src1, src2);
 9622        break;
 9623     case 16:
 9624        kortestwl(src1, src2);
 9625        break;
 9626     case 32:
 9627        kortestdl(src1, src2);
 9628        break;
 9629     case 64:
 9630        kortestql(src1, src2);
 9631        break;
 9632     default:
 9633       fatal("Unexpected mask length %d", masklen);
 9634       break;
 9635   }
 9636 }
 9637 
 9638 
 9639 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
 9640   switch(masklen)  {
 9641     case 8:
 9642        ktestbl(src1, src2);
 9643        break;
 9644     case 16:
 9645        ktestwl(src1, src2);
 9646        break;
 9647     case 32:
 9648        ktestdl(src1, src2);
 9649        break;
 9650     case 64:
 9651        ktestql(src1, src2);
 9652        break;
 9653     default:
 9654       fatal("Unexpected mask length %d", masklen);
 9655       break;
 9656   }
 9657 }
 9658 
 9659 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9660   switch(type) {
 9661     case T_INT:
 9662       evprold(dst, mask, src, shift, merge, vlen_enc); break;
 9663     case T_LONG:
 9664       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
 9665     default:
 9666       fatal("Unexpected type argument %s", type2name(type)); break;
 9667       break;
 9668   }
 9669 }
 9670 
 9671 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9672   switch(type) {
 9673     case T_INT:
 9674       evprord(dst, mask, src, shift, merge, vlen_enc); break;
 9675     case T_LONG:
 9676       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
 9677     default:
 9678       fatal("Unexpected type argument %s", type2name(type)); break;
 9679   }
 9680 }
 9681 
 9682 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9683   switch(type) {
 9684     case T_INT:
 9685       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9686     case T_LONG:
 9687       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9688     default:
 9689       fatal("Unexpected type argument %s", type2name(type)); break;
 9690   }
 9691 }
 9692 
 9693 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9694   switch(type) {
 9695     case T_INT:
 9696       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9697     case T_LONG:
 9698       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9699     default:
 9700       fatal("Unexpected type argument %s", type2name(type)); break;
 9701   }
 9702 }
 9703 
 9704 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9705   assert(rscratch != noreg || always_reachable(src), "missing");
 9706 
 9707   if (reachable(src)) {
 9708     evpandq(dst, nds, as_Address(src), vector_len);
 9709   } else {
 9710     lea(rscratch, src);
 9711     evpandq(dst, nds, Address(rscratch, 0), vector_len);
 9712   }
 9713 }
 9714 
 9715 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 9716   assert(rscratch != noreg || always_reachable(src), "missing");
 9717 
 9718   if (reachable(src)) {
 9719     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
 9720   } else {
 9721     lea(rscratch, src);
 9722     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 9723   }
 9724 }
 9725 
 9726 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9727   assert(rscratch != noreg || always_reachable(src), "missing");
 9728 
 9729   if (reachable(src)) {
 9730     evporq(dst, nds, as_Address(src), vector_len);
 9731   } else {
 9732     lea(rscratch, src);
 9733     evporq(dst, nds, Address(rscratch, 0), vector_len);
 9734   }
 9735 }
 9736 
 9737 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9738   assert(rscratch != noreg || always_reachable(src), "missing");
 9739 
 9740   if (reachable(src)) {
 9741     vpshufb(dst, nds, as_Address(src), vector_len);
 9742   } else {
 9743     lea(rscratch, src);
 9744     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
 9745   }
 9746 }
 9747 
 9748 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9749   assert(rscratch != noreg || always_reachable(src), "missing");
 9750 
 9751   if (reachable(src)) {
 9752     Assembler::vpor(dst, nds, as_Address(src), vector_len);
 9753   } else {
 9754     lea(rscratch, src);
 9755     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
 9756   }
 9757 }
 9758 
 9759 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
 9760   assert(rscratch != noreg || always_reachable(src3), "missing");
 9761 
 9762   if (reachable(src3)) {
 9763     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
 9764   } else {
 9765     lea(rscratch, src3);
 9766     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
 9767   }
 9768 }
 9769 
 9770 #if COMPILER2_OR_JVMCI
 9771 
 9772 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
 9773                                  Register length, Register temp, int vec_enc) {
 9774   // Computing mask for predicated vector store.
 9775   movptr(temp, -1);
 9776   bzhiq(temp, temp, length);
 9777   kmov(mask, temp);
 9778   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
 9779 }
 9780 
 9781 // Set memory operation for length "less than" 64 bytes.
 9782 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
 9783                                        XMMRegister xmm, KRegister mask, Register length,
 9784                                        Register temp, bool use64byteVector) {
 9785   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9786   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9787   if (!use64byteVector) {
 9788     fill32(dst, disp, xmm);
 9789     subptr(length, 32 >> shift);
 9790     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
 9791   } else {
 9792     assert(MaxVectorSize == 64, "vector length != 64");
 9793     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
 9794   }
 9795 }
 9796 
 9797 
 9798 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
 9799                                        XMMRegister xmm, KRegister mask, Register length,
 9800                                        Register temp) {
 9801   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9802   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9803   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
 9804 }
 9805 
 9806 
 9807 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
 9808   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9809   vmovdqu(dst, xmm);
 9810 }
 9811 
 9812 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
 9813   fill32(Address(dst, disp), xmm);
 9814 }
 9815 
 9816 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
 9817   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9818   if (!use64byteVector) {
 9819     fill32(dst, xmm);
 9820     fill32(dst.plus_disp(32), xmm);
 9821   } else {
 9822     evmovdquq(dst, xmm, Assembler::AVX_512bit);
 9823   }
 9824 }
 9825 
 9826 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
 9827   fill64(Address(dst, disp), xmm, use64byteVector);
 9828 }
 9829 
 9830 #ifdef _LP64
 9831 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
 9832                                         Register count, Register rtmp, XMMRegister xtmp) {
 9833   Label L_exit;
 9834   Label L_fill_start;
 9835   Label L_fill_64_bytes;
 9836   Label L_fill_96_bytes;
 9837   Label L_fill_128_bytes;
 9838   Label L_fill_128_bytes_loop;
 9839   Label L_fill_128_loop_header;
 9840   Label L_fill_128_bytes_loop_header;
 9841   Label L_fill_128_bytes_loop_pre_header;
 9842   Label L_fill_zmm_sequence;
 9843 
 9844   int shift = -1;
 9845   int avx3threshold = VM_Version::avx3_threshold();
 9846   switch(type) {
 9847     case T_BYTE:  shift = 0;
 9848       break;
 9849     case T_SHORT: shift = 1;
 9850       break;
 9851     case T_INT:   shift = 2;
 9852       break;
 9853     /* Uncomment when LONG fill stubs are supported.
 9854     case T_LONG:  shift = 3;
 9855       break;
 9856     */
 9857     default:
 9858       fatal("Unhandled type: %s\n", type2name(type));
 9859   }
 9860 
 9861   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
 9862 
 9863     if (MaxVectorSize == 64) {
 9864       cmpq(count, avx3threshold >> shift);
 9865       jcc(Assembler::greater, L_fill_zmm_sequence);
 9866     }
 9867 
 9868     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
 9869 
 9870     bind(L_fill_start);
 9871 
 9872     cmpq(count, 32 >> shift);
 9873     jccb(Assembler::greater, L_fill_64_bytes);
 9874     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9875     jmp(L_exit);
 9876 
 9877     bind(L_fill_64_bytes);
 9878     cmpq(count, 64 >> shift);
 9879     jccb(Assembler::greater, L_fill_96_bytes);
 9880     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9881     jmp(L_exit);
 9882 
 9883     bind(L_fill_96_bytes);
 9884     cmpq(count, 96 >> shift);
 9885     jccb(Assembler::greater, L_fill_128_bytes);
 9886     fill64(to, 0, xtmp);
 9887     subq(count, 64 >> shift);
 9888     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
 9889     jmp(L_exit);
 9890 
 9891     bind(L_fill_128_bytes);
 9892     cmpq(count, 128 >> shift);
 9893     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
 9894     fill64(to, 0, xtmp);
 9895     fill32(to, 64, xtmp);
 9896     subq(count, 96 >> shift);
 9897     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
 9898     jmp(L_exit);
 9899 
 9900     bind(L_fill_128_bytes_loop_pre_header);
 9901     {
 9902       mov(rtmp, to);
 9903       andq(rtmp, 31);
 9904       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
 9905       negq(rtmp);
 9906       addq(rtmp, 32);
 9907       mov64(r8, -1L);
 9908       bzhiq(r8, r8, rtmp);
 9909       kmovql(k2, r8);
 9910       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
 9911       addq(to, rtmp);
 9912       shrq(rtmp, shift);
 9913       subq(count, rtmp);
 9914     }
 9915 
 9916     cmpq(count, 128 >> shift);
 9917     jcc(Assembler::less, L_fill_start);
 9918 
 9919     bind(L_fill_128_bytes_loop_header);
 9920     subq(count, 128 >> shift);
 9921 
 9922     align32();
 9923     bind(L_fill_128_bytes_loop);
 9924       fill64(to, 0, xtmp);
 9925       fill64(to, 64, xtmp);
 9926       addq(to, 128);
 9927       subq(count, 128 >> shift);
 9928       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
 9929 
 9930     addq(count, 128 >> shift);
 9931     jcc(Assembler::zero, L_exit);
 9932     jmp(L_fill_start);
 9933   }
 9934 
 9935   if (MaxVectorSize == 64) {
 9936     // Sequence using 64 byte ZMM register.
 9937     Label L_fill_128_bytes_zmm;
 9938     Label L_fill_192_bytes_zmm;
 9939     Label L_fill_192_bytes_loop_zmm;
 9940     Label L_fill_192_bytes_loop_header_zmm;
 9941     Label L_fill_192_bytes_loop_pre_header_zmm;
 9942     Label L_fill_start_zmm_sequence;
 9943 
 9944     bind(L_fill_zmm_sequence);
 9945     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
 9946 
 9947     bind(L_fill_start_zmm_sequence);
 9948     cmpq(count, 64 >> shift);
 9949     jccb(Assembler::greater, L_fill_128_bytes_zmm);
 9950     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
 9951     jmp(L_exit);
 9952 
 9953     bind(L_fill_128_bytes_zmm);
 9954     cmpq(count, 128 >> shift);
 9955     jccb(Assembler::greater, L_fill_192_bytes_zmm);
 9956     fill64(to, 0, xtmp, true);
 9957     subq(count, 64 >> shift);
 9958     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
 9959     jmp(L_exit);
 9960 
 9961     bind(L_fill_192_bytes_zmm);
 9962     cmpq(count, 192 >> shift);
 9963     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
 9964     fill64(to, 0, xtmp, true);
 9965     fill64(to, 64, xtmp, true);
 9966     subq(count, 128 >> shift);
 9967     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
 9968     jmp(L_exit);
 9969 
 9970     bind(L_fill_192_bytes_loop_pre_header_zmm);
 9971     {
 9972       movq(rtmp, to);
 9973       andq(rtmp, 63);
 9974       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
 9975       negq(rtmp);
 9976       addq(rtmp, 64);
 9977       mov64(r8, -1L);
 9978       bzhiq(r8, r8, rtmp);
 9979       kmovql(k2, r8);
 9980       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
 9981       addq(to, rtmp);
 9982       shrq(rtmp, shift);
 9983       subq(count, rtmp);
 9984     }
 9985 
 9986     cmpq(count, 192 >> shift);
 9987     jcc(Assembler::less, L_fill_start_zmm_sequence);
 9988 
 9989     bind(L_fill_192_bytes_loop_header_zmm);
 9990     subq(count, 192 >> shift);
 9991 
 9992     align32();
 9993     bind(L_fill_192_bytes_loop_zmm);
 9994       fill64(to, 0, xtmp, true);
 9995       fill64(to, 64, xtmp, true);
 9996       fill64(to, 128, xtmp, true);
 9997       addq(to, 192);
 9998       subq(count, 192 >> shift);
 9999       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10000 
10001     addq(count, 192 >> shift);
10002     jcc(Assembler::zero, L_exit);
10003     jmp(L_fill_start_zmm_sequence);
10004   }
10005   bind(L_exit);
10006 }
10007 #endif
10008 #endif //COMPILER2_OR_JVMCI
10009 
10010 
10011 #ifdef _LP64
10012 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10013   Label done;
10014   cvttss2sil(dst, src);
10015   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10016   cmpl(dst, 0x80000000); // float_sign_flip
10017   jccb(Assembler::notEqual, done);
10018   subptr(rsp, 8);
10019   movflt(Address(rsp, 0), src);
10020   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10021   pop(dst);
10022   bind(done);
10023 }
10024 
10025 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10026   Label done;
10027   cvttsd2sil(dst, src);
10028   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10029   cmpl(dst, 0x80000000); // float_sign_flip
10030   jccb(Assembler::notEqual, done);
10031   subptr(rsp, 8);
10032   movdbl(Address(rsp, 0), src);
10033   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10034   pop(dst);
10035   bind(done);
10036 }
10037 
10038 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10039   Label done;
10040   cvttss2siq(dst, src);
10041   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10042   jccb(Assembler::notEqual, done);
10043   subptr(rsp, 8);
10044   movflt(Address(rsp, 0), src);
10045   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10046   pop(dst);
10047   bind(done);
10048 }
10049 
10050 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10051   // Following code is line by line assembly translation rounding algorithm.
10052   // Please refer to java.lang.Math.round(float) algorithm for details.
10053   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10054   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10055   const int32_t FloatConsts_EXP_BIAS = 127;
10056   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10057   const int32_t MINUS_32 = 0xFFFFFFE0;
10058   Label L_special_case, L_block1, L_exit;
10059   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10060   movdl(dst, src);
10061   andl(dst, rtmp);
10062   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10063   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10064   subl(rtmp, dst);
10065   movl(rcx, rtmp);
10066   movl(dst, MINUS_32);
10067   testl(rtmp, dst);
10068   jccb(Assembler::notEqual, L_special_case);
10069   movdl(dst, src);
10070   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10071   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10072   movdl(rtmp, src);
10073   testl(rtmp, rtmp);
10074   jccb(Assembler::greaterEqual, L_block1);
10075   negl(dst);
10076   bind(L_block1);
10077   sarl(dst);
10078   addl(dst, 0x1);
10079   sarl(dst, 0x1);
10080   jmp(L_exit);
10081   bind(L_special_case);
10082   convert_f2i(dst, src);
10083   bind(L_exit);
10084 }
10085 
10086 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10087   // Following code is line by line assembly translation rounding algorithm.
10088   // Please refer to java.lang.Math.round(double) algorithm for details.
10089   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10090   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10091   const int64_t DoubleConsts_EXP_BIAS = 1023;
10092   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10093   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10094   Label L_special_case, L_block1, L_exit;
10095   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10096   movq(dst, src);
10097   andq(dst, rtmp);
10098   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10099   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10100   subq(rtmp, dst);
10101   movq(rcx, rtmp);
10102   mov64(dst, MINUS_64);
10103   testq(rtmp, dst);
10104   jccb(Assembler::notEqual, L_special_case);
10105   movq(dst, src);
10106   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10107   andq(dst, rtmp);
10108   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10109   orq(dst, rtmp);
10110   movq(rtmp, src);
10111   testq(rtmp, rtmp);
10112   jccb(Assembler::greaterEqual, L_block1);
10113   negq(dst);
10114   bind(L_block1);
10115   sarq(dst);
10116   addq(dst, 0x1);
10117   sarq(dst, 0x1);
10118   jmp(L_exit);
10119   bind(L_special_case);
10120   convert_d2l(dst, src);
10121   bind(L_exit);
10122 }
10123 
10124 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10125   Label done;
10126   cvttsd2siq(dst, src);
10127   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10128   jccb(Assembler::notEqual, done);
10129   subptr(rsp, 8);
10130   movdbl(Address(rsp, 0), src);
10131   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10132   pop(dst);
10133   bind(done);
10134 }
10135 
10136 void MacroAssembler::cache_wb(Address line)
10137 {
10138   // 64 bit cpus always support clflush
10139   assert(VM_Version::supports_clflush(), "clflush should be available");
10140   bool optimized = VM_Version::supports_clflushopt();
10141   bool no_evict = VM_Version::supports_clwb();
10142 
10143   // prefer clwb (writeback without evict) otherwise
10144   // prefer clflushopt (potentially parallel writeback with evict)
10145   // otherwise fallback on clflush (serial writeback with evict)
10146 
10147   if (optimized) {
10148     if (no_evict) {
10149       clwb(line);
10150     } else {
10151       clflushopt(line);
10152     }
10153   } else {
10154     // no need for fence when using CLFLUSH
10155     clflush(line);
10156   }
10157 }
10158 
10159 void MacroAssembler::cache_wbsync(bool is_pre)
10160 {
10161   assert(VM_Version::supports_clflush(), "clflush should be available");
10162   bool optimized = VM_Version::supports_clflushopt();
10163   bool no_evict = VM_Version::supports_clwb();
10164 
10165   // pick the correct implementation
10166 
10167   if (!is_pre && (optimized || no_evict)) {
10168     // need an sfence for post flush when using clflushopt or clwb
10169     // otherwise no no need for any synchroniaztion
10170 
10171     sfence();
10172   }
10173 }
10174 
10175 #endif // _LP64
10176 
10177 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10178   switch (cond) {
10179     // Note some conditions are synonyms for others
10180     case Assembler::zero:         return Assembler::notZero;
10181     case Assembler::notZero:      return Assembler::zero;
10182     case Assembler::less:         return Assembler::greaterEqual;
10183     case Assembler::lessEqual:    return Assembler::greater;
10184     case Assembler::greater:      return Assembler::lessEqual;
10185     case Assembler::greaterEqual: return Assembler::less;
10186     case Assembler::below:        return Assembler::aboveEqual;
10187     case Assembler::belowEqual:   return Assembler::above;
10188     case Assembler::above:        return Assembler::belowEqual;
10189     case Assembler::aboveEqual:   return Assembler::below;
10190     case Assembler::overflow:     return Assembler::noOverflow;
10191     case Assembler::noOverflow:   return Assembler::overflow;
10192     case Assembler::negative:     return Assembler::positive;
10193     case Assembler::positive:     return Assembler::negative;
10194     case Assembler::parity:       return Assembler::noParity;
10195     case Assembler::noParity:     return Assembler::parity;
10196   }
10197   ShouldNotReachHere(); return Assembler::overflow;
10198 }
10199 
10200 SkipIfEqual::SkipIfEqual(
10201     MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
10202   _masm = masm;
10203   _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
10204   _masm->jcc(Assembler::equal, _label);
10205 }
10206 
10207 SkipIfEqual::~SkipIfEqual() {
10208   _masm->bind(_label);
10209 }
10210 
10211 // 32-bit Windows has its own fast-path implementation
10212 // of get_thread
10213 #if !defined(WIN32) || defined(_LP64)
10214 
10215 // This is simply a call to Thread::current()
10216 void MacroAssembler::get_thread(Register thread) {
10217   if (thread != rax) {
10218     push(rax);
10219   }
10220   LP64_ONLY(push(rdi);)
10221   LP64_ONLY(push(rsi);)
10222   push(rdx);
10223   push(rcx);
10224 #ifdef _LP64
10225   push(r8);
10226   push(r9);
10227   push(r10);
10228   push(r11);
10229 #endif
10230 
10231   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10232 
10233 #ifdef _LP64
10234   pop(r11);
10235   pop(r10);
10236   pop(r9);
10237   pop(r8);
10238 #endif
10239   pop(rcx);
10240   pop(rdx);
10241   LP64_ONLY(pop(rsi);)
10242   LP64_ONLY(pop(rdi);)
10243   if (thread != rax) {
10244     mov(thread, rax);
10245     pop(rax);
10246   }
10247 }
10248 
10249 
10250 #endif // !WIN32 || _LP64
10251 
10252 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10253   Label L_stack_ok;
10254   if (bias == 0) {
10255     testptr(sp, 2 * wordSize - 1);
10256   } else {
10257     // lea(tmp, Address(rsp, bias);
10258     mov(tmp, sp);
10259     addptr(tmp, bias);
10260     testptr(tmp, 2 * wordSize - 1);
10261   }
10262   jcc(Assembler::equal, L_stack_ok);
10263   block_comment(msg);
10264   stop(msg);
10265   bind(L_stack_ok);
10266 }
10267 
10268 // Implements lightweight-locking.
10269 //
10270 // obj: the object to be locked
10271 // reg_rax: rax
10272 // thread: the thread which attempts to lock obj
10273 // tmp: a temporary register
10274 void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10275   assert(reg_rax == rax, "");
10276   assert_different_registers(obj, reg_rax, thread, tmp);
10277 
10278   Label push;
10279   const Register top = tmp;
10280 
10281   // Preload the markWord. It is important that this is the first
10282   // instruction emitted as it is part of C1's null check semantics.
10283   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10284 
10285   // Load top.
10286   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10287 
10288   // Check if the lock-stack is full.
10289   cmpl(top, LockStack::end_offset());
10290   jcc(Assembler::greaterEqual, slow);
10291 
10292   // Check for recursion.
10293   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10294   jcc(Assembler::equal, push);
10295 
10296   // Check header for monitor (0b10).
10297   testptr(reg_rax, markWord::monitor_value);
10298   jcc(Assembler::notZero, slow);
10299 
10300   // Try to lock. Transition lock bits 0b01 => 0b00
10301   movptr(tmp, reg_rax);
10302   andptr(tmp, ~(int32_t)markWord::unlocked_value);
10303   orptr(reg_rax, markWord::unlocked_value);
10304   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10305   jcc(Assembler::notEqual, slow);
10306 
10307   // Restore top, CAS clobbers register.
10308   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10309 
10310   bind(push);
10311   // After successful lock, push object on lock-stack.
10312   movptr(Address(thread, top), obj);
10313   incrementl(top, oopSize);
10314   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
10315 }
10316 
10317 // Implements lightweight-unlocking.
10318 //
10319 // obj: the object to be unlocked
10320 // reg_rax: rax
10321 // thread: the thread
10322 // tmp: a temporary register
10323 //
10324 // x86_32 Note: reg_rax and thread may alias each other due to limited register
10325 //              availiability.
10326 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10327   assert(reg_rax == rax, "");
10328   assert_different_registers(obj, reg_rax, tmp);
10329   LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
10330 
10331   Label unlocked, push_and_slow;
10332   const Register top = tmp;
10333 
10334   // Check if obj is top of lock-stack.
10335   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10336   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10337   jcc(Assembler::notEqual, slow);
10338 
10339   // Pop lock-stack.
10340   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
10341   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10342 
10343   // Check if recursive.
10344   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
10345   jcc(Assembler::equal, unlocked);
10346 
10347   // Not recursive. Check header for monitor (0b10).
10348   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10349   testptr(reg_rax, markWord::monitor_value);
10350   jcc(Assembler::notZero, push_and_slow);
10351 
10352 #ifdef ASSERT
10353   // Check header not unlocked (0b01).
10354   Label not_unlocked;
10355   testptr(reg_rax, markWord::unlocked_value);
10356   jcc(Assembler::zero, not_unlocked);
10357   stop("lightweight_unlock already unlocked");
10358   bind(not_unlocked);
10359 #endif
10360 
10361   // Try to unlock. Transition lock bits 0b00 => 0b01
10362   movptr(tmp, reg_rax);
10363   orptr(tmp, markWord::unlocked_value);
10364   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10365   jcc(Assembler::equal, unlocked);
10366 
10367   bind(push_and_slow);
10368   // Restore lock-stack and handle the unlock in runtime.
10369   if (thread == reg_rax) {
10370     // On x86_32 we may lose the thread.
10371     get_thread(thread);
10372   }
10373 #ifdef ASSERT
10374   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10375   movptr(Address(thread, top), obj);
10376 #endif
10377   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10378   jmp(slow);
10379 
10380   bind(unlocked);
10381 }