1 /*
    2  * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "code/compiledIC.hpp"
   29 #include "compiler/compiler_globals.hpp"
   30 #include "compiler/disassembler.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "interpreter/interpreterRuntime.hpp"
   39 #include "jvm.h"
   40 #include "memory/resourceArea.hpp"
   41 #include "memory/universe.hpp"
   42 #include "oops/accessDecorators.hpp"
   43 #include "oops/compressedKlass.inline.hpp"
   44 #include "oops/compressedOops.inline.hpp"
   45 #include "oops/klass.inline.hpp"
   46 #include "prims/methodHandles.hpp"
   47 #include "runtime/continuation.hpp"
   48 #include "runtime/interfaceSupport.inline.hpp"
   49 #include "runtime/javaThread.hpp"
   50 #include "runtime/jniHandles.hpp"
   51 #include "runtime/objectMonitor.hpp"
   52 #include "runtime/os.hpp"
   53 #include "runtime/safepoint.hpp"
   54 #include "runtime/safepointMechanism.hpp"
   55 #include "runtime/sharedRuntime.hpp"
   56 #include "runtime/stubRoutines.hpp"
   57 #include "utilities/checkedCast.hpp"
   58 #include "utilities/macros.hpp"
   59 
   60 #ifdef PRODUCT
   61 #define BLOCK_COMMENT(str) /* nothing */
   62 #define STOP(error) stop(error)
   63 #else
   64 #define BLOCK_COMMENT(str) block_comment(str)
   65 #define STOP(error) block_comment(error); stop(error)
   66 #endif
   67 
   68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   69 
   70 #ifdef ASSERT
   71 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   72 #endif
   73 
   74 static const Assembler::Condition reverse[] = {
   75     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   76     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   77     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   78     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   79     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   80     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   81     Assembler::above          /* belowEqual    = 0x6 */ ,
   82     Assembler::belowEqual     /* above         = 0x7 */ ,
   83     Assembler::positive       /* negative      = 0x8 */ ,
   84     Assembler::negative       /* positive      = 0x9 */ ,
   85     Assembler::noParity       /* parity        = 0xa */ ,
   86     Assembler::parity         /* noParity      = 0xb */ ,
   87     Assembler::greaterEqual   /* less          = 0xc */ ,
   88     Assembler::less           /* greaterEqual  = 0xd */ ,
   89     Assembler::greater        /* lessEqual     = 0xe */ ,
   90     Assembler::lessEqual      /* greater       = 0xf, */
   91 
   92 };
   93 
   94 
   95 // Implementation of MacroAssembler
   96 
   97 // First all the versions that have distinct versions depending on 32/64 bit
   98 // Unless the difference is trivial (1 line or so).
   99 
  100 #ifndef _LP64
  101 
  102 // 32bit versions
  103 
  104 Address MacroAssembler::as_Address(AddressLiteral adr) {
  105   return Address(adr.target(), adr.rspec());
  106 }
  107 
  108 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  109   assert(rscratch == noreg, "");
  110   return Address::make_array(adr);
  111 }
  112 
  113 void MacroAssembler::call_VM_leaf_base(address entry_point,
  114                                        int number_of_arguments) {
  115   call(RuntimeAddress(entry_point));
  116   increment(rsp, number_of_arguments * wordSize);
  117 }
  118 
  119 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  120   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  121 }
  122 
  123 
  124 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  125   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  126 }
  127 
  128 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  129   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  130 }
  131 
  132 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  133   assert(rscratch == noreg, "redundant");
  134   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  135 }
  136 
  137 void MacroAssembler::extend_sign(Register hi, Register lo) {
  138   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  139   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  140     cdql();
  141   } else {
  142     movl(hi, lo);
  143     sarl(hi, 31);
  144   }
  145 }
  146 
  147 void MacroAssembler::jC2(Register tmp, Label& L) {
  148   // set parity bit if FPU flag C2 is set (via rax)
  149   save_rax(tmp);
  150   fwait(); fnstsw_ax();
  151   sahf();
  152   restore_rax(tmp);
  153   // branch
  154   jcc(Assembler::parity, L);
  155 }
  156 
  157 void MacroAssembler::jnC2(Register tmp, Label& L) {
  158   // set parity bit if FPU flag C2 is set (via rax)
  159   save_rax(tmp);
  160   fwait(); fnstsw_ax();
  161   sahf();
  162   restore_rax(tmp);
  163   // branch
  164   jcc(Assembler::noParity, L);
  165 }
  166 
  167 // 32bit can do a case table jump in one instruction but we no longer allow the base
  168 // to be installed in the Address class
  169 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  170   assert(rscratch == noreg, "not needed");
  171   jmp(as_Address(entry, noreg));
  172 }
  173 
  174 // Note: y_lo will be destroyed
  175 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  176   // Long compare for Java (semantics as described in JVM spec.)
  177   Label high, low, done;
  178 
  179   cmpl(x_hi, y_hi);
  180   jcc(Assembler::less, low);
  181   jcc(Assembler::greater, high);
  182   // x_hi is the return register
  183   xorl(x_hi, x_hi);
  184   cmpl(x_lo, y_lo);
  185   jcc(Assembler::below, low);
  186   jcc(Assembler::equal, done);
  187 
  188   bind(high);
  189   xorl(x_hi, x_hi);
  190   increment(x_hi);
  191   jmp(done);
  192 
  193   bind(low);
  194   xorl(x_hi, x_hi);
  195   decrementl(x_hi);
  196 
  197   bind(done);
  198 }
  199 
  200 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  201   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  202 }
  203 
  204 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  205   assert(rscratch == noreg, "not needed");
  206 
  207   // leal(dst, as_Address(adr));
  208   // see note in movl as to why we must use a move
  209   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  210 }
  211 
  212 void MacroAssembler::leave() {
  213   mov(rsp, rbp);
  214   pop(rbp);
  215 }
  216 
  217 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  218   // Multiplication of two Java long values stored on the stack
  219   // as illustrated below. Result is in rdx:rax.
  220   //
  221   // rsp ---> [  ??  ] \               \
  222   //            ....    | y_rsp_offset  |
  223   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  224   //          [ y_hi ]                  | (in bytes)
  225   //            ....                    |
  226   //          [ x_lo ]                 /
  227   //          [ x_hi ]
  228   //            ....
  229   //
  230   // Basic idea: lo(result) = lo(x_lo * y_lo)
  231   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  232   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  233   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  234   Label quick;
  235   // load x_hi, y_hi and check if quick
  236   // multiplication is possible
  237   movl(rbx, x_hi);
  238   movl(rcx, y_hi);
  239   movl(rax, rbx);
  240   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  241   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  242   // do full multiplication
  243   // 1st step
  244   mull(y_lo);                                    // x_hi * y_lo
  245   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  246   // 2nd step
  247   movl(rax, x_lo);
  248   mull(rcx);                                     // x_lo * y_hi
  249   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  250   // 3rd step
  251   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  252   movl(rax, x_lo);
  253   mull(y_lo);                                    // x_lo * y_lo
  254   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  255 }
  256 
  257 void MacroAssembler::lneg(Register hi, Register lo) {
  258   negl(lo);
  259   adcl(hi, 0);
  260   negl(hi);
  261 }
  262 
  263 void MacroAssembler::lshl(Register hi, Register lo) {
  264   // Java shift left long support (semantics as described in JVM spec., p.305)
  265   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  266   // shift value is in rcx !
  267   assert(hi != rcx, "must not use rcx");
  268   assert(lo != rcx, "must not use rcx");
  269   const Register s = rcx;                        // shift count
  270   const int      n = BitsPerWord;
  271   Label L;
  272   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  273   cmpl(s, n);                                    // if (s < n)
  274   jcc(Assembler::less, L);                       // else (s >= n)
  275   movl(hi, lo);                                  // x := x << n
  276   xorl(lo, lo);
  277   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  278   bind(L);                                       // s (mod n) < n
  279   shldl(hi, lo);                                 // x := x << s
  280   shll(lo);
  281 }
  282 
  283 
  284 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  285   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  286   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  287   assert(hi != rcx, "must not use rcx");
  288   assert(lo != rcx, "must not use rcx");
  289   const Register s = rcx;                        // shift count
  290   const int      n = BitsPerWord;
  291   Label L;
  292   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  293   cmpl(s, n);                                    // if (s < n)
  294   jcc(Assembler::less, L);                       // else (s >= n)
  295   movl(lo, hi);                                  // x := x >> n
  296   if (sign_extension) sarl(hi, 31);
  297   else                xorl(hi, hi);
  298   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  299   bind(L);                                       // s (mod n) < n
  300   shrdl(lo, hi);                                 // x := x >> s
  301   if (sign_extension) sarl(hi);
  302   else                shrl(hi);
  303 }
  304 
  305 void MacroAssembler::movoop(Register dst, jobject obj) {
  306   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  307 }
  308 
  309 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  310   assert(rscratch == noreg, "redundant");
  311   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  312 }
  313 
  314 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  315   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  316 }
  317 
  318 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  319   assert(rscratch == noreg, "redundant");
  320   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  321 }
  322 
  323 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  324   if (src.is_lval()) {
  325     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  326   } else {
  327     movl(dst, as_Address(src));
  328   }
  329 }
  330 
  331 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  332   assert(rscratch == noreg, "redundant");
  333   movl(as_Address(dst, noreg), src);
  334 }
  335 
  336 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  337   movl(dst, as_Address(src, noreg));
  338 }
  339 
  340 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  341   assert(rscratch == noreg, "redundant");
  342   movl(dst, src);
  343 }
  344 
  345 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  346   assert(rscratch == noreg, "redundant");
  347   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  348 }
  349 
  350 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  351   assert(rscratch == noreg, "redundant");
  352   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  353 }
  354 
  355 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  356   assert(rscratch == noreg, "redundant");
  357   if (src.is_lval()) {
  358     push_literal32((int32_t)src.target(), src.rspec());
  359   } else {
  360     pushl(as_Address(src));
  361   }
  362 }
  363 
  364 static void pass_arg0(MacroAssembler* masm, Register arg) {
  365   masm->push(arg);
  366 }
  367 
  368 static void pass_arg1(MacroAssembler* masm, Register arg) {
  369   masm->push(arg);
  370 }
  371 
  372 static void pass_arg2(MacroAssembler* masm, Register arg) {
  373   masm->push(arg);
  374 }
  375 
  376 static void pass_arg3(MacroAssembler* masm, Register arg) {
  377   masm->push(arg);
  378 }
  379 
  380 #ifndef PRODUCT
  381 extern "C" void findpc(intptr_t x);
  382 #endif
  383 
  384 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  385   // In order to get locks to work, we need to fake a in_VM state
  386   JavaThread* thread = JavaThread::current();
  387   JavaThreadState saved_state = thread->thread_state();
  388   thread->set_thread_state(_thread_in_vm);
  389   if (ShowMessageBoxOnError) {
  390     JavaThread* thread = JavaThread::current();
  391     JavaThreadState saved_state = thread->thread_state();
  392     thread->set_thread_state(_thread_in_vm);
  393     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  394       ttyLocker ttyl;
  395       BytecodeCounter::print();
  396     }
  397     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  398     // This is the value of eip which points to where verify_oop will return.
  399     if (os::message_box(msg, "Execution stopped, print registers?")) {
  400       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  401       BREAKPOINT;
  402     }
  403   }
  404   fatal("DEBUG MESSAGE: %s", msg);
  405 }
  406 
  407 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  408   ttyLocker ttyl;
  409   DebuggingContext debugging{};
  410   tty->print_cr("eip = 0x%08x", eip);
  411 #ifndef PRODUCT
  412   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  413     tty->cr();
  414     findpc(eip);
  415     tty->cr();
  416   }
  417 #endif
  418 #define PRINT_REG(rax) \
  419   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  420   PRINT_REG(rax);
  421   PRINT_REG(rbx);
  422   PRINT_REG(rcx);
  423   PRINT_REG(rdx);
  424   PRINT_REG(rdi);
  425   PRINT_REG(rsi);
  426   PRINT_REG(rbp);
  427   PRINT_REG(rsp);
  428 #undef PRINT_REG
  429   // Print some words near top of staack.
  430   int* dump_sp = (int*) rsp;
  431   for (int col1 = 0; col1 < 8; col1++) {
  432     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  433     os::print_location(tty, *dump_sp++);
  434   }
  435   for (int row = 0; row < 16; row++) {
  436     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  437     for (int col = 0; col < 8; col++) {
  438       tty->print(" 0x%08x", *dump_sp++);
  439     }
  440     tty->cr();
  441   }
  442   // Print some instructions around pc:
  443   Disassembler::decode((address)eip-64, (address)eip);
  444   tty->print_cr("--------");
  445   Disassembler::decode((address)eip, (address)eip+32);
  446 }
  447 
  448 void MacroAssembler::stop(const char* msg) {
  449   // push address of message
  450   ExternalAddress message((address)msg);
  451   pushptr(message.addr(), noreg);
  452   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  453   pusha();                                            // push registers
  454   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  455   hlt();
  456 }
  457 
  458 void MacroAssembler::warn(const char* msg) {
  459   push_CPU_state();
  460 
  461   // push address of message
  462   ExternalAddress message((address)msg);
  463   pushptr(message.addr(), noreg);
  464 
  465   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  466   addl(rsp, wordSize);       // discard argument
  467   pop_CPU_state();
  468 }
  469 
  470 void MacroAssembler::print_state() {
  471   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  472   pusha();                                            // push registers
  473 
  474   push_CPU_state();
  475   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  476   pop_CPU_state();
  477 
  478   popa();
  479   addl(rsp, wordSize);
  480 }
  481 
  482 #else // _LP64
  483 
  484 // 64 bit versions
  485 
  486 Address MacroAssembler::as_Address(AddressLiteral adr) {
  487   // amd64 always does this as a pc-rel
  488   // we can be absolute or disp based on the instruction type
  489   // jmp/call are displacements others are absolute
  490   assert(!adr.is_lval(), "must be rval");
  491   assert(reachable(adr), "must be");
  492   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  493 
  494 }
  495 
  496 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  497   AddressLiteral base = adr.base();
  498   lea(rscratch, base);
  499   Address index = adr.index();
  500   assert(index._disp == 0, "must not have disp"); // maybe it can?
  501   Address array(rscratch, index._index, index._scale, index._disp);
  502   return array;
  503 }
  504 
  505 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  506   Label L, E;
  507 
  508 #ifdef _WIN64
  509   // Windows always allocates space for it's register args
  510   assert(num_args <= 4, "only register arguments supported");
  511   subq(rsp,  frame::arg_reg_save_area_bytes);
  512 #endif
  513 
  514   // Align stack if necessary
  515   testl(rsp, 15);
  516   jcc(Assembler::zero, L);
  517 
  518   subq(rsp, 8);
  519   call(RuntimeAddress(entry_point));
  520   addq(rsp, 8);
  521   jmp(E);
  522 
  523   bind(L);
  524   call(RuntimeAddress(entry_point));
  525 
  526   bind(E);
  527 
  528 #ifdef _WIN64
  529   // restore stack pointer
  530   addq(rsp, frame::arg_reg_save_area_bytes);
  531 #endif
  532 
  533   if (entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter) ||
  534       entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj)) {
  535     Label not_preempted;
  536     movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset()));
  537     cmpptr(rscratch1, NULL_WORD);
  538     jccb(Assembler::zero, not_preempted);
  539     movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD);
  540     jmp(rscratch1);
  541     bind(not_preempted);
  542   }
  543 }
  544 
  545 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  546   assert(!src2.is_lval(), "should use cmpptr");
  547   assert(rscratch != noreg || always_reachable(src2), "missing");
  548 
  549   if (reachable(src2)) {
  550     cmpq(src1, as_Address(src2));
  551   } else {
  552     lea(rscratch, src2);
  553     Assembler::cmpq(src1, Address(rscratch, 0));
  554   }
  555 }
  556 
  557 int MacroAssembler::corrected_idivq(Register reg) {
  558   // Full implementation of Java ldiv and lrem; checks for special
  559   // case as described in JVM spec., p.243 & p.271.  The function
  560   // returns the (pc) offset of the idivl instruction - may be needed
  561   // for implicit exceptions.
  562   //
  563   //         normal case                           special case
  564   //
  565   // input : rax: dividend                         min_long
  566   //         reg: divisor   (may not be eax/edx)   -1
  567   //
  568   // output: rax: quotient  (= rax idiv reg)       min_long
  569   //         rdx: remainder (= rax irem reg)       0
  570   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  571   static const int64_t min_long = 0x8000000000000000;
  572   Label normal_case, special_case;
  573 
  574   // check for special case
  575   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  576   jcc(Assembler::notEqual, normal_case);
  577   xorl(rdx, rdx); // prepare rdx for possible special case (where
  578                   // remainder = 0)
  579   cmpq(reg, -1);
  580   jcc(Assembler::equal, special_case);
  581 
  582   // handle normal case
  583   bind(normal_case);
  584   cdqq();
  585   int idivq_offset = offset();
  586   idivq(reg);
  587 
  588   // normal and special case exit
  589   bind(special_case);
  590 
  591   return idivq_offset;
  592 }
  593 
  594 void MacroAssembler::decrementq(Register reg, int value) {
  595   if (value == min_jint) { subq(reg, value); return; }
  596   if (value <  0) { incrementq(reg, -value); return; }
  597   if (value == 0) {                        ; return; }
  598   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  599   /* else */      { subq(reg, value)       ; return; }
  600 }
  601 
  602 void MacroAssembler::decrementq(Address dst, int value) {
  603   if (value == min_jint) { subq(dst, value); return; }
  604   if (value <  0) { incrementq(dst, -value); return; }
  605   if (value == 0) {                        ; return; }
  606   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  607   /* else */      { subq(dst, value)       ; return; }
  608 }
  609 
  610 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  611   assert(rscratch != noreg || always_reachable(dst), "missing");
  612 
  613   if (reachable(dst)) {
  614     incrementq(as_Address(dst));
  615   } else {
  616     lea(rscratch, dst);
  617     incrementq(Address(rscratch, 0));
  618   }
  619 }
  620 
  621 void MacroAssembler::incrementq(Register reg, int value) {
  622   if (value == min_jint) { addq(reg, value); return; }
  623   if (value <  0) { decrementq(reg, -value); return; }
  624   if (value == 0) {                        ; return; }
  625   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  626   /* else */      { addq(reg, value)       ; return; }
  627 }
  628 
  629 void MacroAssembler::incrementq(Address dst, int value) {
  630   if (value == min_jint) { addq(dst, value); return; }
  631   if (value <  0) { decrementq(dst, -value); return; }
  632   if (value == 0) {                        ; return; }
  633   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  634   /* else */      { addq(dst, value)       ; return; }
  635 }
  636 
  637 // 32bit can do a case table jump in one instruction but we no longer allow the base
  638 // to be installed in the Address class
  639 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  640   lea(rscratch, entry.base());
  641   Address dispatch = entry.index();
  642   assert(dispatch._base == noreg, "must be");
  643   dispatch._base = rscratch;
  644   jmp(dispatch);
  645 }
  646 
  647 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  648   ShouldNotReachHere(); // 64bit doesn't use two regs
  649   cmpq(x_lo, y_lo);
  650 }
  651 
  652 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  653   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  654 }
  655 
  656 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  657   lea(rscratch, adr);
  658   movptr(dst, rscratch);
  659 }
  660 
  661 void MacroAssembler::leave() {
  662   // %%% is this really better? Why not on 32bit too?
  663   emit_int8((unsigned char)0xC9); // LEAVE
  664 }
  665 
  666 void MacroAssembler::lneg(Register hi, Register lo) {
  667   ShouldNotReachHere(); // 64bit doesn't use two regs
  668   negq(lo);
  669 }
  670 
  671 void MacroAssembler::movoop(Register dst, jobject obj) {
  672   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  673 }
  674 
  675 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  676   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  677   movq(dst, rscratch);
  678 }
  679 
  680 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  681   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  682 }
  683 
  684 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  685   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  686   movq(dst, rscratch);
  687 }
  688 
  689 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  690   if (src.is_lval()) {
  691     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  692   } else {
  693     if (reachable(src)) {
  694       movq(dst, as_Address(src));
  695     } else {
  696       lea(dst, src);
  697       movq(dst, Address(dst, 0));
  698     }
  699   }
  700 }
  701 
  702 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  703   movq(as_Address(dst, rscratch), src);
  704 }
  705 
  706 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  707   movq(dst, as_Address(src, dst /*rscratch*/));
  708 }
  709 
  710 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  711 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  712   if (is_simm32(src)) {
  713     movptr(dst, checked_cast<int32_t>(src));
  714   } else {
  715     mov64(rscratch, src);
  716     movq(dst, rscratch);
  717   }
  718 }
  719 
  720 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  721   movoop(rscratch, obj);
  722   push(rscratch);
  723 }
  724 
  725 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  726   mov_metadata(rscratch, obj);
  727   push(rscratch);
  728 }
  729 
  730 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  731   lea(rscratch, src);
  732   if (src.is_lval()) {
  733     push(rscratch);
  734   } else {
  735     pushq(Address(rscratch, 0));
  736   }
  737 }
  738 
  739 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  740   reset_last_Java_frame(r15_thread, clear_fp);
  741 }
  742 
  743 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  744                                          Register last_java_fp,
  745                                          address  last_java_pc,
  746                                          Register rscratch) {
  747   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  748 }
  749 
  750 static void pass_arg0(MacroAssembler* masm, Register arg) {
  751   if (c_rarg0 != arg ) {
  752     masm->mov(c_rarg0, arg);
  753   }
  754 }
  755 
  756 static void pass_arg1(MacroAssembler* masm, Register arg) {
  757   if (c_rarg1 != arg ) {
  758     masm->mov(c_rarg1, arg);
  759   }
  760 }
  761 
  762 static void pass_arg2(MacroAssembler* masm, Register arg) {
  763   if (c_rarg2 != arg ) {
  764     masm->mov(c_rarg2, arg);
  765   }
  766 }
  767 
  768 static void pass_arg3(MacroAssembler* masm, Register arg) {
  769   if (c_rarg3 != arg ) {
  770     masm->mov(c_rarg3, arg);
  771   }
  772 }
  773 
  774 void MacroAssembler::stop(const char* msg) {
  775   if (ShowMessageBoxOnError) {
  776     address rip = pc();
  777     pusha(); // get regs on stack
  778     lea(c_rarg1, InternalAddress(rip));
  779     movq(c_rarg2, rsp); // pass pointer to regs array
  780   }
  781   lea(c_rarg0, ExternalAddress((address) msg));
  782   andq(rsp, -16); // align stack as required by ABI
  783   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  784   hlt();
  785 }
  786 
  787 void MacroAssembler::warn(const char* msg) {
  788   push(rbp);
  789   movq(rbp, rsp);
  790   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  791   push_CPU_state();   // keeps alignment at 16 bytes
  792 
  793   lea(c_rarg0, ExternalAddress((address) msg));
  794   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  795 
  796   pop_CPU_state();
  797   mov(rsp, rbp);
  798   pop(rbp);
  799 }
  800 
  801 void MacroAssembler::print_state() {
  802   address rip = pc();
  803   pusha();            // get regs on stack
  804   push(rbp);
  805   movq(rbp, rsp);
  806   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  807   push_CPU_state();   // keeps alignment at 16 bytes
  808 
  809   lea(c_rarg0, InternalAddress(rip));
  810   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  811   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  812 
  813   pop_CPU_state();
  814   mov(rsp, rbp);
  815   pop(rbp);
  816   popa();
  817 }
  818 
  819 #ifndef PRODUCT
  820 extern "C" void findpc(intptr_t x);
  821 #endif
  822 
  823 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  824   // In order to get locks to work, we need to fake a in_VM state
  825   if (ShowMessageBoxOnError) {
  826     JavaThread* thread = JavaThread::current();
  827     JavaThreadState saved_state = thread->thread_state();
  828     thread->set_thread_state(_thread_in_vm);
  829 #ifndef PRODUCT
  830     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  831       ttyLocker ttyl;
  832       BytecodeCounter::print();
  833     }
  834 #endif
  835     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  836     // XXX correct this offset for amd64
  837     // This is the value of eip which points to where verify_oop will return.
  838     if (os::message_box(msg, "Execution stopped, print registers?")) {
  839       print_state64(pc, regs);
  840       BREAKPOINT;
  841     }
  842   }
  843   fatal("DEBUG MESSAGE: %s", msg);
  844 }
  845 
  846 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  847   ttyLocker ttyl;
  848   DebuggingContext debugging{};
  849   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  850 #ifndef PRODUCT
  851   tty->cr();
  852   findpc(pc);
  853   tty->cr();
  854 #endif
  855 #define PRINT_REG(rax, value) \
  856   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  857   PRINT_REG(rax, regs[15]);
  858   PRINT_REG(rbx, regs[12]);
  859   PRINT_REG(rcx, regs[14]);
  860   PRINT_REG(rdx, regs[13]);
  861   PRINT_REG(rdi, regs[8]);
  862   PRINT_REG(rsi, regs[9]);
  863   PRINT_REG(rbp, regs[10]);
  864   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  865   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  866   PRINT_REG(r8 , regs[7]);
  867   PRINT_REG(r9 , regs[6]);
  868   PRINT_REG(r10, regs[5]);
  869   PRINT_REG(r11, regs[4]);
  870   PRINT_REG(r12, regs[3]);
  871   PRINT_REG(r13, regs[2]);
  872   PRINT_REG(r14, regs[1]);
  873   PRINT_REG(r15, regs[0]);
  874 #undef PRINT_REG
  875   // Print some words near the top of the stack.
  876   int64_t* rsp = &regs[16];
  877   int64_t* dump_sp = rsp;
  878   for (int col1 = 0; col1 < 8; col1++) {
  879     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  880     os::print_location(tty, *dump_sp++);
  881   }
  882   for (int row = 0; row < 25; row++) {
  883     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  884     for (int col = 0; col < 4; col++) {
  885       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  886     }
  887     tty->cr();
  888   }
  889   // Print some instructions around pc:
  890   Disassembler::decode((address)pc-64, (address)pc);
  891   tty->print_cr("--------");
  892   Disassembler::decode((address)pc, (address)pc+32);
  893 }
  894 
  895 // The java_calling_convention describes stack locations as ideal slots on
  896 // a frame with no abi restrictions. Since we must observe abi restrictions
  897 // (like the placement of the register window) the slots must be biased by
  898 // the following value.
  899 static int reg2offset_in(VMReg r) {
  900   // Account for saved rbp and return address
  901   // This should really be in_preserve_stack_slots
  902   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  903 }
  904 
  905 static int reg2offset_out(VMReg r) {
  906   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  907 }
  908 
  909 // A long move
  910 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  911 
  912   // The calling conventions assures us that each VMregpair is either
  913   // all really one physical register or adjacent stack slots.
  914 
  915   if (src.is_single_phys_reg() ) {
  916     if (dst.is_single_phys_reg()) {
  917       if (dst.first() != src.first()) {
  918         mov(dst.first()->as_Register(), src.first()->as_Register());
  919       }
  920     } else {
  921       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  922              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  923       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  924     }
  925   } else if (dst.is_single_phys_reg()) {
  926     assert(src.is_single_reg(),  "not a stack pair");
  927     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  928   } else {
  929     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  930     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  931     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  932   }
  933 }
  934 
  935 // A double move
  936 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  937 
  938   // The calling conventions assures us that each VMregpair is either
  939   // all really one physical register or adjacent stack slots.
  940 
  941   if (src.is_single_phys_reg() ) {
  942     if (dst.is_single_phys_reg()) {
  943       // In theory these overlap but the ordering is such that this is likely a nop
  944       if ( src.first() != dst.first()) {
  945         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  946       }
  947     } else {
  948       assert(dst.is_single_reg(), "not a stack pair");
  949       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  950     }
  951   } else if (dst.is_single_phys_reg()) {
  952     assert(src.is_single_reg(),  "not a stack pair");
  953     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  954   } else {
  955     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  956     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  957     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  958   }
  959 }
  960 
  961 
  962 // A float arg may have to do float reg int reg conversion
  963 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  964   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  965 
  966   // The calling conventions assures us that each VMregpair is either
  967   // all really one physical register or adjacent stack slots.
  968 
  969   if (src.first()->is_stack()) {
  970     if (dst.first()->is_stack()) {
  971       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  972       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  973     } else {
  974       // stack to reg
  975       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  976       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  977     }
  978   } else if (dst.first()->is_stack()) {
  979     // reg to stack
  980     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  981     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  982   } else {
  983     // reg to reg
  984     // In theory these overlap but the ordering is such that this is likely a nop
  985     if ( src.first() != dst.first()) {
  986       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  987     }
  988   }
  989 }
  990 
  991 // On 64 bit we will store integer like items to the stack as
  992 // 64 bits items (x86_32/64 abi) even though java would only store
  993 // 32bits for a parameter. On 32bit it will simply be 32 bits
  994 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  995 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  996   if (src.first()->is_stack()) {
  997     if (dst.first()->is_stack()) {
  998       // stack to stack
  999       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 1000       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 1001     } else {
 1002       // stack to reg
 1003       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 1004     }
 1005   } else if (dst.first()->is_stack()) {
 1006     // reg to stack
 1007     // Do we really have to sign extend???
 1008     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 1009     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 1010   } else {
 1011     // Do we really have to sign extend???
 1012     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1013     if (dst.first() != src.first()) {
 1014       movq(dst.first()->as_Register(), src.first()->as_Register());
 1015     }
 1016   }
 1017 }
 1018 
 1019 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1020   if (src.first()->is_stack()) {
 1021     if (dst.first()->is_stack()) {
 1022       // stack to stack
 1023       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1024       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1025     } else {
 1026       // stack to reg
 1027       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1028     }
 1029   } else if (dst.first()->is_stack()) {
 1030     // reg to stack
 1031     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1032   } else {
 1033     if (dst.first() != src.first()) {
 1034       movq(dst.first()->as_Register(), src.first()->as_Register());
 1035     }
 1036   }
 1037 }
 1038 
 1039 // An oop arg. Must pass a handle not the oop itself
 1040 void MacroAssembler::object_move(OopMap* map,
 1041                         int oop_handle_offset,
 1042                         int framesize_in_slots,
 1043                         VMRegPair src,
 1044                         VMRegPair dst,
 1045                         bool is_receiver,
 1046                         int* receiver_offset) {
 1047 
 1048   // must pass a handle. First figure out the location we use as a handle
 1049 
 1050   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1051 
 1052   // See if oop is null if it is we need no handle
 1053 
 1054   if (src.first()->is_stack()) {
 1055 
 1056     // Oop is already on the stack as an argument
 1057     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1058     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1059     if (is_receiver) {
 1060       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1061     }
 1062 
 1063     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1064     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1065     // conditionally move a null
 1066     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1067   } else {
 1068 
 1069     // Oop is in a register we must store it to the space we reserve
 1070     // on the stack for oop_handles and pass a handle if oop is non-null
 1071 
 1072     const Register rOop = src.first()->as_Register();
 1073     int oop_slot;
 1074     if (rOop == j_rarg0)
 1075       oop_slot = 0;
 1076     else if (rOop == j_rarg1)
 1077       oop_slot = 1;
 1078     else if (rOop == j_rarg2)
 1079       oop_slot = 2;
 1080     else if (rOop == j_rarg3)
 1081       oop_slot = 3;
 1082     else if (rOop == j_rarg4)
 1083       oop_slot = 4;
 1084     else {
 1085       assert(rOop == j_rarg5, "wrong register");
 1086       oop_slot = 5;
 1087     }
 1088 
 1089     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1090     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1091 
 1092     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1093     // Store oop in handle area, may be null
 1094     movptr(Address(rsp, offset), rOop);
 1095     if (is_receiver) {
 1096       *receiver_offset = offset;
 1097     }
 1098 
 1099     cmpptr(rOop, NULL_WORD);
 1100     lea(rHandle, Address(rsp, offset));
 1101     // conditionally move a null from the handle area where it was just stored
 1102     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1103   }
 1104 
 1105   // If arg is on the stack then place it otherwise it is already in correct reg.
 1106   if (dst.first()->is_stack()) {
 1107     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1108   }
 1109 }
 1110 
 1111 #endif // _LP64
 1112 
 1113 // Now versions that are common to 32/64 bit
 1114 
 1115 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1116   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1117 }
 1118 
 1119 void MacroAssembler::addptr(Register dst, Register src) {
 1120   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1121 }
 1122 
 1123 void MacroAssembler::addptr(Address dst, Register src) {
 1124   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1125 }
 1126 
 1127 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1128   assert(rscratch != noreg || always_reachable(src), "missing");
 1129 
 1130   if (reachable(src)) {
 1131     Assembler::addsd(dst, as_Address(src));
 1132   } else {
 1133     lea(rscratch, src);
 1134     Assembler::addsd(dst, Address(rscratch, 0));
 1135   }
 1136 }
 1137 
 1138 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1139   assert(rscratch != noreg || always_reachable(src), "missing");
 1140 
 1141   if (reachable(src)) {
 1142     addss(dst, as_Address(src));
 1143   } else {
 1144     lea(rscratch, src);
 1145     addss(dst, Address(rscratch, 0));
 1146   }
 1147 }
 1148 
 1149 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1150   assert(rscratch != noreg || always_reachable(src), "missing");
 1151 
 1152   if (reachable(src)) {
 1153     Assembler::addpd(dst, as_Address(src));
 1154   } else {
 1155     lea(rscratch, src);
 1156     Assembler::addpd(dst, Address(rscratch, 0));
 1157   }
 1158 }
 1159 
 1160 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1161 // Stub code is generated once and never copied.
 1162 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1163 void MacroAssembler::align64() {
 1164   align(64, (uint)(uintptr_t)pc());
 1165 }
 1166 
 1167 void MacroAssembler::align32() {
 1168   align(32, (uint)(uintptr_t)pc());
 1169 }
 1170 
 1171 void MacroAssembler::align(uint modulus) {
 1172   // 8273459: Ensure alignment is possible with current segment alignment
 1173   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1174   align(modulus, offset());
 1175 }
 1176 
 1177 void MacroAssembler::align(uint modulus, uint target) {
 1178   if (target % modulus != 0) {
 1179     nop(modulus - (target % modulus));
 1180   }
 1181 }
 1182 
 1183 void MacroAssembler::push_f(XMMRegister r) {
 1184   subptr(rsp, wordSize);
 1185   movflt(Address(rsp, 0), r);
 1186 }
 1187 
 1188 void MacroAssembler::pop_f(XMMRegister r) {
 1189   movflt(r, Address(rsp, 0));
 1190   addptr(rsp, wordSize);
 1191 }
 1192 
 1193 void MacroAssembler::push_d(XMMRegister r) {
 1194   subptr(rsp, 2 * wordSize);
 1195   movdbl(Address(rsp, 0), r);
 1196 }
 1197 
 1198 void MacroAssembler::pop_d(XMMRegister r) {
 1199   movdbl(r, Address(rsp, 0));
 1200   addptr(rsp, 2 * Interpreter::stackElementSize);
 1201 }
 1202 
 1203 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1204   // Used in sign-masking with aligned address.
 1205   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1206   assert(rscratch != noreg || always_reachable(src), "missing");
 1207 
 1208   if (reachable(src)) {
 1209     Assembler::andpd(dst, as_Address(src));
 1210   } else {
 1211     lea(rscratch, src);
 1212     Assembler::andpd(dst, Address(rscratch, 0));
 1213   }
 1214 }
 1215 
 1216 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1217   // Used in sign-masking with aligned address.
 1218   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1219   assert(rscratch != noreg || always_reachable(src), "missing");
 1220 
 1221   if (reachable(src)) {
 1222     Assembler::andps(dst, as_Address(src));
 1223   } else {
 1224     lea(rscratch, src);
 1225     Assembler::andps(dst, Address(rscratch, 0));
 1226   }
 1227 }
 1228 
 1229 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1230   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1231 }
 1232 
 1233 #ifdef _LP64
 1234 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1235   assert(rscratch != noreg || always_reachable(src), "missing");
 1236 
 1237   if (reachable(src)) {
 1238     andq(dst, as_Address(src));
 1239   } else {
 1240     lea(rscratch, src);
 1241     andq(dst, Address(rscratch, 0));
 1242   }
 1243 }
 1244 #endif
 1245 
 1246 void MacroAssembler::atomic_incl(Address counter_addr) {
 1247   lock();
 1248   incrementl(counter_addr);
 1249 }
 1250 
 1251 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1252   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1253 
 1254   if (reachable(counter_addr)) {
 1255     atomic_incl(as_Address(counter_addr));
 1256   } else {
 1257     lea(rscratch, counter_addr);
 1258     atomic_incl(Address(rscratch, 0));
 1259   }
 1260 }
 1261 
 1262 #ifdef _LP64
 1263 void MacroAssembler::atomic_incq(Address counter_addr) {
 1264   lock();
 1265   incrementq(counter_addr);
 1266 }
 1267 
 1268 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1269   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1270 
 1271   if (reachable(counter_addr)) {
 1272     atomic_incq(as_Address(counter_addr));
 1273   } else {
 1274     lea(rscratch, counter_addr);
 1275     atomic_incq(Address(rscratch, 0));
 1276   }
 1277 }
 1278 #endif
 1279 
 1280 // Writes to stack successive pages until offset reached to check for
 1281 // stack overflow + shadow pages.  This clobbers tmp.
 1282 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1283   movptr(tmp, rsp);
 1284   // Bang stack for total size given plus shadow page size.
 1285   // Bang one page at a time because large size can bang beyond yellow and
 1286   // red zones.
 1287   Label loop;
 1288   bind(loop);
 1289   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1290   subptr(tmp, (int)os::vm_page_size());
 1291   subl(size, (int)os::vm_page_size());
 1292   jcc(Assembler::greater, loop);
 1293 
 1294   // Bang down shadow pages too.
 1295   // At this point, (tmp-0) is the last address touched, so don't
 1296   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1297   // was post-decremented.)  Skip this address by starting at i=1, and
 1298   // touch a few more pages below.  N.B.  It is important to touch all
 1299   // the way down including all pages in the shadow zone.
 1300   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1301     // this could be any sized move but this is can be a debugging crumb
 1302     // so the bigger the better.
 1303     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1304   }
 1305 }
 1306 
 1307 void MacroAssembler::reserved_stack_check() {
 1308   // testing if reserved zone needs to be enabled
 1309   Label no_reserved_zone_enabling;
 1310   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1311   NOT_LP64(get_thread(rsi);)
 1312 
 1313   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1314   jcc(Assembler::below, no_reserved_zone_enabling);
 1315 
 1316   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1317   jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
 1318   should_not_reach_here();
 1319 
 1320   bind(no_reserved_zone_enabling);
 1321 }
 1322 
 1323 void MacroAssembler::c2bool(Register x) {
 1324   // implements x == 0 ? 0 : 1
 1325   // note: must only look at least-significant byte of x
 1326   //       since C-style booleans are stored in one byte
 1327   //       only! (was bug)
 1328   andl(x, 0xFF);
 1329   setb(Assembler::notZero, x);
 1330 }
 1331 
 1332 // Wouldn't need if AddressLiteral version had new name
 1333 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1334   Assembler::call(L, rtype);
 1335 }
 1336 
 1337 void MacroAssembler::call(Register entry) {
 1338   Assembler::call(entry);
 1339 }
 1340 
 1341 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1342   assert(rscratch != noreg || always_reachable(entry), "missing");
 1343 
 1344   if (reachable(entry)) {
 1345     Assembler::call_literal(entry.target(), entry.rspec());
 1346   } else {
 1347     lea(rscratch, entry);
 1348     Assembler::call(rscratch);
 1349   }
 1350 }
 1351 
 1352 void MacroAssembler::ic_call(address entry, jint method_index) {
 1353   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1354 #ifdef _LP64
 1355   // Needs full 64-bit immediate for later patching.
 1356   mov64(rax, (int64_t)Universe::non_oop_word());
 1357 #else
 1358   movptr(rax, (intptr_t)Universe::non_oop_word());
 1359 #endif
 1360   call(AddressLiteral(entry, rh));
 1361 }
 1362 
 1363 int MacroAssembler::ic_check_size() {
 1364   return LP64_ONLY(14) NOT_LP64(12);
 1365 }
 1366 
 1367 int MacroAssembler::ic_check(int end_alignment) {
 1368   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1369   Register data = rax;
 1370   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1371 
 1372   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1373   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1374   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1375   // before the inline cache check here, and not after
 1376   align(end_alignment, offset() + ic_check_size());
 1377 
 1378   int uep_offset = offset();
 1379 
 1380   if (UseCompressedClassPointers) {
 1381     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1382     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1383   } else {
 1384     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1385     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1386   }
 1387 
 1388   // if inline cache check fails, then jump to runtime routine
 1389   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1390   assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
 1391 
 1392   return uep_offset;
 1393 }
 1394 
 1395 void MacroAssembler::emit_static_call_stub() {
 1396   // Static stub relocation also tags the Method* in the code-stream.
 1397   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1398   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1399   jump(RuntimeAddress(pc()));
 1400 }
 1401 
 1402 // Implementation of call_VM versions
 1403 
 1404 void MacroAssembler::call_VM(Register oop_result,
 1405                              address entry_point,
 1406                              bool check_exceptions) {
 1407   Label C, E;
 1408   call(C, relocInfo::none);
 1409   jmp(E);
 1410 
 1411   bind(C);
 1412   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1413   ret(0);
 1414 
 1415   bind(E);
 1416 }
 1417 
 1418 void MacroAssembler::call_VM(Register oop_result,
 1419                              address entry_point,
 1420                              Register arg_1,
 1421                              bool check_exceptions) {
 1422   Label C, E;
 1423   call(C, relocInfo::none);
 1424   jmp(E);
 1425 
 1426   bind(C);
 1427   pass_arg1(this, arg_1);
 1428   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1429   ret(0);
 1430 
 1431   bind(E);
 1432 }
 1433 
 1434 void MacroAssembler::call_VM(Register oop_result,
 1435                              address entry_point,
 1436                              Register arg_1,
 1437                              Register arg_2,
 1438                              bool check_exceptions) {
 1439   Label C, E;
 1440   call(C, relocInfo::none);
 1441   jmp(E);
 1442 
 1443   bind(C);
 1444 
 1445   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1446 
 1447   pass_arg2(this, arg_2);
 1448   pass_arg1(this, arg_1);
 1449   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1450   ret(0);
 1451 
 1452   bind(E);
 1453 }
 1454 
 1455 void MacroAssembler::call_VM(Register oop_result,
 1456                              address entry_point,
 1457                              Register arg_1,
 1458                              Register arg_2,
 1459                              Register arg_3,
 1460                              bool check_exceptions) {
 1461   Label C, E;
 1462   call(C, relocInfo::none);
 1463   jmp(E);
 1464 
 1465   bind(C);
 1466 
 1467   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1468   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1469   pass_arg3(this, arg_3);
 1470   pass_arg2(this, arg_2);
 1471   pass_arg1(this, arg_1);
 1472   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1473   ret(0);
 1474 
 1475   bind(E);
 1476 }
 1477 
 1478 void MacroAssembler::call_VM(Register oop_result,
 1479                              Register last_java_sp,
 1480                              address entry_point,
 1481                              int number_of_arguments,
 1482                              bool check_exceptions) {
 1483   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1484   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1485 }
 1486 
 1487 void MacroAssembler::call_VM(Register oop_result,
 1488                              Register last_java_sp,
 1489                              address entry_point,
 1490                              Register arg_1,
 1491                              bool check_exceptions) {
 1492   pass_arg1(this, arg_1);
 1493   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1494 }
 1495 
 1496 void MacroAssembler::call_VM(Register oop_result,
 1497                              Register last_java_sp,
 1498                              address entry_point,
 1499                              Register arg_1,
 1500                              Register arg_2,
 1501                              bool check_exceptions) {
 1502 
 1503   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1504   pass_arg2(this, arg_2);
 1505   pass_arg1(this, arg_1);
 1506   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1507 }
 1508 
 1509 void MacroAssembler::call_VM(Register oop_result,
 1510                              Register last_java_sp,
 1511                              address entry_point,
 1512                              Register arg_1,
 1513                              Register arg_2,
 1514                              Register arg_3,
 1515                              bool check_exceptions) {
 1516   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1517   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1518   pass_arg3(this, arg_3);
 1519   pass_arg2(this, arg_2);
 1520   pass_arg1(this, arg_1);
 1521   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1522 }
 1523 
 1524 void MacroAssembler::super_call_VM(Register oop_result,
 1525                                    Register last_java_sp,
 1526                                    address entry_point,
 1527                                    int number_of_arguments,
 1528                                    bool check_exceptions) {
 1529   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1530   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1531 }
 1532 
 1533 void MacroAssembler::super_call_VM(Register oop_result,
 1534                                    Register last_java_sp,
 1535                                    address entry_point,
 1536                                    Register arg_1,
 1537                                    bool check_exceptions) {
 1538   pass_arg1(this, arg_1);
 1539   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1540 }
 1541 
 1542 void MacroAssembler::super_call_VM(Register oop_result,
 1543                                    Register last_java_sp,
 1544                                    address entry_point,
 1545                                    Register arg_1,
 1546                                    Register arg_2,
 1547                                    bool check_exceptions) {
 1548 
 1549   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1550   pass_arg2(this, arg_2);
 1551   pass_arg1(this, arg_1);
 1552   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1553 }
 1554 
 1555 void MacroAssembler::super_call_VM(Register oop_result,
 1556                                    Register last_java_sp,
 1557                                    address entry_point,
 1558                                    Register arg_1,
 1559                                    Register arg_2,
 1560                                    Register arg_3,
 1561                                    bool check_exceptions) {
 1562   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1563   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1564   pass_arg3(this, arg_3);
 1565   pass_arg2(this, arg_2);
 1566   pass_arg1(this, arg_1);
 1567   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1568 }
 1569 
 1570 void MacroAssembler::call_VM_base(Register oop_result,
 1571                                   Register java_thread,
 1572                                   Register last_java_sp,
 1573                                   address  entry_point,
 1574                                   int      number_of_arguments,
 1575                                   bool     check_exceptions) {
 1576   // determine java_thread register
 1577   if (!java_thread->is_valid()) {
 1578 #ifdef _LP64
 1579     java_thread = r15_thread;
 1580 #else
 1581     java_thread = rdi;
 1582     get_thread(java_thread);
 1583 #endif // LP64
 1584   }
 1585   // determine last_java_sp register
 1586   if (!last_java_sp->is_valid()) {
 1587     last_java_sp = rsp;
 1588   }
 1589   // debugging support
 1590   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1591   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1592 #ifdef ASSERT
 1593   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1594   // r12 is the heapbase.
 1595   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1596 #endif // ASSERT
 1597 
 1598   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1599   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1600 
 1601   // push java thread (becomes first argument of C function)
 1602 
 1603   NOT_LP64(push(java_thread); number_of_arguments++);
 1604   LP64_ONLY(mov(c_rarg0, r15_thread));
 1605 
 1606   // set last Java frame before call
 1607   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1608 
 1609   // Only interpreter should have to set fp
 1610   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1611 
 1612   // do the call, remove parameters
 1613   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1614 
 1615   // restore the thread (cannot use the pushed argument since arguments
 1616   // may be overwritten by C code generated by an optimizing compiler);
 1617   // however can use the register value directly if it is callee saved.
 1618   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1619     // rdi & rsi (also r15) are callee saved -> nothing to do
 1620 #ifdef ASSERT
 1621     guarantee(java_thread != rax, "change this code");
 1622     push(rax);
 1623     { Label L;
 1624       get_thread(rax);
 1625       cmpptr(java_thread, rax);
 1626       jcc(Assembler::equal, L);
 1627       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1628       bind(L);
 1629     }
 1630     pop(rax);
 1631 #endif
 1632   } else {
 1633     get_thread(java_thread);
 1634   }
 1635   // reset last Java frame
 1636   // Only interpreter should have to clear fp
 1637   reset_last_Java_frame(java_thread, true);
 1638 
 1639    // C++ interp handles this in the interpreter
 1640   check_and_handle_popframe(java_thread);
 1641   check_and_handle_earlyret(java_thread);
 1642 
 1643   if (check_exceptions) {
 1644     // check for pending exceptions (java_thread is set upon return)
 1645     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1646 #ifndef _LP64
 1647     jump_cc(Assembler::notEqual,
 1648             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1649 #else
 1650     // This used to conditionally jump to forward_exception however it is
 1651     // possible if we relocate that the branch will not reach. So we must jump
 1652     // around so we can always reach
 1653 
 1654     Label ok;
 1655     jcc(Assembler::equal, ok);
 1656     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1657     bind(ok);
 1658 #endif // LP64
 1659   }
 1660 
 1661   // get oop result if there is one and reset the value in the thread
 1662   if (oop_result->is_valid()) {
 1663     get_vm_result(oop_result, java_thread);
 1664   }
 1665 }
 1666 
 1667 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1668 
 1669   // Calculate the value for last_Java_sp
 1670   // somewhat subtle. call_VM does an intermediate call
 1671   // which places a return address on the stack just under the
 1672   // stack pointer as the user finished with it. This allows
 1673   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1674   // On 32bit we then have to push additional args on the stack to accomplish
 1675   // the actual requested call. On 64bit call_VM only can use register args
 1676   // so the only extra space is the return address that call_VM created.
 1677   // This hopefully explains the calculations here.
 1678 
 1679 #ifdef _LP64
 1680   // We've pushed one address, correct last_Java_sp
 1681   lea(rax, Address(rsp, wordSize));
 1682 #else
 1683   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1684 #endif // LP64
 1685 
 1686   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1687 
 1688 }
 1689 
 1690 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1691 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1692   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1693 }
 1694 
 1695 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1696   call_VM_leaf_base(entry_point, number_of_arguments);
 1697 }
 1698 
 1699 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1700   pass_arg0(this, arg_0);
 1701   call_VM_leaf(entry_point, 1);
 1702 }
 1703 
 1704 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1705 
 1706   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1707   pass_arg1(this, arg_1);
 1708   pass_arg0(this, arg_0);
 1709   call_VM_leaf(entry_point, 2);
 1710 }
 1711 
 1712 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1713   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1714   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1715   pass_arg2(this, arg_2);
 1716   pass_arg1(this, arg_1);
 1717   pass_arg0(this, arg_0);
 1718   call_VM_leaf(entry_point, 3);
 1719 }
 1720 
 1721 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1722   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1723   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1724   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1725   pass_arg3(this, arg_3);
 1726   pass_arg2(this, arg_2);
 1727   pass_arg1(this, arg_1);
 1728   pass_arg0(this, arg_0);
 1729   call_VM_leaf(entry_point, 3);
 1730 }
 1731 
 1732 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1733   pass_arg0(this, arg_0);
 1734   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1735 }
 1736 
 1737 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1738   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1739   pass_arg1(this, arg_1);
 1740   pass_arg0(this, arg_0);
 1741   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1742 }
 1743 
 1744 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1745   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1746   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1747   pass_arg2(this, arg_2);
 1748   pass_arg1(this, arg_1);
 1749   pass_arg0(this, arg_0);
 1750   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1751 }
 1752 
 1753 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1754   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1755   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1756   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1757   pass_arg3(this, arg_3);
 1758   pass_arg2(this, arg_2);
 1759   pass_arg1(this, arg_1);
 1760   pass_arg0(this, arg_0);
 1761   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1762 }
 1763 
 1764 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1765   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1766   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1767   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1768 }
 1769 
 1770 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1771   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1772   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1773 }
 1774 
 1775 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1776 }
 1777 
 1778 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1779 }
 1780 
 1781 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1782   assert(rscratch != noreg || always_reachable(src1), "missing");
 1783 
 1784   if (reachable(src1)) {
 1785     cmpl(as_Address(src1), imm);
 1786   } else {
 1787     lea(rscratch, src1);
 1788     cmpl(Address(rscratch, 0), imm);
 1789   }
 1790 }
 1791 
 1792 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1793   assert(!src2.is_lval(), "use cmpptr");
 1794   assert(rscratch != noreg || always_reachable(src2), "missing");
 1795 
 1796   if (reachable(src2)) {
 1797     cmpl(src1, as_Address(src2));
 1798   } else {
 1799     lea(rscratch, src2);
 1800     cmpl(src1, Address(rscratch, 0));
 1801   }
 1802 }
 1803 
 1804 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1805   Assembler::cmpl(src1, imm);
 1806 }
 1807 
 1808 void MacroAssembler::cmp32(Register src1, Address src2) {
 1809   Assembler::cmpl(src1, src2);
 1810 }
 1811 
 1812 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1813   ucomisd(opr1, opr2);
 1814 
 1815   Label L;
 1816   if (unordered_is_less) {
 1817     movl(dst, -1);
 1818     jcc(Assembler::parity, L);
 1819     jcc(Assembler::below , L);
 1820     movl(dst, 0);
 1821     jcc(Assembler::equal , L);
 1822     increment(dst);
 1823   } else { // unordered is greater
 1824     movl(dst, 1);
 1825     jcc(Assembler::parity, L);
 1826     jcc(Assembler::above , L);
 1827     movl(dst, 0);
 1828     jcc(Assembler::equal , L);
 1829     decrementl(dst);
 1830   }
 1831   bind(L);
 1832 }
 1833 
 1834 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1835   ucomiss(opr1, opr2);
 1836 
 1837   Label L;
 1838   if (unordered_is_less) {
 1839     movl(dst, -1);
 1840     jcc(Assembler::parity, L);
 1841     jcc(Assembler::below , L);
 1842     movl(dst, 0);
 1843     jcc(Assembler::equal , L);
 1844     increment(dst);
 1845   } else { // unordered is greater
 1846     movl(dst, 1);
 1847     jcc(Assembler::parity, L);
 1848     jcc(Assembler::above , L);
 1849     movl(dst, 0);
 1850     jcc(Assembler::equal , L);
 1851     decrementl(dst);
 1852   }
 1853   bind(L);
 1854 }
 1855 
 1856 
 1857 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1858   assert(rscratch != noreg || always_reachable(src1), "missing");
 1859 
 1860   if (reachable(src1)) {
 1861     cmpb(as_Address(src1), imm);
 1862   } else {
 1863     lea(rscratch, src1);
 1864     cmpb(Address(rscratch, 0), imm);
 1865   }
 1866 }
 1867 
 1868 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1869 #ifdef _LP64
 1870   assert(rscratch != noreg || always_reachable(src2), "missing");
 1871 
 1872   if (src2.is_lval()) {
 1873     movptr(rscratch, src2);
 1874     Assembler::cmpq(src1, rscratch);
 1875   } else if (reachable(src2)) {
 1876     cmpq(src1, as_Address(src2));
 1877   } else {
 1878     lea(rscratch, src2);
 1879     Assembler::cmpq(src1, Address(rscratch, 0));
 1880   }
 1881 #else
 1882   assert(rscratch == noreg, "not needed");
 1883   if (src2.is_lval()) {
 1884     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1885   } else {
 1886     cmpl(src1, as_Address(src2));
 1887   }
 1888 #endif // _LP64
 1889 }
 1890 
 1891 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1892   assert(src2.is_lval(), "not a mem-mem compare");
 1893 #ifdef _LP64
 1894   // moves src2's literal address
 1895   movptr(rscratch, src2);
 1896   Assembler::cmpq(src1, rscratch);
 1897 #else
 1898   assert(rscratch == noreg, "not needed");
 1899   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1900 #endif // _LP64
 1901 }
 1902 
 1903 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1904   cmpptr(src1, src2);
 1905 }
 1906 
 1907 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1908   cmpptr(src1, src2);
 1909 }
 1910 
 1911 #ifdef _LP64
 1912 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1913   movoop(rscratch, src2);
 1914   cmpptr(src1, rscratch);
 1915 }
 1916 #endif
 1917 
 1918 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1919   assert(rscratch != noreg || always_reachable(adr), "missing");
 1920 
 1921   if (reachable(adr)) {
 1922     lock();
 1923     cmpxchgptr(reg, as_Address(adr));
 1924   } else {
 1925     lea(rscratch, adr);
 1926     lock();
 1927     cmpxchgptr(reg, Address(rscratch, 0));
 1928   }
 1929 }
 1930 
 1931 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1932   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1933 }
 1934 
 1935 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1936   assert(rscratch != noreg || always_reachable(src), "missing");
 1937 
 1938   if (reachable(src)) {
 1939     Assembler::comisd(dst, as_Address(src));
 1940   } else {
 1941     lea(rscratch, src);
 1942     Assembler::comisd(dst, Address(rscratch, 0));
 1943   }
 1944 }
 1945 
 1946 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1947   assert(rscratch != noreg || always_reachable(src), "missing");
 1948 
 1949   if (reachable(src)) {
 1950     Assembler::comiss(dst, as_Address(src));
 1951   } else {
 1952     lea(rscratch, src);
 1953     Assembler::comiss(dst, Address(rscratch, 0));
 1954   }
 1955 }
 1956 
 1957 
 1958 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1959   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1960 
 1961   Condition negated_cond = negate_condition(cond);
 1962   Label L;
 1963   jcc(negated_cond, L);
 1964   pushf(); // Preserve flags
 1965   atomic_incl(counter_addr, rscratch);
 1966   popf();
 1967   bind(L);
 1968 }
 1969 
 1970 int MacroAssembler::corrected_idivl(Register reg) {
 1971   // Full implementation of Java idiv and irem; checks for
 1972   // special case as described in JVM spec., p.243 & p.271.
 1973   // The function returns the (pc) offset of the idivl
 1974   // instruction - may be needed for implicit exceptions.
 1975   //
 1976   //         normal case                           special case
 1977   //
 1978   // input : rax,: dividend                         min_int
 1979   //         reg: divisor   (may not be rax,/rdx)   -1
 1980   //
 1981   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1982   //         rdx: remainder (= rax, irem reg)       0
 1983   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1984   const int min_int = 0x80000000;
 1985   Label normal_case, special_case;
 1986 
 1987   // check for special case
 1988   cmpl(rax, min_int);
 1989   jcc(Assembler::notEqual, normal_case);
 1990   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1991   cmpl(reg, -1);
 1992   jcc(Assembler::equal, special_case);
 1993 
 1994   // handle normal case
 1995   bind(normal_case);
 1996   cdql();
 1997   int idivl_offset = offset();
 1998   idivl(reg);
 1999 
 2000   // normal and special case exit
 2001   bind(special_case);
 2002 
 2003   return idivl_offset;
 2004 }
 2005 
 2006 
 2007 
 2008 void MacroAssembler::decrementl(Register reg, int value) {
 2009   if (value == min_jint) {subl(reg, value) ; return; }
 2010   if (value <  0) { incrementl(reg, -value); return; }
 2011   if (value == 0) {                        ; return; }
 2012   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2013   /* else */      { subl(reg, value)       ; return; }
 2014 }
 2015 
 2016 void MacroAssembler::decrementl(Address dst, int value) {
 2017   if (value == min_jint) {subl(dst, value) ; return; }
 2018   if (value <  0) { incrementl(dst, -value); return; }
 2019   if (value == 0) {                        ; return; }
 2020   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2021   /* else */      { subl(dst, value)       ; return; }
 2022 }
 2023 
 2024 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2025   assert(shift_value > 0, "illegal shift value");
 2026   Label _is_positive;
 2027   testl (reg, reg);
 2028   jcc (Assembler::positive, _is_positive);
 2029   int offset = (1 << shift_value) - 1 ;
 2030 
 2031   if (offset == 1) {
 2032     incrementl(reg);
 2033   } else {
 2034     addl(reg, offset);
 2035   }
 2036 
 2037   bind (_is_positive);
 2038   sarl(reg, shift_value);
 2039 }
 2040 
 2041 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2042   assert(rscratch != noreg || always_reachable(src), "missing");
 2043 
 2044   if (reachable(src)) {
 2045     Assembler::divsd(dst, as_Address(src));
 2046   } else {
 2047     lea(rscratch, src);
 2048     Assembler::divsd(dst, Address(rscratch, 0));
 2049   }
 2050 }
 2051 
 2052 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2053   assert(rscratch != noreg || always_reachable(src), "missing");
 2054 
 2055   if (reachable(src)) {
 2056     Assembler::divss(dst, as_Address(src));
 2057   } else {
 2058     lea(rscratch, src);
 2059     Assembler::divss(dst, Address(rscratch, 0));
 2060   }
 2061 }
 2062 
 2063 void MacroAssembler::enter() {
 2064   push(rbp);
 2065   mov(rbp, rsp);
 2066 }
 2067 
 2068 void MacroAssembler::post_call_nop() {
 2069   if (!Continuations::enabled()) {
 2070     return;
 2071   }
 2072   InstructionMark im(this);
 2073   relocate(post_call_nop_Relocation::spec());
 2074   InlineSkippedInstructionsCounter skipCounter(this);
 2075   emit_int8((uint8_t)0x0f);
 2076   emit_int8((uint8_t)0x1f);
 2077   emit_int8((uint8_t)0x84);
 2078   emit_int8((uint8_t)0x00);
 2079   emit_int32(0x00);
 2080 }
 2081 
 2082 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2083 void MacroAssembler::fat_nop() {
 2084   if (UseAddressNop) {
 2085     addr_nop_5();
 2086   } else {
 2087     emit_int8((uint8_t)0x26); // es:
 2088     emit_int8((uint8_t)0x2e); // cs:
 2089     emit_int8((uint8_t)0x64); // fs:
 2090     emit_int8((uint8_t)0x65); // gs:
 2091     emit_int8((uint8_t)0x90);
 2092   }
 2093 }
 2094 
 2095 #ifndef _LP64
 2096 void MacroAssembler::fcmp(Register tmp) {
 2097   fcmp(tmp, 1, true, true);
 2098 }
 2099 
 2100 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2101   assert(!pop_right || pop_left, "usage error");
 2102   if (VM_Version::supports_cmov()) {
 2103     assert(tmp == noreg, "unneeded temp");
 2104     if (pop_left) {
 2105       fucomip(index);
 2106     } else {
 2107       fucomi(index);
 2108     }
 2109     if (pop_right) {
 2110       fpop();
 2111     }
 2112   } else {
 2113     assert(tmp != noreg, "need temp");
 2114     if (pop_left) {
 2115       if (pop_right) {
 2116         fcompp();
 2117       } else {
 2118         fcomp(index);
 2119       }
 2120     } else {
 2121       fcom(index);
 2122     }
 2123     // convert FPU condition into eflags condition via rax,
 2124     save_rax(tmp);
 2125     fwait(); fnstsw_ax();
 2126     sahf();
 2127     restore_rax(tmp);
 2128   }
 2129   // condition codes set as follows:
 2130   //
 2131   // CF (corresponds to C0) if x < y
 2132   // PF (corresponds to C2) if unordered
 2133   // ZF (corresponds to C3) if x = y
 2134 }
 2135 
 2136 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2137   fcmp2int(dst, unordered_is_less, 1, true, true);
 2138 }
 2139 
 2140 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2141   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2142   Label L;
 2143   if (unordered_is_less) {
 2144     movl(dst, -1);
 2145     jcc(Assembler::parity, L);
 2146     jcc(Assembler::below , L);
 2147     movl(dst, 0);
 2148     jcc(Assembler::equal , L);
 2149     increment(dst);
 2150   } else { // unordered is greater
 2151     movl(dst, 1);
 2152     jcc(Assembler::parity, L);
 2153     jcc(Assembler::above , L);
 2154     movl(dst, 0);
 2155     jcc(Assembler::equal , L);
 2156     decrementl(dst);
 2157   }
 2158   bind(L);
 2159 }
 2160 
 2161 void MacroAssembler::fld_d(AddressLiteral src) {
 2162   fld_d(as_Address(src));
 2163 }
 2164 
 2165 void MacroAssembler::fld_s(AddressLiteral src) {
 2166   fld_s(as_Address(src));
 2167 }
 2168 
 2169 void MacroAssembler::fldcw(AddressLiteral src) {
 2170   fldcw(as_Address(src));
 2171 }
 2172 
 2173 void MacroAssembler::fpop() {
 2174   ffree();
 2175   fincstp();
 2176 }
 2177 
 2178 void MacroAssembler::fremr(Register tmp) {
 2179   save_rax(tmp);
 2180   { Label L;
 2181     bind(L);
 2182     fprem();
 2183     fwait(); fnstsw_ax();
 2184     sahf();
 2185     jcc(Assembler::parity, L);
 2186   }
 2187   restore_rax(tmp);
 2188   // Result is in ST0.
 2189   // Note: fxch & fpop to get rid of ST1
 2190   // (otherwise FPU stack could overflow eventually)
 2191   fxch(1);
 2192   fpop();
 2193 }
 2194 
 2195 void MacroAssembler::empty_FPU_stack() {
 2196   if (VM_Version::supports_mmx()) {
 2197     emms();
 2198   } else {
 2199     for (int i = 8; i-- > 0; ) ffree(i);
 2200   }
 2201 }
 2202 #endif // !LP64
 2203 
 2204 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2205   assert(rscratch != noreg || always_reachable(src), "missing");
 2206   if (reachable(src)) {
 2207     Assembler::mulpd(dst, as_Address(src));
 2208   } else {
 2209     lea(rscratch, src);
 2210     Assembler::mulpd(dst, Address(rscratch, 0));
 2211   }
 2212 }
 2213 
 2214 void MacroAssembler::load_float(Address src) {
 2215 #ifdef _LP64
 2216   movflt(xmm0, src);
 2217 #else
 2218   if (UseSSE >= 1) {
 2219     movflt(xmm0, src);
 2220   } else {
 2221     fld_s(src);
 2222   }
 2223 #endif // LP64
 2224 }
 2225 
 2226 void MacroAssembler::store_float(Address dst) {
 2227 #ifdef _LP64
 2228   movflt(dst, xmm0);
 2229 #else
 2230   if (UseSSE >= 1) {
 2231     movflt(dst, xmm0);
 2232   } else {
 2233     fstp_s(dst);
 2234   }
 2235 #endif // LP64
 2236 }
 2237 
 2238 void MacroAssembler::load_double(Address src) {
 2239 #ifdef _LP64
 2240   movdbl(xmm0, src);
 2241 #else
 2242   if (UseSSE >= 2) {
 2243     movdbl(xmm0, src);
 2244   } else {
 2245     fld_d(src);
 2246   }
 2247 #endif // LP64
 2248 }
 2249 
 2250 void MacroAssembler::store_double(Address dst) {
 2251 #ifdef _LP64
 2252   movdbl(dst, xmm0);
 2253 #else
 2254   if (UseSSE >= 2) {
 2255     movdbl(dst, xmm0);
 2256   } else {
 2257     fstp_d(dst);
 2258   }
 2259 #endif // LP64
 2260 }
 2261 
 2262 // dst = c = a * b + c
 2263 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2264   Assembler::vfmadd231sd(c, a, b);
 2265   if (dst != c) {
 2266     movdbl(dst, c);
 2267   }
 2268 }
 2269 
 2270 // dst = c = a * b + c
 2271 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2272   Assembler::vfmadd231ss(c, a, b);
 2273   if (dst != c) {
 2274     movflt(dst, c);
 2275   }
 2276 }
 2277 
 2278 // dst = c = a * b + c
 2279 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2280   Assembler::vfmadd231pd(c, a, b, vector_len);
 2281   if (dst != c) {
 2282     vmovdqu(dst, c);
 2283   }
 2284 }
 2285 
 2286 // dst = c = a * b + c
 2287 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2288   Assembler::vfmadd231ps(c, a, b, vector_len);
 2289   if (dst != c) {
 2290     vmovdqu(dst, c);
 2291   }
 2292 }
 2293 
 2294 // dst = c = a * b + c
 2295 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2296   Assembler::vfmadd231pd(c, a, b, vector_len);
 2297   if (dst != c) {
 2298     vmovdqu(dst, c);
 2299   }
 2300 }
 2301 
 2302 // dst = c = a * b + c
 2303 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2304   Assembler::vfmadd231ps(c, a, b, vector_len);
 2305   if (dst != c) {
 2306     vmovdqu(dst, c);
 2307   }
 2308 }
 2309 
 2310 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2311   assert(rscratch != noreg || always_reachable(dst), "missing");
 2312 
 2313   if (reachable(dst)) {
 2314     incrementl(as_Address(dst));
 2315   } else {
 2316     lea(rscratch, dst);
 2317     incrementl(Address(rscratch, 0));
 2318   }
 2319 }
 2320 
 2321 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2322   incrementl(as_Address(dst, rscratch));
 2323 }
 2324 
 2325 void MacroAssembler::incrementl(Register reg, int value) {
 2326   if (value == min_jint) {addl(reg, value) ; return; }
 2327   if (value <  0) { decrementl(reg, -value); return; }
 2328   if (value == 0) {                        ; return; }
 2329   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2330   /* else */      { addl(reg, value)       ; return; }
 2331 }
 2332 
 2333 void MacroAssembler::incrementl(Address dst, int value) {
 2334   if (value == min_jint) {addl(dst, value) ; return; }
 2335   if (value <  0) { decrementl(dst, -value); return; }
 2336   if (value == 0) {                        ; return; }
 2337   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2338   /* else */      { addl(dst, value)       ; return; }
 2339 }
 2340 
 2341 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2342   assert(rscratch != noreg || always_reachable(dst), "missing");
 2343 
 2344   if (reachable(dst)) {
 2345     jmp_literal(dst.target(), dst.rspec());
 2346   } else {
 2347     lea(rscratch, dst);
 2348     jmp(rscratch);
 2349   }
 2350 }
 2351 
 2352 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2353   assert(rscratch != noreg || always_reachable(dst), "missing");
 2354 
 2355   if (reachable(dst)) {
 2356     InstructionMark im(this);
 2357     relocate(dst.reloc());
 2358     const int short_size = 2;
 2359     const int long_size = 6;
 2360     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2361     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2362       // 0111 tttn #8-bit disp
 2363       emit_int8(0x70 | cc);
 2364       emit_int8((offs - short_size) & 0xFF);
 2365     } else {
 2366       // 0000 1111 1000 tttn #32-bit disp
 2367       emit_int8(0x0F);
 2368       emit_int8((unsigned char)(0x80 | cc));
 2369       emit_int32(offs - long_size);
 2370     }
 2371   } else {
 2372 #ifdef ASSERT
 2373     warning("reversing conditional branch");
 2374 #endif /* ASSERT */
 2375     Label skip;
 2376     jccb(reverse[cc], skip);
 2377     lea(rscratch, dst);
 2378     Assembler::jmp(rscratch);
 2379     bind(skip);
 2380   }
 2381 }
 2382 
 2383 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2384   assert(rscratch != noreg || always_reachable(src), "missing");
 2385 
 2386   if (reachable(src)) {
 2387     Assembler::ldmxcsr(as_Address(src));
 2388   } else {
 2389     lea(rscratch, src);
 2390     Assembler::ldmxcsr(Address(rscratch, 0));
 2391   }
 2392 }
 2393 
 2394 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2395   int off;
 2396   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2397     off = offset();
 2398     movsbl(dst, src); // movsxb
 2399   } else {
 2400     off = load_unsigned_byte(dst, src);
 2401     shll(dst, 24);
 2402     sarl(dst, 24);
 2403   }
 2404   return off;
 2405 }
 2406 
 2407 // Note: load_signed_short used to be called load_signed_word.
 2408 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2409 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2410 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2411 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2412   int off;
 2413   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2414     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2415     // version but this is what 64bit has always done. This seems to imply
 2416     // that users are only using 32bits worth.
 2417     off = offset();
 2418     movswl(dst, src); // movsxw
 2419   } else {
 2420     off = load_unsigned_short(dst, src);
 2421     shll(dst, 16);
 2422     sarl(dst, 16);
 2423   }
 2424   return off;
 2425 }
 2426 
 2427 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2428   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2429   // and "3.9 Partial Register Penalties", p. 22).
 2430   int off;
 2431   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2432     off = offset();
 2433     movzbl(dst, src); // movzxb
 2434   } else {
 2435     xorl(dst, dst);
 2436     off = offset();
 2437     movb(dst, src);
 2438   }
 2439   return off;
 2440 }
 2441 
 2442 // Note: load_unsigned_short used to be called load_unsigned_word.
 2443 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2444   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2445   // and "3.9 Partial Register Penalties", p. 22).
 2446   int off;
 2447   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2448     off = offset();
 2449     movzwl(dst, src); // movzxw
 2450   } else {
 2451     xorl(dst, dst);
 2452     off = offset();
 2453     movw(dst, src);
 2454   }
 2455   return off;
 2456 }
 2457 
 2458 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2459   switch (size_in_bytes) {
 2460 #ifndef _LP64
 2461   case  8:
 2462     assert(dst2 != noreg, "second dest register required");
 2463     movl(dst,  src);
 2464     movl(dst2, src.plus_disp(BytesPerInt));
 2465     break;
 2466 #else
 2467   case  8:  movq(dst, src); break;
 2468 #endif
 2469   case  4:  movl(dst, src); break;
 2470   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2471   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2472   default:  ShouldNotReachHere();
 2473   }
 2474 }
 2475 
 2476 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2477   switch (size_in_bytes) {
 2478 #ifndef _LP64
 2479   case  8:
 2480     assert(src2 != noreg, "second source register required");
 2481     movl(dst,                        src);
 2482     movl(dst.plus_disp(BytesPerInt), src2);
 2483     break;
 2484 #else
 2485   case  8:  movq(dst, src); break;
 2486 #endif
 2487   case  4:  movl(dst, src); break;
 2488   case  2:  movw(dst, src); break;
 2489   case  1:  movb(dst, src); break;
 2490   default:  ShouldNotReachHere();
 2491   }
 2492 }
 2493 
 2494 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2495   assert(rscratch != noreg || always_reachable(dst), "missing");
 2496 
 2497   if (reachable(dst)) {
 2498     movl(as_Address(dst), src);
 2499   } else {
 2500     lea(rscratch, dst);
 2501     movl(Address(rscratch, 0), src);
 2502   }
 2503 }
 2504 
 2505 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2506   if (reachable(src)) {
 2507     movl(dst, as_Address(src));
 2508   } else {
 2509     lea(dst, src);
 2510     movl(dst, Address(dst, 0));
 2511   }
 2512 }
 2513 
 2514 // C++ bool manipulation
 2515 
 2516 void MacroAssembler::movbool(Register dst, Address src) {
 2517   if(sizeof(bool) == 1)
 2518     movb(dst, src);
 2519   else if(sizeof(bool) == 2)
 2520     movw(dst, src);
 2521   else if(sizeof(bool) == 4)
 2522     movl(dst, src);
 2523   else
 2524     // unsupported
 2525     ShouldNotReachHere();
 2526 }
 2527 
 2528 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2529   if(sizeof(bool) == 1)
 2530     movb(dst, (int) boolconst);
 2531   else if(sizeof(bool) == 2)
 2532     movw(dst, (int) boolconst);
 2533   else if(sizeof(bool) == 4)
 2534     movl(dst, (int) boolconst);
 2535   else
 2536     // unsupported
 2537     ShouldNotReachHere();
 2538 }
 2539 
 2540 void MacroAssembler::movbool(Address dst, Register src) {
 2541   if(sizeof(bool) == 1)
 2542     movb(dst, src);
 2543   else if(sizeof(bool) == 2)
 2544     movw(dst, src);
 2545   else if(sizeof(bool) == 4)
 2546     movl(dst, src);
 2547   else
 2548     // unsupported
 2549     ShouldNotReachHere();
 2550 }
 2551 
 2552 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2553   assert(rscratch != noreg || always_reachable(src), "missing");
 2554 
 2555   if (reachable(src)) {
 2556     movdl(dst, as_Address(src));
 2557   } else {
 2558     lea(rscratch, src);
 2559     movdl(dst, Address(rscratch, 0));
 2560   }
 2561 }
 2562 
 2563 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2564   assert(rscratch != noreg || always_reachable(src), "missing");
 2565 
 2566   if (reachable(src)) {
 2567     movq(dst, as_Address(src));
 2568   } else {
 2569     lea(rscratch, src);
 2570     movq(dst, Address(rscratch, 0));
 2571   }
 2572 }
 2573 
 2574 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2575   assert(rscratch != noreg || always_reachable(src), "missing");
 2576 
 2577   if (reachable(src)) {
 2578     if (UseXmmLoadAndClearUpper) {
 2579       movsd (dst, as_Address(src));
 2580     } else {
 2581       movlpd(dst, as_Address(src));
 2582     }
 2583   } else {
 2584     lea(rscratch, src);
 2585     if (UseXmmLoadAndClearUpper) {
 2586       movsd (dst, Address(rscratch, 0));
 2587     } else {
 2588       movlpd(dst, Address(rscratch, 0));
 2589     }
 2590   }
 2591 }
 2592 
 2593 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2594   assert(rscratch != noreg || always_reachable(src), "missing");
 2595 
 2596   if (reachable(src)) {
 2597     movss(dst, as_Address(src));
 2598   } else {
 2599     lea(rscratch, src);
 2600     movss(dst, Address(rscratch, 0));
 2601   }
 2602 }
 2603 
 2604 void MacroAssembler::movptr(Register dst, Register src) {
 2605   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2606 }
 2607 
 2608 void MacroAssembler::movptr(Register dst, Address src) {
 2609   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2610 }
 2611 
 2612 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2613 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2614 #ifdef _LP64
 2615   if (is_uimm32(src)) {
 2616     movl(dst, checked_cast<uint32_t>(src));
 2617   } else if (is_simm32(src)) {
 2618     movq(dst, checked_cast<int32_t>(src));
 2619   } else {
 2620     mov64(dst, src);
 2621   }
 2622 #else
 2623   movl(dst, src);
 2624 #endif
 2625 }
 2626 
 2627 void MacroAssembler::movptr(Address dst, Register src) {
 2628   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2629 }
 2630 
 2631 void MacroAssembler::movptr(Address dst, int32_t src) {
 2632   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2633 }
 2634 
 2635 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2636   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2637   Assembler::movdqu(dst, src);
 2638 }
 2639 
 2640 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2641   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2642   Assembler::movdqu(dst, src);
 2643 }
 2644 
 2645 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2646   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2647   Assembler::movdqu(dst, src);
 2648 }
 2649 
 2650 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2651   assert(rscratch != noreg || always_reachable(src), "missing");
 2652 
 2653   if (reachable(src)) {
 2654     movdqu(dst, as_Address(src));
 2655   } else {
 2656     lea(rscratch, src);
 2657     movdqu(dst, Address(rscratch, 0));
 2658   }
 2659 }
 2660 
 2661 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2662   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2663   Assembler::vmovdqu(dst, src);
 2664 }
 2665 
 2666 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2667   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2668   Assembler::vmovdqu(dst, src);
 2669 }
 2670 
 2671 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2672   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2673   Assembler::vmovdqu(dst, src);
 2674 }
 2675 
 2676 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2677   assert(rscratch != noreg || always_reachable(src), "missing");
 2678 
 2679   if (reachable(src)) {
 2680     vmovdqu(dst, as_Address(src));
 2681   }
 2682   else {
 2683     lea(rscratch, src);
 2684     vmovdqu(dst, Address(rscratch, 0));
 2685   }
 2686 }
 2687 
 2688 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2689   assert(rscratch != noreg || always_reachable(src), "missing");
 2690 
 2691   if (vector_len == AVX_512bit) {
 2692     evmovdquq(dst, src, AVX_512bit, rscratch);
 2693   } else if (vector_len == AVX_256bit) {
 2694     vmovdqu(dst, src, rscratch);
 2695   } else {
 2696     movdqu(dst, src, rscratch);
 2697   }
 2698 }
 2699 
 2700 void MacroAssembler::kmov(KRegister dst, Address src) {
 2701   if (VM_Version::supports_avx512bw()) {
 2702     kmovql(dst, src);
 2703   } else {
 2704     assert(VM_Version::supports_evex(), "");
 2705     kmovwl(dst, src);
 2706   }
 2707 }
 2708 
 2709 void MacroAssembler::kmov(Address dst, KRegister src) {
 2710   if (VM_Version::supports_avx512bw()) {
 2711     kmovql(dst, src);
 2712   } else {
 2713     assert(VM_Version::supports_evex(), "");
 2714     kmovwl(dst, src);
 2715   }
 2716 }
 2717 
 2718 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2719   if (VM_Version::supports_avx512bw()) {
 2720     kmovql(dst, src);
 2721   } else {
 2722     assert(VM_Version::supports_evex(), "");
 2723     kmovwl(dst, src);
 2724   }
 2725 }
 2726 
 2727 void MacroAssembler::kmov(Register dst, KRegister src) {
 2728   if (VM_Version::supports_avx512bw()) {
 2729     kmovql(dst, src);
 2730   } else {
 2731     assert(VM_Version::supports_evex(), "");
 2732     kmovwl(dst, src);
 2733   }
 2734 }
 2735 
 2736 void MacroAssembler::kmov(KRegister dst, Register src) {
 2737   if (VM_Version::supports_avx512bw()) {
 2738     kmovql(dst, src);
 2739   } else {
 2740     assert(VM_Version::supports_evex(), "");
 2741     kmovwl(dst, src);
 2742   }
 2743 }
 2744 
 2745 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2746   assert(rscratch != noreg || always_reachable(src), "missing");
 2747 
 2748   if (reachable(src)) {
 2749     kmovql(dst, as_Address(src));
 2750   } else {
 2751     lea(rscratch, src);
 2752     kmovql(dst, Address(rscratch, 0));
 2753   }
 2754 }
 2755 
 2756 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2757   assert(rscratch != noreg || always_reachable(src), "missing");
 2758 
 2759   if (reachable(src)) {
 2760     kmovwl(dst, as_Address(src));
 2761   } else {
 2762     lea(rscratch, src);
 2763     kmovwl(dst, Address(rscratch, 0));
 2764   }
 2765 }
 2766 
 2767 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2768                                int vector_len, Register rscratch) {
 2769   assert(rscratch != noreg || always_reachable(src), "missing");
 2770 
 2771   if (reachable(src)) {
 2772     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2773   } else {
 2774     lea(rscratch, src);
 2775     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2776   }
 2777 }
 2778 
 2779 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2780                                int vector_len, Register rscratch) {
 2781   assert(rscratch != noreg || always_reachable(src), "missing");
 2782 
 2783   if (reachable(src)) {
 2784     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2785   } else {
 2786     lea(rscratch, src);
 2787     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2788   }
 2789 }
 2790 
 2791 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2792   assert(rscratch != noreg || always_reachable(src), "missing");
 2793 
 2794   if (reachable(src)) {
 2795     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2796   } else {
 2797     lea(rscratch, src);
 2798     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2799   }
 2800 }
 2801 
 2802 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2803   assert(rscratch != noreg || always_reachable(src), "missing");
 2804 
 2805   if (reachable(src)) {
 2806     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2807   } else {
 2808     lea(rscratch, src);
 2809     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2810   }
 2811 }
 2812 
 2813 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2814   assert(rscratch != noreg || always_reachable(src), "missing");
 2815 
 2816   if (reachable(src)) {
 2817     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2818   } else {
 2819     lea(rscratch, src);
 2820     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2821   }
 2822 }
 2823 
 2824 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2825   assert(rscratch != noreg || always_reachable(src), "missing");
 2826 
 2827   if (reachable(src)) {
 2828     Assembler::movdqa(dst, as_Address(src));
 2829   } else {
 2830     lea(rscratch, src);
 2831     Assembler::movdqa(dst, Address(rscratch, 0));
 2832   }
 2833 }
 2834 
 2835 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2836   assert(rscratch != noreg || always_reachable(src), "missing");
 2837 
 2838   if (reachable(src)) {
 2839     Assembler::movsd(dst, as_Address(src));
 2840   } else {
 2841     lea(rscratch, src);
 2842     Assembler::movsd(dst, Address(rscratch, 0));
 2843   }
 2844 }
 2845 
 2846 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2847   assert(rscratch != noreg || always_reachable(src), "missing");
 2848 
 2849   if (reachable(src)) {
 2850     Assembler::movss(dst, as_Address(src));
 2851   } else {
 2852     lea(rscratch, src);
 2853     Assembler::movss(dst, Address(rscratch, 0));
 2854   }
 2855 }
 2856 
 2857 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2858   assert(rscratch != noreg || always_reachable(src), "missing");
 2859 
 2860   if (reachable(src)) {
 2861     Assembler::movddup(dst, as_Address(src));
 2862   } else {
 2863     lea(rscratch, src);
 2864     Assembler::movddup(dst, Address(rscratch, 0));
 2865   }
 2866 }
 2867 
 2868 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2869   assert(rscratch != noreg || always_reachable(src), "missing");
 2870 
 2871   if (reachable(src)) {
 2872     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2873   } else {
 2874     lea(rscratch, src);
 2875     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2876   }
 2877 }
 2878 
 2879 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2880   assert(rscratch != noreg || always_reachable(src), "missing");
 2881 
 2882   if (reachable(src)) {
 2883     Assembler::mulsd(dst, as_Address(src));
 2884   } else {
 2885     lea(rscratch, src);
 2886     Assembler::mulsd(dst, Address(rscratch, 0));
 2887   }
 2888 }
 2889 
 2890 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2891   assert(rscratch != noreg || always_reachable(src), "missing");
 2892 
 2893   if (reachable(src)) {
 2894     Assembler::mulss(dst, as_Address(src));
 2895   } else {
 2896     lea(rscratch, src);
 2897     Assembler::mulss(dst, Address(rscratch, 0));
 2898   }
 2899 }
 2900 
 2901 void MacroAssembler::null_check(Register reg, int offset) {
 2902   if (needs_explicit_null_check(offset)) {
 2903     // provoke OS null exception if reg is null by
 2904     // accessing M[reg] w/o changing any (non-CC) registers
 2905     // NOTE: cmpl is plenty here to provoke a segv
 2906     cmpptr(rax, Address(reg, 0));
 2907     // Note: should probably use testl(rax, Address(reg, 0));
 2908     //       may be shorter code (however, this version of
 2909     //       testl needs to be implemented first)
 2910   } else {
 2911     // nothing to do, (later) access of M[reg + offset]
 2912     // will provoke OS null exception if reg is null
 2913   }
 2914 }
 2915 
 2916 void MacroAssembler::os_breakpoint() {
 2917   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2918   // (e.g., MSVC can't call ps() otherwise)
 2919   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2920 }
 2921 
 2922 void MacroAssembler::unimplemented(const char* what) {
 2923   const char* buf = nullptr;
 2924   {
 2925     ResourceMark rm;
 2926     stringStream ss;
 2927     ss.print("unimplemented: %s", what);
 2928     buf = code_string(ss.as_string());
 2929   }
 2930   stop(buf);
 2931 }
 2932 
 2933 #ifdef _LP64
 2934 #define XSTATE_BV 0x200
 2935 #endif
 2936 
 2937 void MacroAssembler::pop_CPU_state() {
 2938   pop_FPU_state();
 2939   pop_IU_state();
 2940 }
 2941 
 2942 void MacroAssembler::pop_FPU_state() {
 2943 #ifndef _LP64
 2944   frstor(Address(rsp, 0));
 2945 #else
 2946   fxrstor(Address(rsp, 0));
 2947 #endif
 2948   addptr(rsp, FPUStateSizeInWords * wordSize);
 2949 }
 2950 
 2951 void MacroAssembler::pop_IU_state() {
 2952   popa();
 2953   LP64_ONLY(addq(rsp, 8));
 2954   popf();
 2955 }
 2956 
 2957 // Save Integer and Float state
 2958 // Warning: Stack must be 16 byte aligned (64bit)
 2959 void MacroAssembler::push_CPU_state() {
 2960   push_IU_state();
 2961   push_FPU_state();
 2962 }
 2963 
 2964 void MacroAssembler::push_FPU_state() {
 2965   subptr(rsp, FPUStateSizeInWords * wordSize);
 2966 #ifndef _LP64
 2967   fnsave(Address(rsp, 0));
 2968   fwait();
 2969 #else
 2970   fxsave(Address(rsp, 0));
 2971 #endif // LP64
 2972 }
 2973 
 2974 void MacroAssembler::push_IU_state() {
 2975   // Push flags first because pusha kills them
 2976   pushf();
 2977   // Make sure rsp stays 16-byte aligned
 2978   LP64_ONLY(subq(rsp, 8));
 2979   pusha();
 2980 }
 2981 
 2982 void MacroAssembler::push_cont_fastpath() {
 2983   if (!Continuations::enabled()) return;
 2984 
 2985 #ifndef _LP64
 2986   Register rthread = rax;
 2987   Register rrealsp = rbx;
 2988   push(rthread);
 2989   push(rrealsp);
 2990 
 2991   get_thread(rthread);
 2992 
 2993   // The code below wants the original RSP.
 2994   // Move it back after the pushes above.
 2995   movptr(rrealsp, rsp);
 2996   addptr(rrealsp, 2*wordSize);
 2997 #else
 2998   Register rthread = r15_thread;
 2999   Register rrealsp = rsp;
 3000 #endif
 3001 
 3002   Label done;
 3003   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3004   jccb(Assembler::belowEqual, done);
 3005   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3006   bind(done);
 3007 
 3008 #ifndef _LP64
 3009   pop(rrealsp);
 3010   pop(rthread);
 3011 #endif
 3012 }
 3013 
 3014 void MacroAssembler::pop_cont_fastpath() {
 3015   if (!Continuations::enabled()) return;
 3016 
 3017 #ifndef _LP64
 3018   Register rthread = rax;
 3019   Register rrealsp = rbx;
 3020   push(rthread);
 3021   push(rrealsp);
 3022 
 3023   get_thread(rthread);
 3024 
 3025   // The code below wants the original RSP.
 3026   // Move it back after the pushes above.
 3027   movptr(rrealsp, rsp);
 3028   addptr(rrealsp, 2*wordSize);
 3029 #else
 3030   Register rthread = r15_thread;
 3031   Register rrealsp = rsp;
 3032 #endif
 3033 
 3034   Label done;
 3035   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3036   jccb(Assembler::below, done);
 3037   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3038   bind(done);
 3039 
 3040 #ifndef _LP64
 3041   pop(rrealsp);
 3042   pop(rthread);
 3043 #endif
 3044 }
 3045 
 3046 void MacroAssembler::inc_held_monitor_count() {
 3047 #ifndef _LP64
 3048   Register thread = rax;
 3049   push(thread);
 3050   get_thread(thread);
 3051   incrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3052   pop(thread);
 3053 #else // LP64
 3054   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3055 #endif
 3056 }
 3057 
 3058 void MacroAssembler::dec_held_monitor_count() {
 3059 #ifndef _LP64
 3060   Register thread = rax;
 3061   push(thread);
 3062   get_thread(thread);
 3063   decrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3064   pop(thread);
 3065 #else // LP64
 3066   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3067 #endif
 3068 }
 3069 
 3070 #ifdef ASSERT
 3071 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3072 #ifdef _LP64
 3073   Label no_cont;
 3074   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3075   testl(cont, cont);
 3076   jcc(Assembler::zero, no_cont);
 3077   stop(name);
 3078   bind(no_cont);
 3079 #else
 3080   Unimplemented();
 3081 #endif
 3082 }
 3083 #endif
 3084 
 3085 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3086   if (!java_thread->is_valid()) {
 3087     java_thread = rdi;
 3088     get_thread(java_thread);
 3089   }
 3090   // we must set sp to zero to clear frame
 3091   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3092   // must clear fp, so that compiled frames are not confused; it is
 3093   // possible that we need it only for debugging
 3094   if (clear_fp) {
 3095     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3096   }
 3097   // Always clear the pc because it could have been set by make_walkable()
 3098   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3099   vzeroupper();
 3100 }
 3101 
 3102 void MacroAssembler::restore_rax(Register tmp) {
 3103   if (tmp == noreg) pop(rax);
 3104   else if (tmp != rax) mov(rax, tmp);
 3105 }
 3106 
 3107 void MacroAssembler::round_to(Register reg, int modulus) {
 3108   addptr(reg, modulus - 1);
 3109   andptr(reg, -modulus);
 3110 }
 3111 
 3112 void MacroAssembler::save_rax(Register tmp) {
 3113   if (tmp == noreg) push(rax);
 3114   else if (tmp != rax) mov(tmp, rax);
 3115 }
 3116 
 3117 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3118   if (at_return) {
 3119     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3120     // we may safely use rsp instead to perform the stack watermark check.
 3121     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3122     jcc(Assembler::above, slow_path);
 3123     return;
 3124   }
 3125   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3126   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3127 }
 3128 
 3129 // Calls to C land
 3130 //
 3131 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3132 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3133 // has to be reset to 0. This is required to allow proper stack traversal.
 3134 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3135                                          Register last_java_sp,
 3136                                          Register last_java_fp,
 3137                                          address  last_java_pc,
 3138                                          Register rscratch) {
 3139   vzeroupper();
 3140   // determine java_thread register
 3141   if (!java_thread->is_valid()) {
 3142     java_thread = rdi;
 3143     get_thread(java_thread);
 3144   }
 3145   // determine last_java_sp register
 3146   if (!last_java_sp->is_valid()) {
 3147     last_java_sp = rsp;
 3148   }
 3149   // last_java_fp is optional
 3150   if (last_java_fp->is_valid()) {
 3151     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3152   }
 3153   // last_java_pc is optional
 3154   if (last_java_pc != nullptr) {
 3155     Address java_pc(java_thread,
 3156                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3157     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3158   }
 3159   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3160 }
 3161 
 3162 void MacroAssembler::shlptr(Register dst, int imm8) {
 3163   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3164 }
 3165 
 3166 void MacroAssembler::shrptr(Register dst, int imm8) {
 3167   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3168 }
 3169 
 3170 void MacroAssembler::sign_extend_byte(Register reg) {
 3171   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3172     movsbl(reg, reg); // movsxb
 3173   } else {
 3174     shll(reg, 24);
 3175     sarl(reg, 24);
 3176   }
 3177 }
 3178 
 3179 void MacroAssembler::sign_extend_short(Register reg) {
 3180   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3181     movswl(reg, reg); // movsxw
 3182   } else {
 3183     shll(reg, 16);
 3184     sarl(reg, 16);
 3185   }
 3186 }
 3187 
 3188 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3189   if (imm32 >= 0 && is8bit(imm32)) {
 3190     testb(dst, imm32);
 3191   } else {
 3192     Assembler::testl(dst, imm32);
 3193   }
 3194 }
 3195 
 3196 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3197   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3198     testb(dst, imm32);
 3199   } else {
 3200     Assembler::testl(dst, imm32);
 3201   }
 3202 }
 3203 
 3204 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3205   assert(always_reachable(src), "Address should be reachable");
 3206   testl(dst, as_Address(src));
 3207 }
 3208 
 3209 #ifdef _LP64
 3210 
 3211 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3212   if (imm32 >= 0) {
 3213     testl(dst, imm32);
 3214   } else {
 3215     Assembler::testq(dst, imm32);
 3216   }
 3217 }
 3218 
 3219 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3220   if (imm32 >= 0) {
 3221     testl(dst, imm32);
 3222   } else {
 3223     Assembler::testq(dst, imm32);
 3224   }
 3225 }
 3226 
 3227 #endif
 3228 
 3229 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3230   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3231   Assembler::pcmpeqb(dst, src);
 3232 }
 3233 
 3234 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3235   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3236   Assembler::pcmpeqw(dst, src);
 3237 }
 3238 
 3239 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3240   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3241   Assembler::pcmpestri(dst, src, imm8);
 3242 }
 3243 
 3244 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3245   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3246   Assembler::pcmpestri(dst, src, imm8);
 3247 }
 3248 
 3249 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3250   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3251   Assembler::pmovzxbw(dst, src);
 3252 }
 3253 
 3254 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3255   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3256   Assembler::pmovzxbw(dst, src);
 3257 }
 3258 
 3259 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3260   assert((src->encoding() < 16),"XMM register should be 0-15");
 3261   Assembler::pmovmskb(dst, src);
 3262 }
 3263 
 3264 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3265   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3266   Assembler::ptest(dst, src);
 3267 }
 3268 
 3269 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3270   assert(rscratch != noreg || always_reachable(src), "missing");
 3271 
 3272   if (reachable(src)) {
 3273     Assembler::sqrtss(dst, as_Address(src));
 3274   } else {
 3275     lea(rscratch, src);
 3276     Assembler::sqrtss(dst, Address(rscratch, 0));
 3277   }
 3278 }
 3279 
 3280 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3281   assert(rscratch != noreg || always_reachable(src), "missing");
 3282 
 3283   if (reachable(src)) {
 3284     Assembler::subsd(dst, as_Address(src));
 3285   } else {
 3286     lea(rscratch, src);
 3287     Assembler::subsd(dst, Address(rscratch, 0));
 3288   }
 3289 }
 3290 
 3291 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3292   assert(rscratch != noreg || always_reachable(src), "missing");
 3293 
 3294   if (reachable(src)) {
 3295     Assembler::roundsd(dst, as_Address(src), rmode);
 3296   } else {
 3297     lea(rscratch, src);
 3298     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3299   }
 3300 }
 3301 
 3302 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3303   assert(rscratch != noreg || always_reachable(src), "missing");
 3304 
 3305   if (reachable(src)) {
 3306     Assembler::subss(dst, as_Address(src));
 3307   } else {
 3308     lea(rscratch, src);
 3309     Assembler::subss(dst, Address(rscratch, 0));
 3310   }
 3311 }
 3312 
 3313 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3314   assert(rscratch != noreg || always_reachable(src), "missing");
 3315 
 3316   if (reachable(src)) {
 3317     Assembler::ucomisd(dst, as_Address(src));
 3318   } else {
 3319     lea(rscratch, src);
 3320     Assembler::ucomisd(dst, Address(rscratch, 0));
 3321   }
 3322 }
 3323 
 3324 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3325   assert(rscratch != noreg || always_reachable(src), "missing");
 3326 
 3327   if (reachable(src)) {
 3328     Assembler::ucomiss(dst, as_Address(src));
 3329   } else {
 3330     lea(rscratch, src);
 3331     Assembler::ucomiss(dst, Address(rscratch, 0));
 3332   }
 3333 }
 3334 
 3335 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3336   assert(rscratch != noreg || always_reachable(src), "missing");
 3337 
 3338   // Used in sign-bit flipping with aligned address.
 3339   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3340   if (reachable(src)) {
 3341     Assembler::xorpd(dst, as_Address(src));
 3342   } else {
 3343     lea(rscratch, src);
 3344     Assembler::xorpd(dst, Address(rscratch, 0));
 3345   }
 3346 }
 3347 
 3348 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3349   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3350     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3351   }
 3352   else {
 3353     Assembler::xorpd(dst, src);
 3354   }
 3355 }
 3356 
 3357 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3358   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3359     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3360   } else {
 3361     Assembler::xorps(dst, src);
 3362   }
 3363 }
 3364 
 3365 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3366   assert(rscratch != noreg || always_reachable(src), "missing");
 3367 
 3368   // Used in sign-bit flipping with aligned address.
 3369   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3370   if (reachable(src)) {
 3371     Assembler::xorps(dst, as_Address(src));
 3372   } else {
 3373     lea(rscratch, src);
 3374     Assembler::xorps(dst, Address(rscratch, 0));
 3375   }
 3376 }
 3377 
 3378 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3379   assert(rscratch != noreg || always_reachable(src), "missing");
 3380 
 3381   // Used in sign-bit flipping with aligned address.
 3382   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3383   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3384   if (reachable(src)) {
 3385     Assembler::pshufb(dst, as_Address(src));
 3386   } else {
 3387     lea(rscratch, src);
 3388     Assembler::pshufb(dst, Address(rscratch, 0));
 3389   }
 3390 }
 3391 
 3392 // AVX 3-operands instructions
 3393 
 3394 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3395   assert(rscratch != noreg || always_reachable(src), "missing");
 3396 
 3397   if (reachable(src)) {
 3398     vaddsd(dst, nds, as_Address(src));
 3399   } else {
 3400     lea(rscratch, src);
 3401     vaddsd(dst, nds, Address(rscratch, 0));
 3402   }
 3403 }
 3404 
 3405 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3406   assert(rscratch != noreg || always_reachable(src), "missing");
 3407 
 3408   if (reachable(src)) {
 3409     vaddss(dst, nds, as_Address(src));
 3410   } else {
 3411     lea(rscratch, src);
 3412     vaddss(dst, nds, Address(rscratch, 0));
 3413   }
 3414 }
 3415 
 3416 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3417   assert(UseAVX > 0, "requires some form of AVX");
 3418   assert(rscratch != noreg || always_reachable(src), "missing");
 3419 
 3420   if (reachable(src)) {
 3421     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3422   } else {
 3423     lea(rscratch, src);
 3424     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3425   }
 3426 }
 3427 
 3428 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3429   assert(UseAVX > 0, "requires some form of AVX");
 3430   assert(rscratch != noreg || always_reachable(src), "missing");
 3431 
 3432   if (reachable(src)) {
 3433     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3434   } else {
 3435     lea(rscratch, src);
 3436     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3437   }
 3438 }
 3439 
 3440 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3441   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3442   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3443 
 3444   vandps(dst, nds, negate_field, vector_len, rscratch);
 3445 }
 3446 
 3447 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3448   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3449   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3450 
 3451   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3452 }
 3453 
 3454 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3455   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3456   Assembler::vpaddb(dst, nds, src, vector_len);
 3457 }
 3458 
 3459 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3460   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3461   Assembler::vpaddb(dst, nds, src, vector_len);
 3462 }
 3463 
 3464 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3465   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3466   Assembler::vpaddw(dst, nds, src, vector_len);
 3467 }
 3468 
 3469 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3470   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3471   Assembler::vpaddw(dst, nds, src, vector_len);
 3472 }
 3473 
 3474 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3475   assert(rscratch != noreg || always_reachable(src), "missing");
 3476 
 3477   if (reachable(src)) {
 3478     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3479   } else {
 3480     lea(rscratch, src);
 3481     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3482   }
 3483 }
 3484 
 3485 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3486   assert(rscratch != noreg || always_reachable(src), "missing");
 3487 
 3488   if (reachable(src)) {
 3489     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3490   } else {
 3491     lea(rscratch, src);
 3492     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3493   }
 3494 }
 3495 
 3496 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3497   assert(rscratch != noreg || always_reachable(src), "missing");
 3498 
 3499   if (reachable(src)) {
 3500     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3501   } else {
 3502     lea(rscratch, src);
 3503     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3504   }
 3505 }
 3506 
 3507 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3508   assert(rscratch != noreg || always_reachable(src), "missing");
 3509 
 3510   if (reachable(src)) {
 3511     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3512   } else {
 3513     lea(rscratch, src);
 3514     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3515   }
 3516 }
 3517 
 3518 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3519   assert(rscratch != noreg || always_reachable(src), "missing");
 3520 
 3521   if (reachable(src)) {
 3522     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3523   } else {
 3524     lea(rscratch, src);
 3525     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3526   }
 3527 }
 3528 
 3529 // Vector float blend
 3530 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3531 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3532   // WARN: Allow dst == (src1|src2), mask == scratch
 3533   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3534   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3535   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3536   if (blend_emulation && scratch_available && dst_available) {
 3537     if (compute_mask) {
 3538       vpsrad(scratch, mask, 32, vector_len);
 3539       mask = scratch;
 3540     }
 3541     if (dst == src1) {
 3542       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3543       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3544     } else {
 3545       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3546       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3547     }
 3548     vpor(dst, dst, scratch, vector_len);
 3549   } else {
 3550     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3551   }
 3552 }
 3553 
 3554 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3555 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3556   // WARN: Allow dst == (src1|src2), mask == scratch
 3557   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3558   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3559   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3560   if (blend_emulation && scratch_available && dst_available) {
 3561     if (compute_mask) {
 3562       vpxor(scratch, scratch, scratch, vector_len);
 3563       vpcmpgtq(scratch, scratch, mask, vector_len);
 3564       mask = scratch;
 3565     }
 3566     if (dst == src1) {
 3567       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3568       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3569     } else {
 3570       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3571       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3572     }
 3573     vpor(dst, dst, scratch, vector_len);
 3574   } else {
 3575     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3576   }
 3577 }
 3578 
 3579 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3580   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3581   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3582 }
 3583 
 3584 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 3585   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3586   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
 3587 }
 3588 
 3589 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3590   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3591   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3592 }
 3593 
 3594 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3595   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3596   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3597 }
 3598 
 3599 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3600   assert(rscratch != noreg || always_reachable(src), "missing");
 3601 
 3602   if (reachable(src)) {
 3603     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3604   } else {
 3605     lea(rscratch, src);
 3606     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3607   }
 3608 }
 3609 
 3610 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3611                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3612   assert(rscratch != noreg || always_reachable(src), "missing");
 3613 
 3614   if (reachable(src)) {
 3615     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3616   } else {
 3617     lea(rscratch, src);
 3618     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3619   }
 3620 }
 3621 
 3622 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3623                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3624   assert(rscratch != noreg || always_reachable(src), "missing");
 3625 
 3626   if (reachable(src)) {
 3627     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3628   } else {
 3629     lea(rscratch, src);
 3630     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3631   }
 3632 }
 3633 
 3634 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3635                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3636   assert(rscratch != noreg || always_reachable(src), "missing");
 3637 
 3638   if (reachable(src)) {
 3639     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3640   } else {
 3641     lea(rscratch, src);
 3642     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3643   }
 3644 }
 3645 
 3646 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3647                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3648   assert(rscratch != noreg || always_reachable(src), "missing");
 3649 
 3650   if (reachable(src)) {
 3651     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3652   } else {
 3653     lea(rscratch, src);
 3654     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3655   }
 3656 }
 3657 
 3658 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3659   if (width == Assembler::Q) {
 3660     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3661   } else {
 3662     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3663   }
 3664 }
 3665 
 3666 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3667   int eq_cond_enc = 0x29;
 3668   int gt_cond_enc = 0x37;
 3669   if (width != Assembler::Q) {
 3670     eq_cond_enc = 0x74 + width;
 3671     gt_cond_enc = 0x64 + width;
 3672   }
 3673   switch (cond) {
 3674   case eq:
 3675     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3676     break;
 3677   case neq:
 3678     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3679     vallones(xtmp, vector_len);
 3680     vpxor(dst, xtmp, dst, vector_len);
 3681     break;
 3682   case le:
 3683     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3684     vallones(xtmp, vector_len);
 3685     vpxor(dst, xtmp, dst, vector_len);
 3686     break;
 3687   case nlt:
 3688     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3689     vallones(xtmp, vector_len);
 3690     vpxor(dst, xtmp, dst, vector_len);
 3691     break;
 3692   case lt:
 3693     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3694     break;
 3695   case nle:
 3696     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3697     break;
 3698   default:
 3699     assert(false, "Should not reach here");
 3700   }
 3701 }
 3702 
 3703 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3704   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3705   Assembler::vpmovzxbw(dst, src, vector_len);
 3706 }
 3707 
 3708 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3709   assert((src->encoding() < 16),"XMM register should be 0-15");
 3710   Assembler::vpmovmskb(dst, src, vector_len);
 3711 }
 3712 
 3713 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3714   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3715   Assembler::vpmullw(dst, nds, src, vector_len);
 3716 }
 3717 
 3718 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3719   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3720   Assembler::vpmullw(dst, nds, src, vector_len);
 3721 }
 3722 
 3723 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3724   assert((UseAVX > 0), "AVX support is needed");
 3725   assert(rscratch != noreg || always_reachable(src), "missing");
 3726 
 3727   if (reachable(src)) {
 3728     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3729   } else {
 3730     lea(rscratch, src);
 3731     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3732   }
 3733 }
 3734 
 3735 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3736   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3737   Assembler::vpsubb(dst, nds, src, vector_len);
 3738 }
 3739 
 3740 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3741   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3742   Assembler::vpsubb(dst, nds, src, vector_len);
 3743 }
 3744 
 3745 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3746   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3747   Assembler::vpsubw(dst, nds, src, vector_len);
 3748 }
 3749 
 3750 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3751   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3752   Assembler::vpsubw(dst, nds, src, vector_len);
 3753 }
 3754 
 3755 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3756   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3757   Assembler::vpsraw(dst, nds, shift, vector_len);
 3758 }
 3759 
 3760 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3761   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3762   Assembler::vpsraw(dst, nds, shift, vector_len);
 3763 }
 3764 
 3765 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3766   assert(UseAVX > 2,"");
 3767   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3768      vector_len = 2;
 3769   }
 3770   Assembler::evpsraq(dst, nds, shift, vector_len);
 3771 }
 3772 
 3773 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3774   assert(UseAVX > 2,"");
 3775   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3776      vector_len = 2;
 3777   }
 3778   Assembler::evpsraq(dst, nds, shift, vector_len);
 3779 }
 3780 
 3781 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3782   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3783   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3784 }
 3785 
 3786 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3787   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3788   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3789 }
 3790 
 3791 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3792   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3793   Assembler::vpsllw(dst, nds, shift, vector_len);
 3794 }
 3795 
 3796 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3797   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3798   Assembler::vpsllw(dst, nds, shift, vector_len);
 3799 }
 3800 
 3801 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3802   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3803   Assembler::vptest(dst, src);
 3804 }
 3805 
 3806 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3807   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3808   Assembler::punpcklbw(dst, src);
 3809 }
 3810 
 3811 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3812   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3813   Assembler::pshufd(dst, src, mode);
 3814 }
 3815 
 3816 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3817   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3818   Assembler::pshuflw(dst, src, mode);
 3819 }
 3820 
 3821 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3822   assert(rscratch != noreg || always_reachable(src), "missing");
 3823 
 3824   if (reachable(src)) {
 3825     vandpd(dst, nds, as_Address(src), vector_len);
 3826   } else {
 3827     lea(rscratch, src);
 3828     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3829   }
 3830 }
 3831 
 3832 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3833   assert(rscratch != noreg || always_reachable(src), "missing");
 3834 
 3835   if (reachable(src)) {
 3836     vandps(dst, nds, as_Address(src), vector_len);
 3837   } else {
 3838     lea(rscratch, src);
 3839     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3840   }
 3841 }
 3842 
 3843 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3844                             bool merge, int vector_len, Register rscratch) {
 3845   assert(rscratch != noreg || always_reachable(src), "missing");
 3846 
 3847   if (reachable(src)) {
 3848     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3849   } else {
 3850     lea(rscratch, src);
 3851     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3852   }
 3853 }
 3854 
 3855 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3856   assert(rscratch != noreg || always_reachable(src), "missing");
 3857 
 3858   if (reachable(src)) {
 3859     vdivsd(dst, nds, as_Address(src));
 3860   } else {
 3861     lea(rscratch, src);
 3862     vdivsd(dst, nds, Address(rscratch, 0));
 3863   }
 3864 }
 3865 
 3866 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3867   assert(rscratch != noreg || always_reachable(src), "missing");
 3868 
 3869   if (reachable(src)) {
 3870     vdivss(dst, nds, as_Address(src));
 3871   } else {
 3872     lea(rscratch, src);
 3873     vdivss(dst, nds, Address(rscratch, 0));
 3874   }
 3875 }
 3876 
 3877 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3878   assert(rscratch != noreg || always_reachable(src), "missing");
 3879 
 3880   if (reachable(src)) {
 3881     vmulsd(dst, nds, as_Address(src));
 3882   } else {
 3883     lea(rscratch, src);
 3884     vmulsd(dst, nds, Address(rscratch, 0));
 3885   }
 3886 }
 3887 
 3888 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3889   assert(rscratch != noreg || always_reachable(src), "missing");
 3890 
 3891   if (reachable(src)) {
 3892     vmulss(dst, nds, as_Address(src));
 3893   } else {
 3894     lea(rscratch, src);
 3895     vmulss(dst, nds, Address(rscratch, 0));
 3896   }
 3897 }
 3898 
 3899 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3900   assert(rscratch != noreg || always_reachable(src), "missing");
 3901 
 3902   if (reachable(src)) {
 3903     vsubsd(dst, nds, as_Address(src));
 3904   } else {
 3905     lea(rscratch, src);
 3906     vsubsd(dst, nds, Address(rscratch, 0));
 3907   }
 3908 }
 3909 
 3910 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3911   assert(rscratch != noreg || always_reachable(src), "missing");
 3912 
 3913   if (reachable(src)) {
 3914     vsubss(dst, nds, as_Address(src));
 3915   } else {
 3916     lea(rscratch, src);
 3917     vsubss(dst, nds, Address(rscratch, 0));
 3918   }
 3919 }
 3920 
 3921 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3922   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3923   assert(rscratch != noreg || always_reachable(src), "missing");
 3924 
 3925   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3926 }
 3927 
 3928 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3929   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3930   assert(rscratch != noreg || always_reachable(src), "missing");
 3931 
 3932   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3933 }
 3934 
 3935 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3936   assert(rscratch != noreg || always_reachable(src), "missing");
 3937 
 3938   if (reachable(src)) {
 3939     vxorpd(dst, nds, as_Address(src), vector_len);
 3940   } else {
 3941     lea(rscratch, src);
 3942     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 3943   }
 3944 }
 3945 
 3946 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3947   assert(rscratch != noreg || always_reachable(src), "missing");
 3948 
 3949   if (reachable(src)) {
 3950     vxorps(dst, nds, as_Address(src), vector_len);
 3951   } else {
 3952     lea(rscratch, src);
 3953     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 3954   }
 3955 }
 3956 
 3957 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3958   assert(rscratch != noreg || always_reachable(src), "missing");
 3959 
 3960   if (UseAVX > 1 || (vector_len < 1)) {
 3961     if (reachable(src)) {
 3962       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 3963     } else {
 3964       lea(rscratch, src);
 3965       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 3966     }
 3967   } else {
 3968     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 3969   }
 3970 }
 3971 
 3972 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3973   assert(rscratch != noreg || always_reachable(src), "missing");
 3974 
 3975   if (reachable(src)) {
 3976     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 3977   } else {
 3978     lea(rscratch, src);
 3979     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 3980   }
 3981 }
 3982 
 3983 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 3984   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 3985   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 3986   // The inverted mask is sign-extended
 3987   andptr(possibly_non_local, inverted_mask);
 3988 }
 3989 
 3990 void MacroAssembler::resolve_jobject(Register value,
 3991                                      Register thread,
 3992                                      Register tmp) {
 3993   assert_different_registers(value, thread, tmp);
 3994   Label done, tagged, weak_tagged;
 3995   testptr(value, value);
 3996   jcc(Assembler::zero, done);           // Use null as-is.
 3997   testptr(value, JNIHandles::tag_mask); // Test for tag.
 3998   jcc(Assembler::notZero, tagged);
 3999 
 4000   // Resolve local handle
 4001   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4002   verify_oop(value);
 4003   jmp(done);
 4004 
 4005   bind(tagged);
 4006   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4007   jcc(Assembler::notZero, weak_tagged);
 4008 
 4009   // Resolve global handle
 4010   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4011   verify_oop(value);
 4012   jmp(done);
 4013 
 4014   bind(weak_tagged);
 4015   // Resolve jweak.
 4016   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4017                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4018   verify_oop(value);
 4019 
 4020   bind(done);
 4021 }
 4022 
 4023 void MacroAssembler::resolve_global_jobject(Register value,
 4024                                             Register thread,
 4025                                             Register tmp) {
 4026   assert_different_registers(value, thread, tmp);
 4027   Label done;
 4028 
 4029   testptr(value, value);
 4030   jcc(Assembler::zero, done);           // Use null as-is.
 4031 
 4032 #ifdef ASSERT
 4033   {
 4034     Label valid_global_tag;
 4035     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4036     jcc(Assembler::notZero, valid_global_tag);
 4037     stop("non global jobject using resolve_global_jobject");
 4038     bind(valid_global_tag);
 4039   }
 4040 #endif
 4041 
 4042   // Resolve global handle
 4043   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4044   verify_oop(value);
 4045 
 4046   bind(done);
 4047 }
 4048 
 4049 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4050   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4051 }
 4052 
 4053 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4054 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4055   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4056 }
 4057 
 4058 void MacroAssembler::subptr(Register dst, Register src) {
 4059   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4060 }
 4061 
 4062 // C++ bool manipulation
 4063 void MacroAssembler::testbool(Register dst) {
 4064   if(sizeof(bool) == 1)
 4065     testb(dst, 0xff);
 4066   else if(sizeof(bool) == 2) {
 4067     // testw implementation needed for two byte bools
 4068     ShouldNotReachHere();
 4069   } else if(sizeof(bool) == 4)
 4070     testl(dst, dst);
 4071   else
 4072     // unsupported
 4073     ShouldNotReachHere();
 4074 }
 4075 
 4076 void MacroAssembler::testptr(Register dst, Register src) {
 4077   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4078 }
 4079 
 4080 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4081 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4082                                    Register var_size_in_bytes,
 4083                                    int con_size_in_bytes,
 4084                                    Register t1,
 4085                                    Register t2,
 4086                                    Label& slow_case) {
 4087   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4088   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4089 }
 4090 
 4091 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4092   RegSet regs;
 4093 #ifdef _LP64
 4094   regs += RegSet::of(rax, rcx, rdx);
 4095 #ifndef WINDOWS
 4096   regs += RegSet::of(rsi, rdi);
 4097 #endif
 4098   regs += RegSet::range(r8, r11);
 4099 #else
 4100   regs += RegSet::of(rax, rcx, rdx);
 4101 #endif
 4102 #ifdef _LP64
 4103   if (UseAPX) {
 4104     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
 4105   }
 4106 #endif
 4107   return regs;
 4108 }
 4109 
 4110 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4111   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4112 #if defined(WINDOWS) && defined(_LP64)
 4113   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4114   if (num_xmm_registers > 16) {
 4115      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4116   }
 4117   return result;
 4118 #else
 4119   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4120 #endif
 4121 }
 4122 
 4123 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4124 
 4125 #ifndef _LP64
 4126 static bool use_x87_registers() { return UseSSE < 2; }
 4127 #endif
 4128 static bool use_xmm_registers() { return UseSSE >= 1; }
 4129 
 4130 // C1 only ever uses the first double/float of the XMM register.
 4131 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4132 
 4133 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4134   if (UseSSE == 1) {
 4135     masm->movflt(Address(rsp, offset), reg);
 4136   } else {
 4137     masm->movdbl(Address(rsp, offset), reg);
 4138   }
 4139 }
 4140 
 4141 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4142   if (UseSSE == 1) {
 4143     masm->movflt(reg, Address(rsp, offset));
 4144   } else {
 4145     masm->movdbl(reg, Address(rsp, offset));
 4146   }
 4147 }
 4148 
 4149 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4150                                   bool save_fpu, int& gp_area_size,
 4151                                   int& fp_area_size, int& xmm_area_size) {
 4152 
 4153   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4154                          StackAlignmentInBytes);
 4155 #ifdef _LP64
 4156   fp_area_size = 0;
 4157 #else
 4158   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4159 #endif
 4160   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4161 
 4162   return gp_area_size + fp_area_size + xmm_area_size;
 4163 }
 4164 
 4165 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4166   block_comment("push_call_clobbered_registers start");
 4167   // Regular registers
 4168   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4169 
 4170   int gp_area_size;
 4171   int fp_area_size;
 4172   int xmm_area_size;
 4173   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4174                                                gp_area_size, fp_area_size, xmm_area_size);
 4175   subptr(rsp, total_save_size);
 4176 
 4177   push_set(gp_registers_to_push, 0);
 4178 
 4179 #ifndef _LP64
 4180   if (save_fpu && use_x87_registers()) {
 4181     fnsave(Address(rsp, gp_area_size));
 4182     fwait();
 4183   }
 4184 #endif
 4185   if (save_fpu && use_xmm_registers()) {
 4186     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4187   }
 4188 
 4189   block_comment("push_call_clobbered_registers end");
 4190 }
 4191 
 4192 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4193   block_comment("pop_call_clobbered_registers start");
 4194 
 4195   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4196 
 4197   int gp_area_size;
 4198   int fp_area_size;
 4199   int xmm_area_size;
 4200   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4201                                                gp_area_size, fp_area_size, xmm_area_size);
 4202 
 4203   if (restore_fpu && use_xmm_registers()) {
 4204     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4205   }
 4206 #ifndef _LP64
 4207   if (restore_fpu && use_x87_registers()) {
 4208     frstor(Address(rsp, gp_area_size));
 4209   }
 4210 #endif
 4211 
 4212   pop_set(gp_registers_to_pop, 0);
 4213 
 4214   addptr(rsp, total_save_size);
 4215 
 4216   vzeroupper();
 4217 
 4218   block_comment("pop_call_clobbered_registers end");
 4219 }
 4220 
 4221 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4222   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4223   int spill_offset = offset;
 4224 
 4225   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4226     save_xmm_register(this, spill_offset, *it);
 4227     spill_offset += xmm_save_size();
 4228   }
 4229 }
 4230 
 4231 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4232   int restore_size = set.size() * xmm_save_size();
 4233   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4234 
 4235   int restore_offset = offset + restore_size - xmm_save_size();
 4236 
 4237   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4238     restore_xmm_register(this, restore_offset, *it);
 4239     restore_offset -= xmm_save_size();
 4240   }
 4241 }
 4242 
 4243 void MacroAssembler::push_set(RegSet set, int offset) {
 4244   int spill_offset;
 4245   if (offset == -1) {
 4246     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4247     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4248     subptr(rsp, aligned_size);
 4249     spill_offset = 0;
 4250   } else {
 4251     spill_offset = offset;
 4252   }
 4253 
 4254   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4255     movptr(Address(rsp, spill_offset), *it);
 4256     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4257   }
 4258 }
 4259 
 4260 void MacroAssembler::pop_set(RegSet set, int offset) {
 4261 
 4262   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4263   int restore_size = set.size() * gp_reg_size;
 4264   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4265 
 4266   int restore_offset;
 4267   if (offset == -1) {
 4268     restore_offset = restore_size - gp_reg_size;
 4269   } else {
 4270     restore_offset = offset + restore_size - gp_reg_size;
 4271   }
 4272   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4273     movptr(*it, Address(rsp, restore_offset));
 4274     restore_offset -= gp_reg_size;
 4275   }
 4276 
 4277   if (offset == -1) {
 4278     addptr(rsp, aligned_size);
 4279   }
 4280 }
 4281 
 4282 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4283 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4284   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4285   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4286   Label done;
 4287 
 4288   testptr(length_in_bytes, length_in_bytes);
 4289   jcc(Assembler::zero, done);
 4290 
 4291   // initialize topmost word, divide index by 2, check if odd and test if zero
 4292   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4293 #ifdef ASSERT
 4294   {
 4295     Label L;
 4296     testptr(length_in_bytes, BytesPerWord - 1);
 4297     jcc(Assembler::zero, L);
 4298     stop("length must be a multiple of BytesPerWord");
 4299     bind(L);
 4300   }
 4301 #endif
 4302   Register index = length_in_bytes;
 4303   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4304   if (UseIncDec) {
 4305     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4306   } else {
 4307     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4308     shrptr(index, 1);
 4309   }
 4310 #ifndef _LP64
 4311   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4312   {
 4313     Label even;
 4314     // note: if index was a multiple of 8, then it cannot
 4315     //       be 0 now otherwise it must have been 0 before
 4316     //       => if it is even, we don't need to check for 0 again
 4317     jcc(Assembler::carryClear, even);
 4318     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4319     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4320     // index could be 0 now, must check again
 4321     jcc(Assembler::zero, done);
 4322     bind(even);
 4323   }
 4324 #endif // !_LP64
 4325   // initialize remaining object fields: index is a multiple of 2 now
 4326   {
 4327     Label loop;
 4328     bind(loop);
 4329     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4330     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4331     decrement(index);
 4332     jcc(Assembler::notZero, loop);
 4333   }
 4334 
 4335   bind(done);
 4336 }
 4337 
 4338 // Look up the method for a megamorphic invokeinterface call.
 4339 // The target method is determined by <intf_klass, itable_index>.
 4340 // The receiver klass is in recv_klass.
 4341 // On success, the result will be in method_result, and execution falls through.
 4342 // On failure, execution transfers to the given label.
 4343 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4344                                              Register intf_klass,
 4345                                              RegisterOrConstant itable_index,
 4346                                              Register method_result,
 4347                                              Register scan_temp,
 4348                                              Label& L_no_such_interface,
 4349                                              bool return_method) {
 4350   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4351   assert_different_registers(method_result, intf_klass, scan_temp);
 4352   assert(recv_klass != method_result || !return_method,
 4353          "recv_klass can be destroyed when method isn't needed");
 4354 
 4355   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4356          "caller must use same register for non-constant itable index as for method");
 4357 
 4358   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4359   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4360   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4361   int scan_step   = itableOffsetEntry::size() * wordSize;
 4362   int vte_size    = vtableEntry::size_in_bytes();
 4363   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4364   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4365 
 4366   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4367 
 4368   // Could store the aligned, prescaled offset in the klass.
 4369   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4370 
 4371   if (return_method) {
 4372     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4373     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4374     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4375   }
 4376 
 4377   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4378   //   if (scan->interface() == intf) {
 4379   //     result = (klass + scan->offset() + itable_index);
 4380   //   }
 4381   // }
 4382   Label search, found_method;
 4383 
 4384   for (int peel = 1; peel >= 0; peel--) {
 4385     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4386     cmpptr(intf_klass, method_result);
 4387 
 4388     if (peel) {
 4389       jccb(Assembler::equal, found_method);
 4390     } else {
 4391       jccb(Assembler::notEqual, search);
 4392       // (invert the test to fall through to found_method...)
 4393     }
 4394 
 4395     if (!peel)  break;
 4396 
 4397     bind(search);
 4398 
 4399     // Check that the previous entry is non-null.  A null entry means that
 4400     // the receiver class doesn't implement the interface, and wasn't the
 4401     // same as when the caller was compiled.
 4402     testptr(method_result, method_result);
 4403     jcc(Assembler::zero, L_no_such_interface);
 4404     addptr(scan_temp, scan_step);
 4405   }
 4406 
 4407   bind(found_method);
 4408 
 4409   if (return_method) {
 4410     // Got a hit.
 4411     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4412     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4413   }
 4414 }
 4415 
 4416 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4417 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4418 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4419 // The target method is determined by <holder_klass, itable_index>.
 4420 // The receiver klass is in recv_klass.
 4421 // On success, the result will be in method_result, and execution falls through.
 4422 // On failure, execution transfers to the given label.
 4423 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4424                                                   Register holder_klass,
 4425                                                   Register resolved_klass,
 4426                                                   Register method_result,
 4427                                                   Register scan_temp,
 4428                                                   Register temp_reg2,
 4429                                                   Register receiver,
 4430                                                   int itable_index,
 4431                                                   Label& L_no_such_interface) {
 4432   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4433   Register temp_itbl_klass = method_result;
 4434   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4435 
 4436   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4437   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4438   int scan_step = itableOffsetEntry::size() * wordSize;
 4439   int vte_size = vtableEntry::size_in_bytes();
 4440   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4441   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4442   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4443   assert(vte_size == wordSize, "adjust times_vte_scale");
 4444 
 4445   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4446 
 4447   // temp_itbl_klass = recv_klass.itable[0]
 4448   // scan_temp = &recv_klass.itable[0] + step
 4449   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4450   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4451   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4452   xorptr(temp_reg, temp_reg);
 4453 
 4454   // Initial checks:
 4455   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4456   //   - if (itable[0] == 0), no such interface
 4457   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4458   cmpptr(holder_klass, resolved_klass);
 4459   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4460   testptr(temp_itbl_klass, temp_itbl_klass);
 4461   jccb(Assembler::zero, L_no_such_interface);
 4462   cmpptr(holder_klass, temp_itbl_klass);
 4463   jccb(Assembler::equal, L_holder_found);
 4464 
 4465   // Loop: Look for holder_klass record in itable
 4466   //   do {
 4467   //     tmp = itable[index];
 4468   //     index += step;
 4469   //     if (tmp == holder_klass) {
 4470   //       goto L_holder_found; // Found!
 4471   //     }
 4472   //   } while (tmp != 0);
 4473   //   goto L_no_such_interface // Not found.
 4474   Label L_scan_holder;
 4475   bind(L_scan_holder);
 4476     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4477     addptr(scan_temp, scan_step);
 4478     cmpptr(holder_klass, temp_itbl_klass);
 4479     jccb(Assembler::equal, L_holder_found);
 4480     testptr(temp_itbl_klass, temp_itbl_klass);
 4481     jccb(Assembler::notZero, L_scan_holder);
 4482 
 4483   jmpb(L_no_such_interface);
 4484 
 4485   // Loop: Look for resolved_class record in itable
 4486   //   do {
 4487   //     tmp = itable[index];
 4488   //     index += step;
 4489   //     if (tmp == holder_klass) {
 4490   //        // Also check if we have met a holder klass
 4491   //        holder_tmp = itable[index-step-ioffset];
 4492   //     }
 4493   //     if (tmp == resolved_klass) {
 4494   //        goto L_resolved_found;  // Found!
 4495   //     }
 4496   //   } while (tmp != 0);
 4497   //   goto L_no_such_interface // Not found.
 4498   //
 4499   Label L_loop_scan_resolved;
 4500   bind(L_loop_scan_resolved);
 4501     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4502     addptr(scan_temp, scan_step);
 4503     bind(L_loop_scan_resolved_entry);
 4504     cmpptr(holder_klass, temp_itbl_klass);
 4505     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4506     cmpptr(resolved_klass, temp_itbl_klass);
 4507     jccb(Assembler::equal, L_resolved_found);
 4508     testptr(temp_itbl_klass, temp_itbl_klass);
 4509     jccb(Assembler::notZero, L_loop_scan_resolved);
 4510 
 4511   jmpb(L_no_such_interface);
 4512 
 4513   Label L_ready;
 4514 
 4515   // See if we already have a holder klass. If not, go and scan for it.
 4516   bind(L_resolved_found);
 4517   testptr(temp_reg, temp_reg);
 4518   jccb(Assembler::zero, L_scan_holder);
 4519   jmpb(L_ready);
 4520 
 4521   bind(L_holder_found);
 4522   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4523 
 4524   // Finally, temp_reg contains holder_klass vtable offset
 4525   bind(L_ready);
 4526   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4527   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4528     load_klass(scan_temp, receiver, noreg);
 4529     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4530   } else {
 4531     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4532   }
 4533 }
 4534 
 4535 
 4536 // virtual method calling
 4537 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4538                                            RegisterOrConstant vtable_index,
 4539                                            Register method_result) {
 4540   const ByteSize base = Klass::vtable_start_offset();
 4541   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4542   Address vtable_entry_addr(recv_klass,
 4543                             vtable_index, Address::times_ptr,
 4544                             base + vtableEntry::method_offset());
 4545   movptr(method_result, vtable_entry_addr);
 4546 }
 4547 
 4548 
 4549 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4550                            Register super_klass,
 4551                            Register temp_reg,
 4552                            Label& L_success) {
 4553   Label L_failure;
 4554   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4555   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4556   bind(L_failure);
 4557 }
 4558 
 4559 
 4560 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4561                                                    Register super_klass,
 4562                                                    Register temp_reg,
 4563                                                    Label* L_success,
 4564                                                    Label* L_failure,
 4565                                                    Label* L_slow_path,
 4566                                         RegisterOrConstant super_check_offset) {
 4567   assert_different_registers(sub_klass, super_klass, temp_reg);
 4568   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4569   if (super_check_offset.is_register()) {
 4570     assert_different_registers(sub_klass, super_klass,
 4571                                super_check_offset.as_register());
 4572   } else if (must_load_sco) {
 4573     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4574   }
 4575 
 4576   Label L_fallthrough;
 4577   int label_nulls = 0;
 4578   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4579   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4580   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4581   assert(label_nulls <= 1, "at most one null in the batch");
 4582 
 4583   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4584   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4585   Address super_check_offset_addr(super_klass, sco_offset);
 4586 
 4587   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4588   // range of a jccb.  If this routine grows larger, reconsider at
 4589   // least some of these.
 4590 #define local_jcc(assembler_cond, label)                                \
 4591   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4592   else                             jcc( assembler_cond, label) /*omit semi*/
 4593 
 4594   // Hacked jmp, which may only be used just before L_fallthrough.
 4595 #define final_jmp(label)                                                \
 4596   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4597   else                            jmp(label)                /*omit semi*/
 4598 
 4599   // If the pointers are equal, we are done (e.g., String[] elements).
 4600   // This self-check enables sharing of secondary supertype arrays among
 4601   // non-primary types such as array-of-interface.  Otherwise, each such
 4602   // type would need its own customized SSA.
 4603   // We move this check to the front of the fast path because many
 4604   // type checks are in fact trivially successful in this manner,
 4605   // so we get a nicely predicted branch right at the start of the check.
 4606   cmpptr(sub_klass, super_klass);
 4607   local_jcc(Assembler::equal, *L_success);
 4608 
 4609   // Check the supertype display:
 4610   if (must_load_sco) {
 4611     // Positive movl does right thing on LP64.
 4612     movl(temp_reg, super_check_offset_addr);
 4613     super_check_offset = RegisterOrConstant(temp_reg);
 4614   }
 4615   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4616   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4617 
 4618   // This check has worked decisively for primary supers.
 4619   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4620   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4621   // This works in the same check above because of a tricky aliasing
 4622   // between the super_cache and the primary super display elements.
 4623   // (The 'super_check_addr' can address either, as the case requires.)
 4624   // Note that the cache is updated below if it does not help us find
 4625   // what we need immediately.
 4626   // So if it was a primary super, we can just fail immediately.
 4627   // Otherwise, it's the slow path for us (no success at this point).
 4628 
 4629   if (super_check_offset.is_register()) {
 4630     local_jcc(Assembler::equal, *L_success);
 4631     cmpl(super_check_offset.as_register(), sc_offset);
 4632     if (L_failure == &L_fallthrough) {
 4633       local_jcc(Assembler::equal, *L_slow_path);
 4634     } else {
 4635       local_jcc(Assembler::notEqual, *L_failure);
 4636       final_jmp(*L_slow_path);
 4637     }
 4638   } else if (super_check_offset.as_constant() == sc_offset) {
 4639     // Need a slow path; fast failure is impossible.
 4640     if (L_slow_path == &L_fallthrough) {
 4641       local_jcc(Assembler::equal, *L_success);
 4642     } else {
 4643       local_jcc(Assembler::notEqual, *L_slow_path);
 4644       final_jmp(*L_success);
 4645     }
 4646   } else {
 4647     // No slow path; it's a fast decision.
 4648     if (L_failure == &L_fallthrough) {
 4649       local_jcc(Assembler::equal, *L_success);
 4650     } else {
 4651       local_jcc(Assembler::notEqual, *L_failure);
 4652       final_jmp(*L_success);
 4653     }
 4654   }
 4655 
 4656   bind(L_fallthrough);
 4657 
 4658 #undef local_jcc
 4659 #undef final_jmp
 4660 }
 4661 
 4662 
 4663 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4664                                                    Register super_klass,
 4665                                                    Register temp_reg,
 4666                                                    Register temp2_reg,
 4667                                                    Label* L_success,
 4668                                                    Label* L_failure,
 4669                                                    bool set_cond_codes) {
 4670   assert_different_registers(sub_klass, super_klass, temp_reg);
 4671   if (temp2_reg != noreg)
 4672     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4673 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4674 
 4675   Label L_fallthrough;
 4676   int label_nulls = 0;
 4677   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4678   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4679   assert(label_nulls <= 1, "at most one null in the batch");
 4680 
 4681   // a couple of useful fields in sub_klass:
 4682   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4683   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4684   Address secondary_supers_addr(sub_klass, ss_offset);
 4685   Address super_cache_addr(     sub_klass, sc_offset);
 4686 
 4687   // Do a linear scan of the secondary super-klass chain.
 4688   // This code is rarely used, so simplicity is a virtue here.
 4689   // The repne_scan instruction uses fixed registers, which we must spill.
 4690   // Don't worry too much about pre-existing connections with the input regs.
 4691 
 4692   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4693   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4694 
 4695   // Get super_klass value into rax (even if it was in rdi or rcx).
 4696   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4697   if (super_klass != rax) {
 4698     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4699     mov(rax, super_klass);
 4700   }
 4701   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4702   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4703 
 4704 #ifndef PRODUCT
 4705   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4706   ExternalAddress pst_counter_addr((address) pst_counter);
 4707   NOT_LP64(  incrementl(pst_counter_addr) );
 4708   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4709   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4710 #endif //PRODUCT
 4711 
 4712   // We will consult the secondary-super array.
 4713   movptr(rdi, secondary_supers_addr);
 4714   // Load the array length.  (Positive movl does right thing on LP64.)
 4715   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4716   // Skip to start of data.
 4717   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4718 
 4719   // Scan RCX words at [RDI] for an occurrence of RAX.
 4720   // Set NZ/Z based on last compare.
 4721   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4722   // not change flags (only scas instruction which is repeated sets flags).
 4723   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4724 
 4725     testptr(rax,rax); // Set Z = 0
 4726     repne_scan();
 4727 
 4728   // Unspill the temp. registers:
 4729   if (pushed_rdi)  pop(rdi);
 4730   if (pushed_rcx)  pop(rcx);
 4731   if (pushed_rax)  pop(rax);
 4732 
 4733   if (set_cond_codes) {
 4734     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4735     assert(!pushed_rdi, "rdi must be left non-null");
 4736     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4737   }
 4738 
 4739   if (L_failure == &L_fallthrough)
 4740         jccb(Assembler::notEqual, *L_failure);
 4741   else  jcc(Assembler::notEqual, *L_failure);
 4742 
 4743   // Success.  Cache the super we found and proceed in triumph.
 4744   movptr(super_cache_addr, super_klass);
 4745 
 4746   if (L_success != &L_fallthrough) {
 4747     jmp(*L_success);
 4748   }
 4749 
 4750 #undef IS_A_TEMP
 4751 
 4752   bind(L_fallthrough);
 4753 }
 4754 
 4755 #ifdef _LP64
 4756 
 4757 // population_count variant for running without the POPCNT
 4758 // instruction, which was introduced with SSE4.2 in 2008.
 4759 void MacroAssembler::population_count(Register dst, Register src,
 4760                                       Register scratch1, Register scratch2) {
 4761   assert_different_registers(src, scratch1, scratch2);
 4762   if (UsePopCountInstruction) {
 4763     Assembler::popcntq(dst, src);
 4764   } else {
 4765     assert_different_registers(src, scratch1, scratch2);
 4766     assert_different_registers(dst, scratch1, scratch2);
 4767     Label loop, done;
 4768 
 4769     mov(scratch1, src);
 4770     // dst = 0;
 4771     // while(scratch1 != 0) {
 4772     //   dst++;
 4773     //   scratch1 &= (scratch1 - 1);
 4774     // }
 4775     xorl(dst, dst);
 4776     testq(scratch1, scratch1);
 4777     jccb(Assembler::equal, done);
 4778     {
 4779       bind(loop);
 4780       incq(dst);
 4781       movq(scratch2, scratch1);
 4782       decq(scratch2);
 4783       andq(scratch1, scratch2);
 4784       jccb(Assembler::notEqual, loop);
 4785     }
 4786     bind(done);
 4787   }
 4788 }
 4789 
 4790 // Ensure that the inline code and the stub are using the same registers.
 4791 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
 4792 do {                                                                 \
 4793   assert(r_super_klass  == rax, "mismatch");                         \
 4794   assert(r_array_base   == rbx, "mismatch");                         \
 4795   assert(r_array_length == rcx, "mismatch");                         \
 4796   assert(r_array_index  == rdx, "mismatch");                         \
 4797   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
 4798   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
 4799   assert(result         == rdi || result      == noreg, "mismatch"); \
 4800 } while(0)
 4801 
 4802 void MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass,
 4803                                                    Register r_super_klass,
 4804                                                    Register temp1,
 4805                                                    Register temp2,
 4806                                                    Register temp3,
 4807                                                    Register temp4,
 4808                                                    Register result,
 4809                                                    u1 super_klass_slot) {
 4810   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 4811 
 4812   Label L_fallthrough, L_success, L_failure;
 4813 
 4814   BLOCK_COMMENT("lookup_secondary_supers_table {");
 4815 
 4816   const Register
 4817     r_array_index  = temp1,
 4818     r_array_length = temp2,
 4819     r_array_base   = temp3,
 4820     r_bitmap       = temp4;
 4821 
 4822   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 4823 
 4824   xorq(result, result); // = 0
 4825 
 4826   movq(r_bitmap, Address(r_sub_klass, Klass::bitmap_offset()));
 4827   movq(r_array_index, r_bitmap);
 4828 
 4829   // First check the bitmap to see if super_klass might be present. If
 4830   // the bit is zero, we are certain that super_klass is not one of
 4831   // the secondary supers.
 4832   u1 bit = super_klass_slot;
 4833   {
 4834     // NB: If the count in a x86 shift instruction is 0, the flags are
 4835     // not affected, so we do a testq instead.
 4836     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
 4837     if (shift_count != 0) {
 4838       salq(r_array_index, shift_count);
 4839     } else {
 4840       testq(r_array_index, r_array_index);
 4841     }
 4842   }
 4843   // We test the MSB of r_array_index, i.e. its sign bit
 4844   jcc(Assembler::positive, L_failure);
 4845 
 4846   // Get the first array index that can contain super_klass into r_array_index.
 4847   if (bit != 0) {
 4848     population_count(r_array_index, r_array_index, temp2, temp3);
 4849   } else {
 4850     movl(r_array_index, 1);
 4851   }
 4852   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 4853 
 4854   // We will consult the secondary-super array.
 4855   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 4856 
 4857   // We're asserting that the first word in an Array<Klass*> is the
 4858   // length, and the second word is the first word of the data. If
 4859   // that ever changes, r_array_base will have to be adjusted here.
 4860   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 4861   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 4862 
 4863   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 4864   jccb(Assembler::equal, L_success);
 4865 
 4866   // Is there another entry to check? Consult the bitmap.
 4867   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
 4868   jccb(Assembler::carryClear, L_failure);
 4869 
 4870   // Linear probe. Rotate the bitmap so that the next bit to test is
 4871   // in Bit 1.
 4872   if (bit != 0) {
 4873     rorq(r_bitmap, bit);
 4874   }
 4875 
 4876   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 4877   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 4878   // Kills: r_array_length.
 4879   // Returns: result.
 4880   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
 4881   // Result (0/1) is in rdi
 4882   jmpb(L_fallthrough);
 4883 
 4884   bind(L_failure);
 4885   incq(result); // 0 => 1
 4886 
 4887   bind(L_success);
 4888   // result = 0;
 4889 
 4890   bind(L_fallthrough);
 4891   BLOCK_COMMENT("} lookup_secondary_supers_table");
 4892 
 4893   if (VerifySecondarySupers) {
 4894     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 4895                                   temp1, temp2, temp3);
 4896   }
 4897 }
 4898 
 4899 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
 4900                                  Label* L_success, Label* L_failure) {
 4901   Label L_loop, L_fallthrough;
 4902   {
 4903     int label_nulls = 0;
 4904     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
 4905     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
 4906     assert(label_nulls <= 1, "at most one null in the batch");
 4907   }
 4908   bind(L_loop);
 4909   cmpq(value, Address(addr, count, Address::times_8));
 4910   jcc(Assembler::equal, *L_success);
 4911   addl(count, 1);
 4912   cmpl(count, limit);
 4913   jcc(Assembler::less, L_loop);
 4914 
 4915   if (&L_fallthrough != L_failure) {
 4916     jmp(*L_failure);
 4917   }
 4918   bind(L_fallthrough);
 4919 }
 4920 
 4921 // Called by code generated by check_klass_subtype_slow_path
 4922 // above. This is called when there is a collision in the hashed
 4923 // lookup in the secondary supers array.
 4924 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
 4925                                                              Register r_array_base,
 4926                                                              Register r_array_index,
 4927                                                              Register r_bitmap,
 4928                                                              Register temp1,
 4929                                                              Register temp2,
 4930                                                              Label* L_success,
 4931                                                              Label* L_failure) {
 4932   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
 4933 
 4934   const Register
 4935     r_array_length = temp1,
 4936     r_sub_klass    = noreg,
 4937     result         = noreg;
 4938 
 4939   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 4940 
 4941   Label L_fallthrough;
 4942   int label_nulls = 0;
 4943   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4944   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4945   assert(label_nulls <= 1, "at most one null in the batch");
 4946 
 4947   // Load the array length.
 4948   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 4949   // And adjust the array base to point to the data.
 4950   // NB! Effectively increments current slot index by 1.
 4951   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
 4952   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 4953 
 4954   // Linear probe
 4955   Label L_huge;
 4956 
 4957   // The bitmap is full to bursting.
 4958   // Implicit invariant: BITMAP_FULL implies (length > 0)
 4959   assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), "");
 4960   cmpq(r_bitmap, (int32_t)-1); // sign-extends immediate to 64-bit value
 4961   jcc(Assembler::equal, L_huge);
 4962 
 4963   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
 4964   // current slot (at secondary_supers[r_array_index]) has not yet
 4965   // been inspected, and r_array_index may be out of bounds if we
 4966   // wrapped around the end of the array.
 4967 
 4968   { // This is conventional linear probing, but instead of terminating
 4969     // when a null entry is found in the table, we maintain a bitmap
 4970     // in which a 0 indicates missing entries.
 4971     // The check above guarantees there are 0s in the bitmap, so the loop
 4972     // eventually terminates.
 4973 
 4974     xorl(temp2, temp2); // = 0;
 4975 
 4976     Label L_again;
 4977     bind(L_again);
 4978 
 4979     // Check for array wraparound.
 4980     cmpl(r_array_index, r_array_length);
 4981     cmovl(Assembler::greaterEqual, r_array_index, temp2);
 4982 
 4983     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 4984     jcc(Assembler::equal, *L_success);
 4985 
 4986     // If the next bit in bitmap is zero, we're done.
 4987     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
 4988     jcc(Assembler::carryClear, *L_failure);
 4989 
 4990     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
 4991     addl(r_array_index, 1);
 4992 
 4993     jmp(L_again);
 4994   }
 4995 
 4996   { // Degenerate case: more than 64 secondary supers.
 4997     // FIXME: We could do something smarter here, maybe a vectorized
 4998     // comparison or a binary search, but is that worth any added
 4999     // complexity?
 5000     bind(L_huge);
 5001     xorl(r_array_index, r_array_index); // = 0
 5002     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
 5003                 L_success,
 5004                 (&L_fallthrough != L_failure ? L_failure : nullptr));
 5005 
 5006     bind(L_fallthrough);
 5007   }
 5008 }
 5009 
 5010 struct VerifyHelperArguments {
 5011   Klass* _super;
 5012   Klass* _sub;
 5013   intptr_t _linear_result;
 5014   intptr_t _table_result;
 5015 };
 5016 
 5017 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
 5018   Klass::on_secondary_supers_verification_failure(args->_super,
 5019                                                   args->_sub,
 5020                                                   args->_linear_result,
 5021                                                   args->_table_result,
 5022                                                   msg);
 5023 }
 5024 
 5025 // Make sure that the hashed lookup and a linear scan agree.
 5026 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
 5027                                                    Register r_super_klass,
 5028                                                    Register result,
 5029                                                    Register temp1,
 5030                                                    Register temp2,
 5031                                                    Register temp3) {
 5032   const Register
 5033       r_array_index  = temp1,
 5034       r_array_length = temp2,
 5035       r_array_base   = temp3,
 5036       r_bitmap       = noreg;
 5037 
 5038   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 5039 
 5040   BLOCK_COMMENT("verify_secondary_supers_table {");
 5041 
 5042   Label L_success, L_failure, L_check, L_done;
 5043 
 5044   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5045   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5046   // And adjust the array base to point to the data.
 5047   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5048 
 5049   testl(r_array_length, r_array_length); // array_length == 0?
 5050   jcc(Assembler::zero, L_failure);
 5051 
 5052   movl(r_array_index, 0);
 5053   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
 5054   // fall through to L_failure
 5055 
 5056   const Register linear_result = r_array_index; // reuse temp1
 5057 
 5058   bind(L_failure); // not present
 5059   movl(linear_result, 1);
 5060   jmp(L_check);
 5061 
 5062   bind(L_success); // present
 5063   movl(linear_result, 0);
 5064 
 5065   bind(L_check);
 5066   cmpl(linear_result, result);
 5067   jcc(Assembler::equal, L_done);
 5068 
 5069   { // To avoid calling convention issues, build a record on the stack
 5070     // and pass the pointer to that instead.
 5071     push(result);
 5072     push(linear_result);
 5073     push(r_sub_klass);
 5074     push(r_super_klass);
 5075     movptr(c_rarg1, rsp);
 5076     movptr(c_rarg0, (uintptr_t) "mismatch");
 5077     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
 5078     should_not_reach_here();
 5079   }
 5080   bind(L_done);
 5081 
 5082   BLOCK_COMMENT("} verify_secondary_supers_table");
 5083 }
 5084 
 5085 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
 5086 
 5087 #endif // LP64
 5088 
 5089 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 5090   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5091 
 5092   Label L_fallthrough;
 5093   if (L_fast_path == nullptr) {
 5094     L_fast_path = &L_fallthrough;
 5095   } else if (L_slow_path == nullptr) {
 5096     L_slow_path = &L_fallthrough;
 5097   }
 5098 
 5099   // Fast path check: class is fully initialized
 5100   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5101   jcc(Assembler::equal, *L_fast_path);
 5102 
 5103   // Fast path check: current thread is initializer thread
 5104   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 5105   if (L_slow_path == &L_fallthrough) {
 5106     jcc(Assembler::equal, *L_fast_path);
 5107     bind(*L_slow_path);
 5108   } else if (L_fast_path == &L_fallthrough) {
 5109     jcc(Assembler::notEqual, *L_slow_path);
 5110     bind(*L_fast_path);
 5111   } else {
 5112     Unimplemented();
 5113   }
 5114 }
 5115 
 5116 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5117   if (VM_Version::supports_cmov()) {
 5118     cmovl(cc, dst, src);
 5119   } else {
 5120     Label L;
 5121     jccb(negate_condition(cc), L);
 5122     movl(dst, src);
 5123     bind(L);
 5124   }
 5125 }
 5126 
 5127 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5128   if (VM_Version::supports_cmov()) {
 5129     cmovl(cc, dst, src);
 5130   } else {
 5131     Label L;
 5132     jccb(negate_condition(cc), L);
 5133     movl(dst, src);
 5134     bind(L);
 5135   }
 5136 }
 5137 
 5138 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5139   if (!VerifyOops) return;
 5140 
 5141   BLOCK_COMMENT("verify_oop {");
 5142 #ifdef _LP64
 5143   push(rscratch1);
 5144 #endif
 5145   push(rax);                          // save rax
 5146   push(reg);                          // pass register argument
 5147 
 5148   // Pass register number to verify_oop_subroutine
 5149   const char* b = nullptr;
 5150   {
 5151     ResourceMark rm;
 5152     stringStream ss;
 5153     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5154     b = code_string(ss.as_string());
 5155   }
 5156   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5157   pushptr(buffer.addr(), rscratch1);
 5158 
 5159   // call indirectly to solve generation ordering problem
 5160   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5161   call(rax);
 5162   // Caller pops the arguments (oop, message) and restores rax, r10
 5163   BLOCK_COMMENT("} verify_oop");
 5164 }
 5165 
 5166 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5167   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5168     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5169     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5170     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5171   } else if (VM_Version::supports_avx()) {
 5172     vpcmpeqd(dst, dst, dst, vector_len);
 5173   } else {
 5174     assert(VM_Version::supports_sse2(), "");
 5175     pcmpeqd(dst, dst);
 5176   }
 5177 }
 5178 
 5179 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5180                                          int extra_slot_offset) {
 5181   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5182   int stackElementSize = Interpreter::stackElementSize;
 5183   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5184 #ifdef ASSERT
 5185   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5186   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5187 #endif
 5188   Register             scale_reg    = noreg;
 5189   Address::ScaleFactor scale_factor = Address::no_scale;
 5190   if (arg_slot.is_constant()) {
 5191     offset += arg_slot.as_constant() * stackElementSize;
 5192   } else {
 5193     scale_reg    = arg_slot.as_register();
 5194     scale_factor = Address::times(stackElementSize);
 5195   }
 5196   offset += wordSize;           // return PC is on stack
 5197   return Address(rsp, scale_reg, scale_factor, offset);
 5198 }
 5199 
 5200 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5201   if (!VerifyOops) return;
 5202 
 5203 #ifdef _LP64
 5204   push(rscratch1);
 5205 #endif
 5206   push(rax); // save rax,
 5207   // addr may contain rsp so we will have to adjust it based on the push
 5208   // we just did (and on 64 bit we do two pushes)
 5209   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5210   // stores rax into addr which is backwards of what was intended.
 5211   if (addr.uses(rsp)) {
 5212     lea(rax, addr);
 5213     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5214   } else {
 5215     pushptr(addr);
 5216   }
 5217 
 5218   // Pass register number to verify_oop_subroutine
 5219   const char* b = nullptr;
 5220   {
 5221     ResourceMark rm;
 5222     stringStream ss;
 5223     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5224     b = code_string(ss.as_string());
 5225   }
 5226   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5227   pushptr(buffer.addr(), rscratch1);
 5228 
 5229   // call indirectly to solve generation ordering problem
 5230   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5231   call(rax);
 5232   // Caller pops the arguments (addr, message) and restores rax, r10.
 5233 }
 5234 
 5235 void MacroAssembler::verify_tlab() {
 5236 #ifdef ASSERT
 5237   if (UseTLAB && VerifyOops) {
 5238     Label next, ok;
 5239     Register t1 = rsi;
 5240     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5241 
 5242     push(t1);
 5243     NOT_LP64(push(thread_reg));
 5244     NOT_LP64(get_thread(thread_reg));
 5245 
 5246     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5247     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5248     jcc(Assembler::aboveEqual, next);
 5249     STOP("assert(top >= start)");
 5250     should_not_reach_here();
 5251 
 5252     bind(next);
 5253     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5254     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5255     jcc(Assembler::aboveEqual, ok);
 5256     STOP("assert(top <= end)");
 5257     should_not_reach_here();
 5258 
 5259     bind(ok);
 5260     NOT_LP64(pop(thread_reg));
 5261     pop(t1);
 5262   }
 5263 #endif
 5264 }
 5265 
 5266 class ControlWord {
 5267  public:
 5268   int32_t _value;
 5269 
 5270   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5271   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5272   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5273   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5274   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5275   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5276   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5277   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5278 
 5279   void print() const {
 5280     // rounding control
 5281     const char* rc;
 5282     switch (rounding_control()) {
 5283       case 0: rc = "round near"; break;
 5284       case 1: rc = "round down"; break;
 5285       case 2: rc = "round up  "; break;
 5286       case 3: rc = "chop      "; break;
 5287       default:
 5288         rc = nullptr; // silence compiler warnings
 5289         fatal("Unknown rounding control: %d", rounding_control());
 5290     };
 5291     // precision control
 5292     const char* pc;
 5293     switch (precision_control()) {
 5294       case 0: pc = "24 bits "; break;
 5295       case 1: pc = "reserved"; break;
 5296       case 2: pc = "53 bits "; break;
 5297       case 3: pc = "64 bits "; break;
 5298       default:
 5299         pc = nullptr; // silence compiler warnings
 5300         fatal("Unknown precision control: %d", precision_control());
 5301     };
 5302     // flags
 5303     char f[9];
 5304     f[0] = ' ';
 5305     f[1] = ' ';
 5306     f[2] = (precision   ()) ? 'P' : 'p';
 5307     f[3] = (underflow   ()) ? 'U' : 'u';
 5308     f[4] = (overflow    ()) ? 'O' : 'o';
 5309     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5310     f[6] = (denormalized()) ? 'D' : 'd';
 5311     f[7] = (invalid     ()) ? 'I' : 'i';
 5312     f[8] = '\x0';
 5313     // output
 5314     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5315   }
 5316 
 5317 };
 5318 
 5319 class StatusWord {
 5320  public:
 5321   int32_t _value;
 5322 
 5323   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5324   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5325   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5326   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5327   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5328   int  top() const                     { return  (_value >> 11) & 7      ; }
 5329   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5330   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5331   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5332   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5333   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5334   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5335   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5336   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5337 
 5338   void print() const {
 5339     // condition codes
 5340     char c[5];
 5341     c[0] = (C3()) ? '3' : '-';
 5342     c[1] = (C2()) ? '2' : '-';
 5343     c[2] = (C1()) ? '1' : '-';
 5344     c[3] = (C0()) ? '0' : '-';
 5345     c[4] = '\x0';
 5346     // flags
 5347     char f[9];
 5348     f[0] = (error_status()) ? 'E' : '-';
 5349     f[1] = (stack_fault ()) ? 'S' : '-';
 5350     f[2] = (precision   ()) ? 'P' : '-';
 5351     f[3] = (underflow   ()) ? 'U' : '-';
 5352     f[4] = (overflow    ()) ? 'O' : '-';
 5353     f[5] = (zero_divide ()) ? 'Z' : '-';
 5354     f[6] = (denormalized()) ? 'D' : '-';
 5355     f[7] = (invalid     ()) ? 'I' : '-';
 5356     f[8] = '\x0';
 5357     // output
 5358     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5359   }
 5360 
 5361 };
 5362 
 5363 class TagWord {
 5364  public:
 5365   int32_t _value;
 5366 
 5367   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5368 
 5369   void print() const {
 5370     printf("%04x", _value & 0xFFFF);
 5371   }
 5372 
 5373 };
 5374 
 5375 class FPU_Register {
 5376  public:
 5377   int32_t _m0;
 5378   int32_t _m1;
 5379   int16_t _ex;
 5380 
 5381   bool is_indefinite() const           {
 5382     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5383   }
 5384 
 5385   void print() const {
 5386     char  sign = (_ex < 0) ? '-' : '+';
 5387     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5388     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5389   };
 5390 
 5391 };
 5392 
 5393 class FPU_State {
 5394  public:
 5395   enum {
 5396     register_size       = 10,
 5397     number_of_registers =  8,
 5398     register_mask       =  7
 5399   };
 5400 
 5401   ControlWord  _control_word;
 5402   StatusWord   _status_word;
 5403   TagWord      _tag_word;
 5404   int32_t      _error_offset;
 5405   int32_t      _error_selector;
 5406   int32_t      _data_offset;
 5407   int32_t      _data_selector;
 5408   int8_t       _register[register_size * number_of_registers];
 5409 
 5410   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5411   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5412 
 5413   const char* tag_as_string(int tag) const {
 5414     switch (tag) {
 5415       case 0: return "valid";
 5416       case 1: return "zero";
 5417       case 2: return "special";
 5418       case 3: return "empty";
 5419     }
 5420     ShouldNotReachHere();
 5421     return nullptr;
 5422   }
 5423 
 5424   void print() const {
 5425     // print computation registers
 5426     { int t = _status_word.top();
 5427       for (int i = 0; i < number_of_registers; i++) {
 5428         int j = (i - t) & register_mask;
 5429         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5430         st(j)->print();
 5431         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5432       }
 5433     }
 5434     printf("\n");
 5435     // print control registers
 5436     printf("ctrl = "); _control_word.print(); printf("\n");
 5437     printf("stat = "); _status_word .print(); printf("\n");
 5438     printf("tags = "); _tag_word    .print(); printf("\n");
 5439   }
 5440 
 5441 };
 5442 
 5443 class Flag_Register {
 5444  public:
 5445   int32_t _value;
 5446 
 5447   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5448   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5449   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5450   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5451   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5452   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5453   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5454 
 5455   void print() const {
 5456     // flags
 5457     char f[8];
 5458     f[0] = (overflow       ()) ? 'O' : '-';
 5459     f[1] = (direction      ()) ? 'D' : '-';
 5460     f[2] = (sign           ()) ? 'S' : '-';
 5461     f[3] = (zero           ()) ? 'Z' : '-';
 5462     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5463     f[5] = (parity         ()) ? 'P' : '-';
 5464     f[6] = (carry          ()) ? 'C' : '-';
 5465     f[7] = '\x0';
 5466     // output
 5467     printf("%08x  flags = %s", _value, f);
 5468   }
 5469 
 5470 };
 5471 
 5472 class IU_Register {
 5473  public:
 5474   int32_t _value;
 5475 
 5476   void print() const {
 5477     printf("%08x  %11d", _value, _value);
 5478   }
 5479 
 5480 };
 5481 
 5482 class IU_State {
 5483  public:
 5484   Flag_Register _eflags;
 5485   IU_Register   _rdi;
 5486   IU_Register   _rsi;
 5487   IU_Register   _rbp;
 5488   IU_Register   _rsp;
 5489   IU_Register   _rbx;
 5490   IU_Register   _rdx;
 5491   IU_Register   _rcx;
 5492   IU_Register   _rax;
 5493 
 5494   void print() const {
 5495     // computation registers
 5496     printf("rax,  = "); _rax.print(); printf("\n");
 5497     printf("rbx,  = "); _rbx.print(); printf("\n");
 5498     printf("rcx  = "); _rcx.print(); printf("\n");
 5499     printf("rdx  = "); _rdx.print(); printf("\n");
 5500     printf("rdi  = "); _rdi.print(); printf("\n");
 5501     printf("rsi  = "); _rsi.print(); printf("\n");
 5502     printf("rbp,  = "); _rbp.print(); printf("\n");
 5503     printf("rsp  = "); _rsp.print(); printf("\n");
 5504     printf("\n");
 5505     // control registers
 5506     printf("flgs = "); _eflags.print(); printf("\n");
 5507   }
 5508 };
 5509 
 5510 
 5511 class CPU_State {
 5512  public:
 5513   FPU_State _fpu_state;
 5514   IU_State  _iu_state;
 5515 
 5516   void print() const {
 5517     printf("--------------------------------------------------\n");
 5518     _iu_state .print();
 5519     printf("\n");
 5520     _fpu_state.print();
 5521     printf("--------------------------------------------------\n");
 5522   }
 5523 
 5524 };
 5525 
 5526 
 5527 static void _print_CPU_state(CPU_State* state) {
 5528   state->print();
 5529 };
 5530 
 5531 
 5532 void MacroAssembler::print_CPU_state() {
 5533   push_CPU_state();
 5534   push(rsp);                // pass CPU state
 5535   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5536   addptr(rsp, wordSize);       // discard argument
 5537   pop_CPU_state();
 5538 }
 5539 
 5540 
 5541 #ifndef _LP64
 5542 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5543   static int counter = 0;
 5544   FPU_State* fs = &state->_fpu_state;
 5545   counter++;
 5546   // For leaf calls, only verify that the top few elements remain empty.
 5547   // We only need 1 empty at the top for C2 code.
 5548   if( stack_depth < 0 ) {
 5549     if( fs->tag_for_st(7) != 3 ) {
 5550       printf("FPR7 not empty\n");
 5551       state->print();
 5552       assert(false, "error");
 5553       return false;
 5554     }
 5555     return true;                // All other stack states do not matter
 5556   }
 5557 
 5558   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5559          "bad FPU control word");
 5560 
 5561   // compute stack depth
 5562   int i = 0;
 5563   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5564   int d = i;
 5565   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5566   // verify findings
 5567   if (i != FPU_State::number_of_registers) {
 5568     // stack not contiguous
 5569     printf("%s: stack not contiguous at ST%d\n", s, i);
 5570     state->print();
 5571     assert(false, "error");
 5572     return false;
 5573   }
 5574   // check if computed stack depth corresponds to expected stack depth
 5575   if (stack_depth < 0) {
 5576     // expected stack depth is -stack_depth or less
 5577     if (d > -stack_depth) {
 5578       // too many elements on the stack
 5579       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5580       state->print();
 5581       assert(false, "error");
 5582       return false;
 5583     }
 5584   } else {
 5585     // expected stack depth is stack_depth
 5586     if (d != stack_depth) {
 5587       // wrong stack depth
 5588       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5589       state->print();
 5590       assert(false, "error");
 5591       return false;
 5592     }
 5593   }
 5594   // everything is cool
 5595   return true;
 5596 }
 5597 
 5598 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5599   if (!VerifyFPU) return;
 5600   push_CPU_state();
 5601   push(rsp);                // pass CPU state
 5602   ExternalAddress msg((address) s);
 5603   // pass message string s
 5604   pushptr(msg.addr(), noreg);
 5605   push(stack_depth);        // pass stack depth
 5606   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5607   addptr(rsp, 3 * wordSize);   // discard arguments
 5608   // check for error
 5609   { Label L;
 5610     testl(rax, rax);
 5611     jcc(Assembler::notZero, L);
 5612     int3();                  // break if error condition
 5613     bind(L);
 5614   }
 5615   pop_CPU_state();
 5616 }
 5617 #endif // _LP64
 5618 
 5619 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5620   // Either restore the MXCSR register after returning from the JNI Call
 5621   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5622   if (VM_Version::supports_sse()) {
 5623     if (RestoreMXCSROnJNICalls) {
 5624       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5625     } else if (CheckJNICalls) {
 5626       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5627     }
 5628   }
 5629   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5630   vzeroupper();
 5631 
 5632 #ifndef _LP64
 5633   // Either restore the x87 floating pointer control word after returning
 5634   // from the JNI call or verify that it wasn't changed.
 5635   if (CheckJNICalls) {
 5636     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5637   }
 5638 #endif // _LP64
 5639 }
 5640 
 5641 // ((OopHandle)result).resolve();
 5642 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5643   assert_different_registers(result, tmp);
 5644 
 5645   // Only 64 bit platforms support GCs that require a tmp register
 5646   // Only IN_HEAP loads require a thread_tmp register
 5647   // OopHandle::resolve is an indirection like jobject.
 5648   access_load_at(T_OBJECT, IN_NATIVE,
 5649                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5650 }
 5651 
 5652 // ((WeakHandle)result).resolve();
 5653 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5654   assert_different_registers(rresult, rtmp);
 5655   Label resolved;
 5656 
 5657   // A null weak handle resolves to null.
 5658   cmpptr(rresult, 0);
 5659   jcc(Assembler::equal, resolved);
 5660 
 5661   // Only 64 bit platforms support GCs that require a tmp register
 5662   // Only IN_HEAP loads require a thread_tmp register
 5663   // WeakHandle::resolve is an indirection like jweak.
 5664   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5665                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5666   bind(resolved);
 5667 }
 5668 
 5669 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5670   // get mirror
 5671   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5672   load_method_holder(mirror, method);
 5673   movptr(mirror, Address(mirror, mirror_offset));
 5674   resolve_oop_handle(mirror, tmp);
 5675 }
 5676 
 5677 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5678   load_method_holder(rresult, rmethod);
 5679   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5680 }
 5681 
 5682 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5683   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5684   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5685   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5686 }
 5687 
 5688 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5689   assert_different_registers(src, tmp);
 5690   assert_different_registers(dst, tmp);
 5691 #ifdef _LP64
 5692   if (UseCompressedClassPointers) {
 5693     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5694     decode_klass_not_null(dst, tmp);
 5695   } else
 5696 #endif
 5697     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5698 }
 5699 
 5700 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5701   assert_different_registers(src, tmp);
 5702   assert_different_registers(dst, tmp);
 5703 #ifdef _LP64
 5704   if (UseCompressedClassPointers) {
 5705     encode_klass_not_null(src, tmp);
 5706     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5707   } else
 5708 #endif
 5709     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5710 }
 5711 
 5712 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5713                                     Register tmp1, Register thread_tmp) {
 5714   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5715   decorators = AccessInternal::decorator_fixup(decorators, type);
 5716   bool as_raw = (decorators & AS_RAW) != 0;
 5717   if (as_raw) {
 5718     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5719   } else {
 5720     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5721   }
 5722 }
 5723 
 5724 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5725                                      Register tmp1, Register tmp2, Register tmp3) {
 5726   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5727   decorators = AccessInternal::decorator_fixup(decorators, type);
 5728   bool as_raw = (decorators & AS_RAW) != 0;
 5729   if (as_raw) {
 5730     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5731   } else {
 5732     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5733   }
 5734 }
 5735 
 5736 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5737                                    Register thread_tmp, DecoratorSet decorators) {
 5738   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5739 }
 5740 
 5741 // Doesn't do verification, generates fixed size code
 5742 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5743                                             Register thread_tmp, DecoratorSet decorators) {
 5744   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5745 }
 5746 
 5747 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5748                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5749   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5750 }
 5751 
 5752 // Used for storing nulls.
 5753 void MacroAssembler::store_heap_oop_null(Address dst) {
 5754   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5755 }
 5756 
 5757 #ifdef _LP64
 5758 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 5759   if (UseCompressedClassPointers) {
 5760     // Store to klass gap in destination
 5761     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 5762   }
 5763 }
 5764 
 5765 #ifdef ASSERT
 5766 void MacroAssembler::verify_heapbase(const char* msg) {
 5767   assert (UseCompressedOops, "should be compressed");
 5768   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5769   if (CheckCompressedOops) {
 5770     Label ok;
 5771     ExternalAddress src2(CompressedOops::ptrs_base_addr());
 5772     const bool is_src2_reachable = reachable(src2);
 5773     if (!is_src2_reachable) {
 5774       push(rscratch1);  // cmpptr trashes rscratch1
 5775     }
 5776     cmpptr(r12_heapbase, src2, rscratch1);
 5777     jcc(Assembler::equal, ok);
 5778     STOP(msg);
 5779     bind(ok);
 5780     if (!is_src2_reachable) {
 5781       pop(rscratch1);
 5782     }
 5783   }
 5784 }
 5785 #endif
 5786 
 5787 // Algorithm must match oop.inline.hpp encode_heap_oop.
 5788 void MacroAssembler::encode_heap_oop(Register r) {
 5789 #ifdef ASSERT
 5790   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 5791 #endif
 5792   verify_oop_msg(r, "broken oop in encode_heap_oop");
 5793   if (CompressedOops::base() == nullptr) {
 5794     if (CompressedOops::shift() != 0) {
 5795       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5796       shrq(r, LogMinObjAlignmentInBytes);
 5797     }
 5798     return;
 5799   }
 5800   testq(r, r);
 5801   cmovq(Assembler::equal, r, r12_heapbase);
 5802   subq(r, r12_heapbase);
 5803   shrq(r, LogMinObjAlignmentInBytes);
 5804 }
 5805 
 5806 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 5807 #ifdef ASSERT
 5808   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 5809   if (CheckCompressedOops) {
 5810     Label ok;
 5811     testq(r, r);
 5812     jcc(Assembler::notEqual, ok);
 5813     STOP("null oop passed to encode_heap_oop_not_null");
 5814     bind(ok);
 5815   }
 5816 #endif
 5817   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 5818   if (CompressedOops::base() != nullptr) {
 5819     subq(r, r12_heapbase);
 5820   }
 5821   if (CompressedOops::shift() != 0) {
 5822     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5823     shrq(r, LogMinObjAlignmentInBytes);
 5824   }
 5825 }
 5826 
 5827 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 5828 #ifdef ASSERT
 5829   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 5830   if (CheckCompressedOops) {
 5831     Label ok;
 5832     testq(src, src);
 5833     jcc(Assembler::notEqual, ok);
 5834     STOP("null oop passed to encode_heap_oop_not_null2");
 5835     bind(ok);
 5836   }
 5837 #endif
 5838   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 5839   if (dst != src) {
 5840     movq(dst, src);
 5841   }
 5842   if (CompressedOops::base() != nullptr) {
 5843     subq(dst, r12_heapbase);
 5844   }
 5845   if (CompressedOops::shift() != 0) {
 5846     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5847     shrq(dst, LogMinObjAlignmentInBytes);
 5848   }
 5849 }
 5850 
 5851 void  MacroAssembler::decode_heap_oop(Register r) {
 5852 #ifdef ASSERT
 5853   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 5854 #endif
 5855   if (CompressedOops::base() == nullptr) {
 5856     if (CompressedOops::shift() != 0) {
 5857       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5858       shlq(r, LogMinObjAlignmentInBytes);
 5859     }
 5860   } else {
 5861     Label done;
 5862     shlq(r, LogMinObjAlignmentInBytes);
 5863     jccb(Assembler::equal, done);
 5864     addq(r, r12_heapbase);
 5865     bind(done);
 5866   }
 5867   verify_oop_msg(r, "broken oop in decode_heap_oop");
 5868 }
 5869 
 5870 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 5871   // Note: it will change flags
 5872   assert (UseCompressedOops, "should only be used for compressed headers");
 5873   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5874   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5875   // vtableStubs also counts instructions in pd_code_size_limit.
 5876   // Also do not verify_oop as this is called by verify_oop.
 5877   if (CompressedOops::shift() != 0) {
 5878     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5879     shlq(r, LogMinObjAlignmentInBytes);
 5880     if (CompressedOops::base() != nullptr) {
 5881       addq(r, r12_heapbase);
 5882     }
 5883   } else {
 5884     assert (CompressedOops::base() == nullptr, "sanity");
 5885   }
 5886 }
 5887 
 5888 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 5889   // Note: it will change flags
 5890   assert (UseCompressedOops, "should only be used for compressed headers");
 5891   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5892   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5893   // vtableStubs also counts instructions in pd_code_size_limit.
 5894   // Also do not verify_oop as this is called by verify_oop.
 5895   if (CompressedOops::shift() != 0) {
 5896     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5897     if (LogMinObjAlignmentInBytes == Address::times_8) {
 5898       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 5899     } else {
 5900       if (dst != src) {
 5901         movq(dst, src);
 5902       }
 5903       shlq(dst, LogMinObjAlignmentInBytes);
 5904       if (CompressedOops::base() != nullptr) {
 5905         addq(dst, r12_heapbase);
 5906       }
 5907     }
 5908   } else {
 5909     assert (CompressedOops::base() == nullptr, "sanity");
 5910     if (dst != src) {
 5911       movq(dst, src);
 5912     }
 5913   }
 5914 }
 5915 
 5916 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 5917   assert_different_registers(r, tmp);
 5918   if (CompressedKlassPointers::base() != nullptr) {
 5919     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5920     subq(r, tmp);
 5921   }
 5922   if (CompressedKlassPointers::shift() != 0) {
 5923     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5924     shrq(r, LogKlassAlignmentInBytes);
 5925   }
 5926 }
 5927 
 5928 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 5929   assert_different_registers(src, dst);
 5930   if (CompressedKlassPointers::base() != nullptr) {
 5931     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 5932     addq(dst, src);
 5933   } else {
 5934     movptr(dst, src);
 5935   }
 5936   if (CompressedKlassPointers::shift() != 0) {
 5937     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5938     shrq(dst, LogKlassAlignmentInBytes);
 5939   }
 5940 }
 5941 
 5942 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 5943   assert_different_registers(r, tmp);
 5944   // Note: it will change flags
 5945   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 5946   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5947   // vtableStubs also counts instructions in pd_code_size_limit.
 5948   // Also do not verify_oop as this is called by verify_oop.
 5949   if (CompressedKlassPointers::shift() != 0) {
 5950     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5951     shlq(r, LogKlassAlignmentInBytes);
 5952   }
 5953   if (CompressedKlassPointers::base() != nullptr) {
 5954     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5955     addq(r, tmp);
 5956   }
 5957 }
 5958 
 5959 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 5960   assert_different_registers(src, dst);
 5961   // Note: it will change flags
 5962   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5963   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5964   // vtableStubs also counts instructions in pd_code_size_limit.
 5965   // Also do not verify_oop as this is called by verify_oop.
 5966 
 5967   if (CompressedKlassPointers::base() == nullptr &&
 5968       CompressedKlassPointers::shift() == 0) {
 5969     // The best case scenario is that there is no base or shift. Then it is already
 5970     // a pointer that needs nothing but a register rename.
 5971     movl(dst, src);
 5972   } else {
 5973     if (CompressedKlassPointers::base() != nullptr) {
 5974       mov64(dst, (int64_t)CompressedKlassPointers::base());
 5975     } else {
 5976       xorq(dst, dst);
 5977     }
 5978     if (CompressedKlassPointers::shift() != 0) {
 5979       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5980       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 5981       leaq(dst, Address(dst, src, Address::times_8, 0));
 5982     } else {
 5983       addq(dst, src);
 5984     }
 5985   }
 5986 }
 5987 
 5988 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 5989   assert (UseCompressedOops, "should only be used for compressed headers");
 5990   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5991   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5992   int oop_index = oop_recorder()->find_index(obj);
 5993   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5994   mov_narrow_oop(dst, oop_index, rspec);
 5995 }
 5996 
 5997 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 5998   assert (UseCompressedOops, "should only be used for compressed headers");
 5999   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6000   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6001   int oop_index = oop_recorder()->find_index(obj);
 6002   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6003   mov_narrow_oop(dst, oop_index, rspec);
 6004 }
 6005 
 6006 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 6007   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6008   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6009   int klass_index = oop_recorder()->find_index(k);
 6010   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6011   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6012 }
 6013 
 6014 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 6015   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6016   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6017   int klass_index = oop_recorder()->find_index(k);
 6018   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6019   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6020 }
 6021 
 6022 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6023   assert (UseCompressedOops, "should only be used for compressed headers");
 6024   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6025   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6026   int oop_index = oop_recorder()->find_index(obj);
 6027   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6028   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6029 }
 6030 
 6031 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6032   assert (UseCompressedOops, "should only be used for compressed headers");
 6033   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6034   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6035   int oop_index = oop_recorder()->find_index(obj);
 6036   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6037   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6038 }
 6039 
 6040 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6041   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6042   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6043   int klass_index = oop_recorder()->find_index(k);
 6044   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6045   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6046 }
 6047 
 6048 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6049   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6050   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6051   int klass_index = oop_recorder()->find_index(k);
 6052   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6053   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6054 }
 6055 
 6056 void MacroAssembler::reinit_heapbase() {
 6057   if (UseCompressedOops) {
 6058     if (Universe::heap() != nullptr) {
 6059       if (CompressedOops::base() == nullptr) {
 6060         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6061       } else {
 6062         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 6063       }
 6064     } else {
 6065       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 6066     }
 6067   }
 6068 }
 6069 
 6070 #endif // _LP64
 6071 
 6072 #if COMPILER2_OR_JVMCI
 6073 
 6074 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6075 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6076   // cnt - number of qwords (8-byte words).
 6077   // base - start address, qword aligned.
 6078   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6079   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6080   if (use64byteVector) {
 6081     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 6082   } else if (MaxVectorSize >= 32) {
 6083     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
 6084   } else {
 6085     pxor(xtmp, xtmp);
 6086   }
 6087   jmp(L_zero_64_bytes);
 6088 
 6089   BIND(L_loop);
 6090   if (MaxVectorSize >= 32) {
 6091     fill64(base, 0, xtmp, use64byteVector);
 6092   } else {
 6093     movdqu(Address(base,  0), xtmp);
 6094     movdqu(Address(base, 16), xtmp);
 6095     movdqu(Address(base, 32), xtmp);
 6096     movdqu(Address(base, 48), xtmp);
 6097   }
 6098   addptr(base, 64);
 6099 
 6100   BIND(L_zero_64_bytes);
 6101   subptr(cnt, 8);
 6102   jccb(Assembler::greaterEqual, L_loop);
 6103 
 6104   // Copy trailing 64 bytes
 6105   if (use64byteVector) {
 6106     addptr(cnt, 8);
 6107     jccb(Assembler::equal, L_end);
 6108     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 6109     jmp(L_end);
 6110   } else {
 6111     addptr(cnt, 4);
 6112     jccb(Assembler::less, L_tail);
 6113     if (MaxVectorSize >= 32) {
 6114       vmovdqu(Address(base, 0), xtmp);
 6115     } else {
 6116       movdqu(Address(base,  0), xtmp);
 6117       movdqu(Address(base, 16), xtmp);
 6118     }
 6119   }
 6120   addptr(base, 32);
 6121   subptr(cnt, 4);
 6122 
 6123   BIND(L_tail);
 6124   addptr(cnt, 4);
 6125   jccb(Assembler::lessEqual, L_end);
 6126   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6127     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 6128   } else {
 6129     decrement(cnt);
 6130 
 6131     BIND(L_sloop);
 6132     movq(Address(base, 0), xtmp);
 6133     addptr(base, 8);
 6134     decrement(cnt);
 6135     jccb(Assembler::greaterEqual, L_sloop);
 6136   }
 6137   BIND(L_end);
 6138 }
 6139 
 6140 // Clearing constant sized memory using YMM/ZMM registers.
 6141 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6142   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6143   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6144 
 6145   int vector64_count = (cnt & (~0x7)) >> 3;
 6146   cnt = cnt & 0x7;
 6147   const int fill64_per_loop = 4;
 6148   const int max_unrolled_fill64 = 8;
 6149 
 6150   // 64 byte initialization loop.
 6151   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6152   int start64 = 0;
 6153   if (vector64_count > max_unrolled_fill64) {
 6154     Label LOOP;
 6155     Register index = rtmp;
 6156 
 6157     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6158 
 6159     movl(index, 0);
 6160     BIND(LOOP);
 6161     for (int i = 0; i < fill64_per_loop; i++) {
 6162       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6163     }
 6164     addl(index, fill64_per_loop * 64);
 6165     cmpl(index, start64 * 64);
 6166     jccb(Assembler::less, LOOP);
 6167   }
 6168   for (int i = start64; i < vector64_count; i++) {
 6169     fill64(base, i * 64, xtmp, use64byteVector);
 6170   }
 6171 
 6172   // Clear remaining 64 byte tail.
 6173   int disp = vector64_count * 64;
 6174   if (cnt) {
 6175     switch (cnt) {
 6176       case 1:
 6177         movq(Address(base, disp), xtmp);
 6178         break;
 6179       case 2:
 6180         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6181         break;
 6182       case 3:
 6183         movl(rtmp, 0x7);
 6184         kmovwl(mask, rtmp);
 6185         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6186         break;
 6187       case 4:
 6188         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6189         break;
 6190       case 5:
 6191         if (use64byteVector) {
 6192           movl(rtmp, 0x1F);
 6193           kmovwl(mask, rtmp);
 6194           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6195         } else {
 6196           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6197           movq(Address(base, disp + 32), xtmp);
 6198         }
 6199         break;
 6200       case 6:
 6201         if (use64byteVector) {
 6202           movl(rtmp, 0x3F);
 6203           kmovwl(mask, rtmp);
 6204           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6205         } else {
 6206           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6207           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6208         }
 6209         break;
 6210       case 7:
 6211         if (use64byteVector) {
 6212           movl(rtmp, 0x7F);
 6213           kmovwl(mask, rtmp);
 6214           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6215         } else {
 6216           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6217           movl(rtmp, 0x7);
 6218           kmovwl(mask, rtmp);
 6219           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6220         }
 6221         break;
 6222       default:
 6223         fatal("Unexpected length : %d\n",cnt);
 6224         break;
 6225     }
 6226   }
 6227 }
 6228 
 6229 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 6230                                bool is_large, KRegister mask) {
 6231   // cnt      - number of qwords (8-byte words).
 6232   // base     - start address, qword aligned.
 6233   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6234   assert(base==rdi, "base register must be edi for rep stos");
 6235   assert(tmp==rax,   "tmp register must be eax for rep stos");
 6236   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6237   assert(InitArrayShortSize % BytesPerLong == 0,
 6238     "InitArrayShortSize should be the multiple of BytesPerLong");
 6239 
 6240   Label DONE;
 6241   if (!is_large || !UseXMMForObjInit) {
 6242     xorptr(tmp, tmp);
 6243   }
 6244 
 6245   if (!is_large) {
 6246     Label LOOP, LONG;
 6247     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6248     jccb(Assembler::greater, LONG);
 6249 
 6250     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6251 
 6252     decrement(cnt);
 6253     jccb(Assembler::negative, DONE); // Zero length
 6254 
 6255     // Use individual pointer-sized stores for small counts:
 6256     BIND(LOOP);
 6257     movptr(Address(base, cnt, Address::times_ptr), tmp);
 6258     decrement(cnt);
 6259     jccb(Assembler::greaterEqual, LOOP);
 6260     jmpb(DONE);
 6261 
 6262     BIND(LONG);
 6263   }
 6264 
 6265   // Use longer rep-prefixed ops for non-small counts:
 6266   if (UseFastStosb) {
 6267     shlptr(cnt, 3); // convert to number of bytes
 6268     rep_stosb();
 6269   } else if (UseXMMForObjInit) {
 6270     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 6271   } else {
 6272     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6273     rep_stos();
 6274   }
 6275 
 6276   BIND(DONE);
 6277 }
 6278 
 6279 #endif //COMPILER2_OR_JVMCI
 6280 
 6281 
 6282 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6283                                    Register to, Register value, Register count,
 6284                                    Register rtmp, XMMRegister xtmp) {
 6285   ShortBranchVerifier sbv(this);
 6286   assert_different_registers(to, value, count, rtmp);
 6287   Label L_exit;
 6288   Label L_fill_2_bytes, L_fill_4_bytes;
 6289 
 6290 #if defined(COMPILER2) && defined(_LP64)
 6291   if(MaxVectorSize >=32 &&
 6292      VM_Version::supports_avx512vlbw() &&
 6293      VM_Version::supports_bmi2()) {
 6294     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6295     return;
 6296   }
 6297 #endif
 6298 
 6299   int shift = -1;
 6300   switch (t) {
 6301     case T_BYTE:
 6302       shift = 2;
 6303       break;
 6304     case T_SHORT:
 6305       shift = 1;
 6306       break;
 6307     case T_INT:
 6308       shift = 0;
 6309       break;
 6310     default: ShouldNotReachHere();
 6311   }
 6312 
 6313   if (t == T_BYTE) {
 6314     andl(value, 0xff);
 6315     movl(rtmp, value);
 6316     shll(rtmp, 8);
 6317     orl(value, rtmp);
 6318   }
 6319   if (t == T_SHORT) {
 6320     andl(value, 0xffff);
 6321   }
 6322   if (t == T_BYTE || t == T_SHORT) {
 6323     movl(rtmp, value);
 6324     shll(rtmp, 16);
 6325     orl(value, rtmp);
 6326   }
 6327 
 6328   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6329   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6330   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6331     Label L_skip_align2;
 6332     // align source address at 4 bytes address boundary
 6333     if (t == T_BYTE) {
 6334       Label L_skip_align1;
 6335       // One byte misalignment happens only for byte arrays
 6336       testptr(to, 1);
 6337       jccb(Assembler::zero, L_skip_align1);
 6338       movb(Address(to, 0), value);
 6339       increment(to);
 6340       decrement(count);
 6341       BIND(L_skip_align1);
 6342     }
 6343     // Two bytes misalignment happens only for byte and short (char) arrays
 6344     testptr(to, 2);
 6345     jccb(Assembler::zero, L_skip_align2);
 6346     movw(Address(to, 0), value);
 6347     addptr(to, 2);
 6348     subptr(count, 1<<(shift-1));
 6349     BIND(L_skip_align2);
 6350   }
 6351   if (UseSSE < 2) {
 6352     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6353     // Fill 32-byte chunks
 6354     subptr(count, 8 << shift);
 6355     jcc(Assembler::less, L_check_fill_8_bytes);
 6356     align(16);
 6357 
 6358     BIND(L_fill_32_bytes_loop);
 6359 
 6360     for (int i = 0; i < 32; i += 4) {
 6361       movl(Address(to, i), value);
 6362     }
 6363 
 6364     addptr(to, 32);
 6365     subptr(count, 8 << shift);
 6366     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6367     BIND(L_check_fill_8_bytes);
 6368     addptr(count, 8 << shift);
 6369     jccb(Assembler::zero, L_exit);
 6370     jmpb(L_fill_8_bytes);
 6371 
 6372     //
 6373     // length is too short, just fill qwords
 6374     //
 6375     BIND(L_fill_8_bytes_loop);
 6376     movl(Address(to, 0), value);
 6377     movl(Address(to, 4), value);
 6378     addptr(to, 8);
 6379     BIND(L_fill_8_bytes);
 6380     subptr(count, 1 << (shift + 1));
 6381     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6382     // fall through to fill 4 bytes
 6383   } else {
 6384     Label L_fill_32_bytes;
 6385     if (!UseUnalignedLoadStores) {
 6386       // align to 8 bytes, we know we are 4 byte aligned to start
 6387       testptr(to, 4);
 6388       jccb(Assembler::zero, L_fill_32_bytes);
 6389       movl(Address(to, 0), value);
 6390       addptr(to, 4);
 6391       subptr(count, 1<<shift);
 6392     }
 6393     BIND(L_fill_32_bytes);
 6394     {
 6395       assert( UseSSE >= 2, "supported cpu only" );
 6396       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6397       movdl(xtmp, value);
 6398       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6399         Label L_check_fill_32_bytes;
 6400         if (UseAVX > 2) {
 6401           // Fill 64-byte chunks
 6402           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6403 
 6404           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6405           cmpptr(count, VM_Version::avx3_threshold());
 6406           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6407 
 6408           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6409 
 6410           subptr(count, 16 << shift);
 6411           jccb(Assembler::less, L_check_fill_32_bytes);
 6412           align(16);
 6413 
 6414           BIND(L_fill_64_bytes_loop_avx3);
 6415           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6416           addptr(to, 64);
 6417           subptr(count, 16 << shift);
 6418           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6419           jmpb(L_check_fill_32_bytes);
 6420 
 6421           BIND(L_check_fill_64_bytes_avx2);
 6422         }
 6423         // Fill 64-byte chunks
 6424         Label L_fill_64_bytes_loop;
 6425         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6426 
 6427         subptr(count, 16 << shift);
 6428         jcc(Assembler::less, L_check_fill_32_bytes);
 6429         align(16);
 6430 
 6431         BIND(L_fill_64_bytes_loop);
 6432         vmovdqu(Address(to, 0), xtmp);
 6433         vmovdqu(Address(to, 32), xtmp);
 6434         addptr(to, 64);
 6435         subptr(count, 16 << shift);
 6436         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6437 
 6438         BIND(L_check_fill_32_bytes);
 6439         addptr(count, 8 << shift);
 6440         jccb(Assembler::less, L_check_fill_8_bytes);
 6441         vmovdqu(Address(to, 0), xtmp);
 6442         addptr(to, 32);
 6443         subptr(count, 8 << shift);
 6444 
 6445         BIND(L_check_fill_8_bytes);
 6446         // clean upper bits of YMM registers
 6447         movdl(xtmp, value);
 6448         pshufd(xtmp, xtmp, 0);
 6449       } else {
 6450         // Fill 32-byte chunks
 6451         pshufd(xtmp, xtmp, 0);
 6452 
 6453         subptr(count, 8 << shift);
 6454         jcc(Assembler::less, L_check_fill_8_bytes);
 6455         align(16);
 6456 
 6457         BIND(L_fill_32_bytes_loop);
 6458 
 6459         if (UseUnalignedLoadStores) {
 6460           movdqu(Address(to, 0), xtmp);
 6461           movdqu(Address(to, 16), xtmp);
 6462         } else {
 6463           movq(Address(to, 0), xtmp);
 6464           movq(Address(to, 8), xtmp);
 6465           movq(Address(to, 16), xtmp);
 6466           movq(Address(to, 24), xtmp);
 6467         }
 6468 
 6469         addptr(to, 32);
 6470         subptr(count, 8 << shift);
 6471         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6472 
 6473         BIND(L_check_fill_8_bytes);
 6474       }
 6475       addptr(count, 8 << shift);
 6476       jccb(Assembler::zero, L_exit);
 6477       jmpb(L_fill_8_bytes);
 6478 
 6479       //
 6480       // length is too short, just fill qwords
 6481       //
 6482       BIND(L_fill_8_bytes_loop);
 6483       movq(Address(to, 0), xtmp);
 6484       addptr(to, 8);
 6485       BIND(L_fill_8_bytes);
 6486       subptr(count, 1 << (shift + 1));
 6487       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6488     }
 6489   }
 6490   // fill trailing 4 bytes
 6491   BIND(L_fill_4_bytes);
 6492   testl(count, 1<<shift);
 6493   jccb(Assembler::zero, L_fill_2_bytes);
 6494   movl(Address(to, 0), value);
 6495   if (t == T_BYTE || t == T_SHORT) {
 6496     Label L_fill_byte;
 6497     addptr(to, 4);
 6498     BIND(L_fill_2_bytes);
 6499     // fill trailing 2 bytes
 6500     testl(count, 1<<(shift-1));
 6501     jccb(Assembler::zero, L_fill_byte);
 6502     movw(Address(to, 0), value);
 6503     if (t == T_BYTE) {
 6504       addptr(to, 2);
 6505       BIND(L_fill_byte);
 6506       // fill trailing byte
 6507       testl(count, 1);
 6508       jccb(Assembler::zero, L_exit);
 6509       movb(Address(to, 0), value);
 6510     } else {
 6511       BIND(L_fill_byte);
 6512     }
 6513   } else {
 6514     BIND(L_fill_2_bytes);
 6515   }
 6516   BIND(L_exit);
 6517 }
 6518 
 6519 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6520   switch(type) {
 6521     case T_BYTE:
 6522     case T_BOOLEAN:
 6523       evpbroadcastb(dst, src, vector_len);
 6524       break;
 6525     case T_SHORT:
 6526     case T_CHAR:
 6527       evpbroadcastw(dst, src, vector_len);
 6528       break;
 6529     case T_INT:
 6530     case T_FLOAT:
 6531       evpbroadcastd(dst, src, vector_len);
 6532       break;
 6533     case T_LONG:
 6534     case T_DOUBLE:
 6535       evpbroadcastq(dst, src, vector_len);
 6536       break;
 6537     default:
 6538       fatal("Unhandled type : %s", type2name(type));
 6539       break;
 6540   }
 6541 }
 6542 
 6543 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6544    //@IntrinsicCandidate
 6545    //private static int implEncodeISOArray(byte[] sa, int sp,
 6546    //byte[] da, int dp, int len) {
 6547    //  int i = 0;
 6548    //  for (; i < len; i++) {
 6549    //    char c = StringUTF16.getChar(sa, sp++);
 6550    //    if (c > '\u00FF')
 6551    //      break;
 6552    //    da[dp++] = (byte)c;
 6553    //  }
 6554    //  return i;
 6555    //}
 6556    //
 6557    //@IntrinsicCandidate
 6558    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6559    //    byte[] da, int dp, int len) {
 6560    //  int i = 0;
 6561    //  for (; i < len; i++) {
 6562    //    char c = sa[sp++];
 6563    //    if (c >= '\u0080')
 6564    //      break;
 6565    //    da[dp++] = (byte)c;
 6566    //  }
 6567    //  return i;
 6568    //}
 6569 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6570   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6571   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6572   Register tmp5, Register result, bool ascii) {
 6573 
 6574   // rsi: src
 6575   // rdi: dst
 6576   // rdx: len
 6577   // rcx: tmp5
 6578   // rax: result
 6579   ShortBranchVerifier sbv(this);
 6580   assert_different_registers(src, dst, len, tmp5, result);
 6581   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6582 
 6583   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6584   int short_mask = ascii ? 0xff80 : 0xff00;
 6585 
 6586   // set result
 6587   xorl(result, result);
 6588   // check for zero length
 6589   testl(len, len);
 6590   jcc(Assembler::zero, L_done);
 6591 
 6592   movl(result, len);
 6593 
 6594   // Setup pointers
 6595   lea(src, Address(src, len, Address::times_2)); // char[]
 6596   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6597   negptr(len);
 6598 
 6599   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6600     Label L_copy_8_chars, L_copy_8_chars_exit;
 6601     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6602 
 6603     if (UseAVX >= 2) {
 6604       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6605       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6606       movdl(tmp1Reg, tmp5);
 6607       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 6608       jmp(L_chars_32_check);
 6609 
 6610       bind(L_copy_32_chars);
 6611       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 6612       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 6613       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6614       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6615       jccb(Assembler::notZero, L_copy_32_chars_exit);
 6616       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6617       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 6618       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 6619 
 6620       bind(L_chars_32_check);
 6621       addptr(len, 32);
 6622       jcc(Assembler::lessEqual, L_copy_32_chars);
 6623 
 6624       bind(L_copy_32_chars_exit);
 6625       subptr(len, 16);
 6626       jccb(Assembler::greater, L_copy_16_chars_exit);
 6627 
 6628     } else if (UseSSE42Intrinsics) {
 6629       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6630       movdl(tmp1Reg, tmp5);
 6631       pshufd(tmp1Reg, tmp1Reg, 0);
 6632       jmpb(L_chars_16_check);
 6633     }
 6634 
 6635     bind(L_copy_16_chars);
 6636     if (UseAVX >= 2) {
 6637       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 6638       vptest(tmp2Reg, tmp1Reg);
 6639       jcc(Assembler::notZero, L_copy_16_chars_exit);
 6640       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 6641       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 6642     } else {
 6643       if (UseAVX > 0) {
 6644         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6645         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6646         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 6647       } else {
 6648         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6649         por(tmp2Reg, tmp3Reg);
 6650         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6651         por(tmp2Reg, tmp4Reg);
 6652       }
 6653       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6654       jccb(Assembler::notZero, L_copy_16_chars_exit);
 6655       packuswb(tmp3Reg, tmp4Reg);
 6656     }
 6657     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 6658 
 6659     bind(L_chars_16_check);
 6660     addptr(len, 16);
 6661     jcc(Assembler::lessEqual, L_copy_16_chars);
 6662 
 6663     bind(L_copy_16_chars_exit);
 6664     if (UseAVX >= 2) {
 6665       // clean upper bits of YMM registers
 6666       vpxor(tmp2Reg, tmp2Reg);
 6667       vpxor(tmp3Reg, tmp3Reg);
 6668       vpxor(tmp4Reg, tmp4Reg);
 6669       movdl(tmp1Reg, tmp5);
 6670       pshufd(tmp1Reg, tmp1Reg, 0);
 6671     }
 6672     subptr(len, 8);
 6673     jccb(Assembler::greater, L_copy_8_chars_exit);
 6674 
 6675     bind(L_copy_8_chars);
 6676     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 6677     ptest(tmp3Reg, tmp1Reg);
 6678     jccb(Assembler::notZero, L_copy_8_chars_exit);
 6679     packuswb(tmp3Reg, tmp1Reg);
 6680     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 6681     addptr(len, 8);
 6682     jccb(Assembler::lessEqual, L_copy_8_chars);
 6683 
 6684     bind(L_copy_8_chars_exit);
 6685     subptr(len, 8);
 6686     jccb(Assembler::zero, L_done);
 6687   }
 6688 
 6689   bind(L_copy_1_char);
 6690   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 6691   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 6692   jccb(Assembler::notZero, L_copy_1_char_exit);
 6693   movb(Address(dst, len, Address::times_1, 0), tmp5);
 6694   addptr(len, 1);
 6695   jccb(Assembler::less, L_copy_1_char);
 6696 
 6697   bind(L_copy_1_char_exit);
 6698   addptr(result, len); // len is negative count of not processed elements
 6699 
 6700   bind(L_done);
 6701 }
 6702 
 6703 #ifdef _LP64
 6704 /**
 6705  * Helper for multiply_to_len().
 6706  */
 6707 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 6708   addq(dest_lo, src1);
 6709   adcq(dest_hi, 0);
 6710   addq(dest_lo, src2);
 6711   adcq(dest_hi, 0);
 6712 }
 6713 
 6714 /**
 6715  * Multiply 64 bit by 64 bit first loop.
 6716  */
 6717 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 6718                                            Register y, Register y_idx, Register z,
 6719                                            Register carry, Register product,
 6720                                            Register idx, Register kdx) {
 6721   //
 6722   //  jlong carry, x[], y[], z[];
 6723   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 6724   //    huge_128 product = y[idx] * x[xstart] + carry;
 6725   //    z[kdx] = (jlong)product;
 6726   //    carry  = (jlong)(product >>> 64);
 6727   //  }
 6728   //  z[xstart] = carry;
 6729   //
 6730 
 6731   Label L_first_loop, L_first_loop_exit;
 6732   Label L_one_x, L_one_y, L_multiply;
 6733 
 6734   decrementl(xstart);
 6735   jcc(Assembler::negative, L_one_x);
 6736 
 6737   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 6738   rorq(x_xstart, 32); // convert big-endian to little-endian
 6739 
 6740   bind(L_first_loop);
 6741   decrementl(idx);
 6742   jcc(Assembler::negative, L_first_loop_exit);
 6743   decrementl(idx);
 6744   jcc(Assembler::negative, L_one_y);
 6745   movq(y_idx, Address(y, idx, Address::times_4,  0));
 6746   rorq(y_idx, 32); // convert big-endian to little-endian
 6747   bind(L_multiply);
 6748   movq(product, x_xstart);
 6749   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 6750   addq(product, carry);
 6751   adcq(rdx, 0);
 6752   subl(kdx, 2);
 6753   movl(Address(z, kdx, Address::times_4,  4), product);
 6754   shrq(product, 32);
 6755   movl(Address(z, kdx, Address::times_4,  0), product);
 6756   movq(carry, rdx);
 6757   jmp(L_first_loop);
 6758 
 6759   bind(L_one_y);
 6760   movl(y_idx, Address(y,  0));
 6761   jmp(L_multiply);
 6762 
 6763   bind(L_one_x);
 6764   movl(x_xstart, Address(x,  0));
 6765   jmp(L_first_loop);
 6766 
 6767   bind(L_first_loop_exit);
 6768 }
 6769 
 6770 /**
 6771  * Multiply 64 bit by 64 bit and add 128 bit.
 6772  */
 6773 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 6774                                             Register yz_idx, Register idx,
 6775                                             Register carry, Register product, int offset) {
 6776   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 6777   //     z[kdx] = (jlong)product;
 6778 
 6779   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 6780   rorq(yz_idx, 32); // convert big-endian to little-endian
 6781   movq(product, x_xstart);
 6782   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 6783   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 6784   rorq(yz_idx, 32); // convert big-endian to little-endian
 6785 
 6786   add2_with_carry(rdx, product, carry, yz_idx);
 6787 
 6788   movl(Address(z, idx, Address::times_4,  offset+4), product);
 6789   shrq(product, 32);
 6790   movl(Address(z, idx, Address::times_4,  offset), product);
 6791 
 6792 }
 6793 
 6794 /**
 6795  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 6796  */
 6797 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 6798                                              Register yz_idx, Register idx, Register jdx,
 6799                                              Register carry, Register product,
 6800                                              Register carry2) {
 6801   //   jlong carry, x[], y[], z[];
 6802   //   int kdx = ystart+1;
 6803   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6804   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 6805   //     z[kdx+idx+1] = (jlong)product;
 6806   //     jlong carry2  = (jlong)(product >>> 64);
 6807   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 6808   //     z[kdx+idx] = (jlong)product;
 6809   //     carry  = (jlong)(product >>> 64);
 6810   //   }
 6811   //   idx += 2;
 6812   //   if (idx > 0) {
 6813   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 6814   //     z[kdx+idx] = (jlong)product;
 6815   //     carry  = (jlong)(product >>> 64);
 6816   //   }
 6817   //
 6818 
 6819   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6820 
 6821   movl(jdx, idx);
 6822   andl(jdx, 0xFFFFFFFC);
 6823   shrl(jdx, 2);
 6824 
 6825   bind(L_third_loop);
 6826   subl(jdx, 1);
 6827   jcc(Assembler::negative, L_third_loop_exit);
 6828   subl(idx, 4);
 6829 
 6830   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 6831   movq(carry2, rdx);
 6832 
 6833   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 6834   movq(carry, rdx);
 6835   jmp(L_third_loop);
 6836 
 6837   bind (L_third_loop_exit);
 6838 
 6839   andl (idx, 0x3);
 6840   jcc(Assembler::zero, L_post_third_loop_done);
 6841 
 6842   Label L_check_1;
 6843   subl(idx, 2);
 6844   jcc(Assembler::negative, L_check_1);
 6845 
 6846   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 6847   movq(carry, rdx);
 6848 
 6849   bind (L_check_1);
 6850   addl (idx, 0x2);
 6851   andl (idx, 0x1);
 6852   subl(idx, 1);
 6853   jcc(Assembler::negative, L_post_third_loop_done);
 6854 
 6855   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 6856   movq(product, x_xstart);
 6857   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 6858   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 6859 
 6860   add2_with_carry(rdx, product, yz_idx, carry);
 6861 
 6862   movl(Address(z, idx, Address::times_4,  0), product);
 6863   shrq(product, 32);
 6864 
 6865   shlq(rdx, 32);
 6866   orq(product, rdx);
 6867   movq(carry, product);
 6868 
 6869   bind(L_post_third_loop_done);
 6870 }
 6871 
 6872 /**
 6873  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 6874  *
 6875  */
 6876 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 6877                                                   Register carry, Register carry2,
 6878                                                   Register idx, Register jdx,
 6879                                                   Register yz_idx1, Register yz_idx2,
 6880                                                   Register tmp, Register tmp3, Register tmp4) {
 6881   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 6882 
 6883   //   jlong carry, x[], y[], z[];
 6884   //   int kdx = ystart+1;
 6885   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6886   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 6887   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 6888   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 6889   //     carry  = (jlong)(tmp4 >>> 64);
 6890   //     z[kdx+idx+1] = (jlong)tmp3;
 6891   //     z[kdx+idx] = (jlong)tmp4;
 6892   //   }
 6893   //   idx += 2;
 6894   //   if (idx > 0) {
 6895   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 6896   //     z[kdx+idx] = (jlong)yz_idx1;
 6897   //     carry  = (jlong)(yz_idx1 >>> 64);
 6898   //   }
 6899   //
 6900 
 6901   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6902 
 6903   movl(jdx, idx);
 6904   andl(jdx, 0xFFFFFFFC);
 6905   shrl(jdx, 2);
 6906 
 6907   bind(L_third_loop);
 6908   subl(jdx, 1);
 6909   jcc(Assembler::negative, L_third_loop_exit);
 6910   subl(idx, 4);
 6911 
 6912   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 6913   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 6914   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 6915   rorxq(yz_idx2, yz_idx2, 32);
 6916 
 6917   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 6918   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 6919 
 6920   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 6921   rorxq(yz_idx1, yz_idx1, 32);
 6922   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6923   rorxq(yz_idx2, yz_idx2, 32);
 6924 
 6925   if (VM_Version::supports_adx()) {
 6926     adcxq(tmp3, carry);
 6927     adoxq(tmp3, yz_idx1);
 6928 
 6929     adcxq(tmp4, tmp);
 6930     adoxq(tmp4, yz_idx2);
 6931 
 6932     movl(carry, 0); // does not affect flags
 6933     adcxq(carry2, carry);
 6934     adoxq(carry2, carry);
 6935   } else {
 6936     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 6937     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 6938   }
 6939   movq(carry, carry2);
 6940 
 6941   movl(Address(z, idx, Address::times_4, 12), tmp3);
 6942   shrq(tmp3, 32);
 6943   movl(Address(z, idx, Address::times_4,  8), tmp3);
 6944 
 6945   movl(Address(z, idx, Address::times_4,  4), tmp4);
 6946   shrq(tmp4, 32);
 6947   movl(Address(z, idx, Address::times_4,  0), tmp4);
 6948 
 6949   jmp(L_third_loop);
 6950 
 6951   bind (L_third_loop_exit);
 6952 
 6953   andl (idx, 0x3);
 6954   jcc(Assembler::zero, L_post_third_loop_done);
 6955 
 6956   Label L_check_1;
 6957   subl(idx, 2);
 6958   jcc(Assembler::negative, L_check_1);
 6959 
 6960   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 6961   rorxq(yz_idx1, yz_idx1, 32);
 6962   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 6963   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6964   rorxq(yz_idx2, yz_idx2, 32);
 6965 
 6966   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 6967 
 6968   movl(Address(z, idx, Address::times_4,  4), tmp3);
 6969   shrq(tmp3, 32);
 6970   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6971   movq(carry, tmp4);
 6972 
 6973   bind (L_check_1);
 6974   addl (idx, 0x2);
 6975   andl (idx, 0x1);
 6976   subl(idx, 1);
 6977   jcc(Assembler::negative, L_post_third_loop_done);
 6978   movl(tmp4, Address(y, idx, Address::times_4,  0));
 6979   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 6980   movl(tmp4, Address(z, idx, Address::times_4,  0));
 6981 
 6982   add2_with_carry(carry2, tmp3, tmp4, carry);
 6983 
 6984   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6985   shrq(tmp3, 32);
 6986 
 6987   shlq(carry2, 32);
 6988   orq(tmp3, carry2);
 6989   movq(carry, tmp3);
 6990 
 6991   bind(L_post_third_loop_done);
 6992 }
 6993 
 6994 /**
 6995  * Code for BigInteger::multiplyToLen() intrinsic.
 6996  *
 6997  * rdi: x
 6998  * rax: xlen
 6999  * rsi: y
 7000  * rcx: ylen
 7001  * r8:  z
 7002  * r11: tmp0
 7003  * r12: tmp1
 7004  * r13: tmp2
 7005  * r14: tmp3
 7006  * r15: tmp4
 7007  * rbx: tmp5
 7008  *
 7009  */
 7010 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
 7011                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 7012   ShortBranchVerifier sbv(this);
 7013   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 7014 
 7015   push(tmp0);
 7016   push(tmp1);
 7017   push(tmp2);
 7018   push(tmp3);
 7019   push(tmp4);
 7020   push(tmp5);
 7021 
 7022   push(xlen);
 7023 
 7024   const Register idx = tmp1;
 7025   const Register kdx = tmp2;
 7026   const Register xstart = tmp3;
 7027 
 7028   const Register y_idx = tmp4;
 7029   const Register carry = tmp5;
 7030   const Register product  = xlen;
 7031   const Register x_xstart = tmp0;
 7032 
 7033   // First Loop.
 7034   //
 7035   //  final static long LONG_MASK = 0xffffffffL;
 7036   //  int xstart = xlen - 1;
 7037   //  int ystart = ylen - 1;
 7038   //  long carry = 0;
 7039   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7040   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7041   //    z[kdx] = (int)product;
 7042   //    carry = product >>> 32;
 7043   //  }
 7044   //  z[xstart] = (int)carry;
 7045   //
 7046 
 7047   movl(idx, ylen);               // idx = ylen;
 7048   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
 7049   xorq(carry, carry);            // carry = 0;
 7050 
 7051   Label L_done;
 7052 
 7053   movl(xstart, xlen);
 7054   decrementl(xstart);
 7055   jcc(Assembler::negative, L_done);
 7056 
 7057   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7058 
 7059   Label L_second_loop;
 7060   testl(kdx, kdx);
 7061   jcc(Assembler::zero, L_second_loop);
 7062 
 7063   Label L_carry;
 7064   subl(kdx, 1);
 7065   jcc(Assembler::zero, L_carry);
 7066 
 7067   movl(Address(z, kdx, Address::times_4,  0), carry);
 7068   shrq(carry, 32);
 7069   subl(kdx, 1);
 7070 
 7071   bind(L_carry);
 7072   movl(Address(z, kdx, Address::times_4,  0), carry);
 7073 
 7074   // Second and third (nested) loops.
 7075   //
 7076   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7077   //   carry = 0;
 7078   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7079   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7080   //                    (z[k] & LONG_MASK) + carry;
 7081   //     z[k] = (int)product;
 7082   //     carry = product >>> 32;
 7083   //   }
 7084   //   z[i] = (int)carry;
 7085   // }
 7086   //
 7087   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7088 
 7089   const Register jdx = tmp1;
 7090 
 7091   bind(L_second_loop);
 7092   xorl(carry, carry);    // carry = 0;
 7093   movl(jdx, ylen);       // j = ystart+1
 7094 
 7095   subl(xstart, 1);       // i = xstart-1;
 7096   jcc(Assembler::negative, L_done);
 7097 
 7098   push (z);
 7099 
 7100   Label L_last_x;
 7101   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7102   subl(xstart, 1);       // i = xstart-1;
 7103   jcc(Assembler::negative, L_last_x);
 7104 
 7105   if (UseBMI2Instructions) {
 7106     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7107     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7108   } else {
 7109     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7110     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7111   }
 7112 
 7113   Label L_third_loop_prologue;
 7114   bind(L_third_loop_prologue);
 7115 
 7116   push (x);
 7117   push (xstart);
 7118   push (ylen);
 7119 
 7120 
 7121   if (UseBMI2Instructions) {
 7122     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7123   } else { // !UseBMI2Instructions
 7124     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7125   }
 7126 
 7127   pop(ylen);
 7128   pop(xlen);
 7129   pop(x);
 7130   pop(z);
 7131 
 7132   movl(tmp3, xlen);
 7133   addl(tmp3, 1);
 7134   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7135   subl(tmp3, 1);
 7136   jccb(Assembler::negative, L_done);
 7137 
 7138   shrq(carry, 32);
 7139   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7140   jmp(L_second_loop);
 7141 
 7142   // Next infrequent code is moved outside loops.
 7143   bind(L_last_x);
 7144   if (UseBMI2Instructions) {
 7145     movl(rdx, Address(x,  0));
 7146   } else {
 7147     movl(x_xstart, Address(x,  0));
 7148   }
 7149   jmp(L_third_loop_prologue);
 7150 
 7151   bind(L_done);
 7152 
 7153   pop(xlen);
 7154 
 7155   pop(tmp5);
 7156   pop(tmp4);
 7157   pop(tmp3);
 7158   pop(tmp2);
 7159   pop(tmp1);
 7160   pop(tmp0);
 7161 }
 7162 
 7163 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7164   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7165   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7166   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7167   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7168   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7169   Label SAME_TILL_END, DONE;
 7170   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7171 
 7172   //scale is in rcx in both Win64 and Unix
 7173   ShortBranchVerifier sbv(this);
 7174 
 7175   shlq(length);
 7176   xorq(result, result);
 7177 
 7178   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7179       VM_Version::supports_avx512vlbw()) {
 7180     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7181 
 7182     cmpq(length, 64);
 7183     jcc(Assembler::less, VECTOR32_TAIL);
 7184 
 7185     movq(tmp1, length);
 7186     andq(tmp1, 0x3F);      // tail count
 7187     andq(length, ~(0x3F)); //vector count
 7188 
 7189     bind(VECTOR64_LOOP);
 7190     // AVX512 code to compare 64 byte vectors.
 7191     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7192     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7193     kortestql(k7, k7);
 7194     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7195     addq(result, 64);
 7196     subq(length, 64);
 7197     jccb(Assembler::notZero, VECTOR64_LOOP);
 7198 
 7199     //bind(VECTOR64_TAIL);
 7200     testq(tmp1, tmp1);
 7201     jcc(Assembler::zero, SAME_TILL_END);
 7202 
 7203     //bind(VECTOR64_TAIL);
 7204     // AVX512 code to compare up to 63 byte vectors.
 7205     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7206     shlxq(tmp2, tmp2, tmp1);
 7207     notq(tmp2);
 7208     kmovql(k3, tmp2);
 7209 
 7210     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7211     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7212 
 7213     ktestql(k7, k3);
 7214     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7215 
 7216     bind(VECTOR64_NOT_EQUAL);
 7217     kmovql(tmp1, k7);
 7218     notq(tmp1);
 7219     tzcntq(tmp1, tmp1);
 7220     addq(result, tmp1);
 7221     shrq(result);
 7222     jmp(DONE);
 7223     bind(VECTOR32_TAIL);
 7224   }
 7225 
 7226   cmpq(length, 8);
 7227   jcc(Assembler::equal, VECTOR8_LOOP);
 7228   jcc(Assembler::less, VECTOR4_TAIL);
 7229 
 7230   if (UseAVX >= 2) {
 7231     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7232 
 7233     cmpq(length, 16);
 7234     jcc(Assembler::equal, VECTOR16_LOOP);
 7235     jcc(Assembler::less, VECTOR8_LOOP);
 7236 
 7237     cmpq(length, 32);
 7238     jccb(Assembler::less, VECTOR16_TAIL);
 7239 
 7240     subq(length, 32);
 7241     bind(VECTOR32_LOOP);
 7242     vmovdqu(rymm0, Address(obja, result));
 7243     vmovdqu(rymm1, Address(objb, result));
 7244     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7245     vptest(rymm2, rymm2);
 7246     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7247     addq(result, 32);
 7248     subq(length, 32);
 7249     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7250     addq(length, 32);
 7251     jcc(Assembler::equal, SAME_TILL_END);
 7252     //falling through if less than 32 bytes left //close the branch here.
 7253 
 7254     bind(VECTOR16_TAIL);
 7255     cmpq(length, 16);
 7256     jccb(Assembler::less, VECTOR8_TAIL);
 7257     bind(VECTOR16_LOOP);
 7258     movdqu(rymm0, Address(obja, result));
 7259     movdqu(rymm1, Address(objb, result));
 7260     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7261     ptest(rymm2, rymm2);
 7262     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7263     addq(result, 16);
 7264     subq(length, 16);
 7265     jcc(Assembler::equal, SAME_TILL_END);
 7266     //falling through if less than 16 bytes left
 7267   } else {//regular intrinsics
 7268 
 7269     cmpq(length, 16);
 7270     jccb(Assembler::less, VECTOR8_TAIL);
 7271 
 7272     subq(length, 16);
 7273     bind(VECTOR16_LOOP);
 7274     movdqu(rymm0, Address(obja, result));
 7275     movdqu(rymm1, Address(objb, result));
 7276     pxor(rymm0, rymm1);
 7277     ptest(rymm0, rymm0);
 7278     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7279     addq(result, 16);
 7280     subq(length, 16);
 7281     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7282     addq(length, 16);
 7283     jcc(Assembler::equal, SAME_TILL_END);
 7284     //falling through if less than 16 bytes left
 7285   }
 7286 
 7287   bind(VECTOR8_TAIL);
 7288   cmpq(length, 8);
 7289   jccb(Assembler::less, VECTOR4_TAIL);
 7290   bind(VECTOR8_LOOP);
 7291   movq(tmp1, Address(obja, result));
 7292   movq(tmp2, Address(objb, result));
 7293   xorq(tmp1, tmp2);
 7294   testq(tmp1, tmp1);
 7295   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7296   addq(result, 8);
 7297   subq(length, 8);
 7298   jcc(Assembler::equal, SAME_TILL_END);
 7299   //falling through if less than 8 bytes left
 7300 
 7301   bind(VECTOR4_TAIL);
 7302   cmpq(length, 4);
 7303   jccb(Assembler::less, BYTES_TAIL);
 7304   bind(VECTOR4_LOOP);
 7305   movl(tmp1, Address(obja, result));
 7306   xorl(tmp1, Address(objb, result));
 7307   testl(tmp1, tmp1);
 7308   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7309   addq(result, 4);
 7310   subq(length, 4);
 7311   jcc(Assembler::equal, SAME_TILL_END);
 7312   //falling through if less than 4 bytes left
 7313 
 7314   bind(BYTES_TAIL);
 7315   bind(BYTES_LOOP);
 7316   load_unsigned_byte(tmp1, Address(obja, result));
 7317   load_unsigned_byte(tmp2, Address(objb, result));
 7318   xorl(tmp1, tmp2);
 7319   testl(tmp1, tmp1);
 7320   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7321   decq(length);
 7322   jcc(Assembler::zero, SAME_TILL_END);
 7323   incq(result);
 7324   load_unsigned_byte(tmp1, Address(obja, result));
 7325   load_unsigned_byte(tmp2, Address(objb, result));
 7326   xorl(tmp1, tmp2);
 7327   testl(tmp1, tmp1);
 7328   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7329   decq(length);
 7330   jcc(Assembler::zero, SAME_TILL_END);
 7331   incq(result);
 7332   load_unsigned_byte(tmp1, Address(obja, result));
 7333   load_unsigned_byte(tmp2, Address(objb, result));
 7334   xorl(tmp1, tmp2);
 7335   testl(tmp1, tmp1);
 7336   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7337   jmp(SAME_TILL_END);
 7338 
 7339   if (UseAVX >= 2) {
 7340     bind(VECTOR32_NOT_EQUAL);
 7341     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7342     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7343     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7344     vpmovmskb(tmp1, rymm0);
 7345     bsfq(tmp1, tmp1);
 7346     addq(result, tmp1);
 7347     shrq(result);
 7348     jmp(DONE);
 7349   }
 7350 
 7351   bind(VECTOR16_NOT_EQUAL);
 7352   if (UseAVX >= 2) {
 7353     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7354     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7355     pxor(rymm0, rymm2);
 7356   } else {
 7357     pcmpeqb(rymm2, rymm2);
 7358     pxor(rymm0, rymm1);
 7359     pcmpeqb(rymm0, rymm1);
 7360     pxor(rymm0, rymm2);
 7361   }
 7362   pmovmskb(tmp1, rymm0);
 7363   bsfq(tmp1, tmp1);
 7364   addq(result, tmp1);
 7365   shrq(result);
 7366   jmpb(DONE);
 7367 
 7368   bind(VECTOR8_NOT_EQUAL);
 7369   bind(VECTOR4_NOT_EQUAL);
 7370   bsfq(tmp1, tmp1);
 7371   shrq(tmp1, 3);
 7372   addq(result, tmp1);
 7373   bind(BYTES_NOT_EQUAL);
 7374   shrq(result);
 7375   jmpb(DONE);
 7376 
 7377   bind(SAME_TILL_END);
 7378   mov64(result, -1);
 7379 
 7380   bind(DONE);
 7381 }
 7382 
 7383 //Helper functions for square_to_len()
 7384 
 7385 /**
 7386  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7387  * Preserves x and z and modifies rest of the registers.
 7388  */
 7389 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7390   // Perform square and right shift by 1
 7391   // Handle odd xlen case first, then for even xlen do the following
 7392   // jlong carry = 0;
 7393   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7394   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7395   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7396   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7397   //     carry = (jlong)product;
 7398   // }
 7399 
 7400   xorq(tmp5, tmp5);     // carry
 7401   xorq(rdxReg, rdxReg);
 7402   xorl(tmp1, tmp1);     // index for x
 7403   xorl(tmp4, tmp4);     // index for z
 7404 
 7405   Label L_first_loop, L_first_loop_exit;
 7406 
 7407   testl(xlen, 1);
 7408   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7409 
 7410   // Square and right shift by 1 the odd element using 32 bit multiply
 7411   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7412   imulq(raxReg, raxReg);
 7413   shrq(raxReg, 1);
 7414   adcq(tmp5, 0);
 7415   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7416   incrementl(tmp1);
 7417   addl(tmp4, 2);
 7418 
 7419   // Square and  right shift by 1 the rest using 64 bit multiply
 7420   bind(L_first_loop);
 7421   cmpptr(tmp1, xlen);
 7422   jccb(Assembler::equal, L_first_loop_exit);
 7423 
 7424   // Square
 7425   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7426   rorq(raxReg, 32);    // convert big-endian to little-endian
 7427   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7428 
 7429   // Right shift by 1 and save carry
 7430   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7431   rcrq(rdxReg, 1);
 7432   rcrq(raxReg, 1);
 7433   adcq(tmp5, 0);
 7434 
 7435   // Store result in z
 7436   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7437   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7438 
 7439   // Update indices for x and z
 7440   addl(tmp1, 2);
 7441   addl(tmp4, 4);
 7442   jmp(L_first_loop);
 7443 
 7444   bind(L_first_loop_exit);
 7445 }
 7446 
 7447 
 7448 /**
 7449  * Perform the following multiply add operation using BMI2 instructions
 7450  * carry:sum = sum + op1*op2 + carry
 7451  * op2 should be in rdx
 7452  * op2 is preserved, all other registers are modified
 7453  */
 7454 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7455   // assert op2 is rdx
 7456   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7457   addq(sum, carry);
 7458   adcq(tmp2, 0);
 7459   addq(sum, op1);
 7460   adcq(tmp2, 0);
 7461   movq(carry, tmp2);
 7462 }
 7463 
 7464 /**
 7465  * Perform the following multiply add operation:
 7466  * carry:sum = sum + op1*op2 + carry
 7467  * Preserves op1, op2 and modifies rest of registers
 7468  */
 7469 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7470   // rdx:rax = op1 * op2
 7471   movq(raxReg, op2);
 7472   mulq(op1);
 7473 
 7474   //  rdx:rax = sum + carry + rdx:rax
 7475   addq(sum, carry);
 7476   adcq(rdxReg, 0);
 7477   addq(sum, raxReg);
 7478   adcq(rdxReg, 0);
 7479 
 7480   // carry:sum = rdx:sum
 7481   movq(carry, rdxReg);
 7482 }
 7483 
 7484 /**
 7485  * Add 64 bit long carry into z[] with carry propagation.
 7486  * Preserves z and carry register values and modifies rest of registers.
 7487  *
 7488  */
 7489 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7490   Label L_fourth_loop, L_fourth_loop_exit;
 7491 
 7492   movl(tmp1, 1);
 7493   subl(zlen, 2);
 7494   addq(Address(z, zlen, Address::times_4, 0), carry);
 7495 
 7496   bind(L_fourth_loop);
 7497   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7498   subl(zlen, 2);
 7499   jccb(Assembler::negative, L_fourth_loop_exit);
 7500   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7501   jmp(L_fourth_loop);
 7502   bind(L_fourth_loop_exit);
 7503 }
 7504 
 7505 /**
 7506  * Shift z[] left by 1 bit.
 7507  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7508  *
 7509  */
 7510 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7511 
 7512   Label L_fifth_loop, L_fifth_loop_exit;
 7513 
 7514   // Fifth loop
 7515   // Perform primitiveLeftShift(z, zlen, 1)
 7516 
 7517   const Register prev_carry = tmp1;
 7518   const Register new_carry = tmp4;
 7519   const Register value = tmp2;
 7520   const Register zidx = tmp3;
 7521 
 7522   // int zidx, carry;
 7523   // long value;
 7524   // carry = 0;
 7525   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7526   //    (carry:value)  = (z[i] << 1) | carry ;
 7527   //    z[i] = value;
 7528   // }
 7529 
 7530   movl(zidx, zlen);
 7531   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7532 
 7533   bind(L_fifth_loop);
 7534   decl(zidx);  // Use decl to preserve carry flag
 7535   decl(zidx);
 7536   jccb(Assembler::negative, L_fifth_loop_exit);
 7537 
 7538   if (UseBMI2Instructions) {
 7539      movq(value, Address(z, zidx, Address::times_4, 0));
 7540      rclq(value, 1);
 7541      rorxq(value, value, 32);
 7542      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7543   }
 7544   else {
 7545     // clear new_carry
 7546     xorl(new_carry, new_carry);
 7547 
 7548     // Shift z[i] by 1, or in previous carry and save new carry
 7549     movq(value, Address(z, zidx, Address::times_4, 0));
 7550     shlq(value, 1);
 7551     adcl(new_carry, 0);
 7552 
 7553     orq(value, prev_carry);
 7554     rorq(value, 0x20);
 7555     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7556 
 7557     // Set previous carry = new carry
 7558     movl(prev_carry, new_carry);
 7559   }
 7560   jmp(L_fifth_loop);
 7561 
 7562   bind(L_fifth_loop_exit);
 7563 }
 7564 
 7565 
 7566 /**
 7567  * Code for BigInteger::squareToLen() intrinsic
 7568  *
 7569  * rdi: x
 7570  * rsi: len
 7571  * r8:  z
 7572  * rcx: zlen
 7573  * r12: tmp1
 7574  * r13: tmp2
 7575  * r14: tmp3
 7576  * r15: tmp4
 7577  * rbx: tmp5
 7578  *
 7579  */
 7580 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7581 
 7582   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7583   push(tmp1);
 7584   push(tmp2);
 7585   push(tmp3);
 7586   push(tmp4);
 7587   push(tmp5);
 7588 
 7589   // First loop
 7590   // Store the squares, right shifted one bit (i.e., divided by 2).
 7591   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7592 
 7593   // Add in off-diagonal sums.
 7594   //
 7595   // Second, third (nested) and fourth loops.
 7596   // zlen +=2;
 7597   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7598   //    carry = 0;
 7599   //    long op2 = x[xidx:xidx+1];
 7600   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7601   //       k -= 2;
 7602   //       long op1 = x[j:j+1];
 7603   //       long sum = z[k:k+1];
 7604   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7605   //       z[k:k+1] = sum;
 7606   //    }
 7607   //    add_one_64(z, k, carry, tmp_regs);
 7608   // }
 7609 
 7610   const Register carry = tmp5;
 7611   const Register sum = tmp3;
 7612   const Register op1 = tmp4;
 7613   Register op2 = tmp2;
 7614 
 7615   push(zlen);
 7616   push(len);
 7617   addl(zlen,2);
 7618   bind(L_second_loop);
 7619   xorq(carry, carry);
 7620   subl(zlen, 4);
 7621   subl(len, 2);
 7622   push(zlen);
 7623   push(len);
 7624   cmpl(len, 0);
 7625   jccb(Assembler::lessEqual, L_second_loop_exit);
 7626 
 7627   // Multiply an array by one 64 bit long.
 7628   if (UseBMI2Instructions) {
 7629     op2 = rdxReg;
 7630     movq(op2, Address(x, len, Address::times_4,  0));
 7631     rorxq(op2, op2, 32);
 7632   }
 7633   else {
 7634     movq(op2, Address(x, len, Address::times_4,  0));
 7635     rorq(op2, 32);
 7636   }
 7637 
 7638   bind(L_third_loop);
 7639   decrementl(len);
 7640   jccb(Assembler::negative, L_third_loop_exit);
 7641   decrementl(len);
 7642   jccb(Assembler::negative, L_last_x);
 7643 
 7644   movq(op1, Address(x, len, Address::times_4,  0));
 7645   rorq(op1, 32);
 7646 
 7647   bind(L_multiply);
 7648   subl(zlen, 2);
 7649   movq(sum, Address(z, zlen, Address::times_4,  0));
 7650 
 7651   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 7652   if (UseBMI2Instructions) {
 7653     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 7654   }
 7655   else {
 7656     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7657   }
 7658 
 7659   movq(Address(z, zlen, Address::times_4, 0), sum);
 7660 
 7661   jmp(L_third_loop);
 7662   bind(L_third_loop_exit);
 7663 
 7664   // Fourth loop
 7665   // Add 64 bit long carry into z with carry propagation.
 7666   // Uses offsetted zlen.
 7667   add_one_64(z, zlen, carry, tmp1);
 7668 
 7669   pop(len);
 7670   pop(zlen);
 7671   jmp(L_second_loop);
 7672 
 7673   // Next infrequent code is moved outside loops.
 7674   bind(L_last_x);
 7675   movl(op1, Address(x, 0));
 7676   jmp(L_multiply);
 7677 
 7678   bind(L_second_loop_exit);
 7679   pop(len);
 7680   pop(zlen);
 7681   pop(len);
 7682   pop(zlen);
 7683 
 7684   // Fifth loop
 7685   // Shift z left 1 bit.
 7686   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 7687 
 7688   // z[zlen-1] |= x[len-1] & 1;
 7689   movl(tmp3, Address(x, len, Address::times_4, -4));
 7690   andl(tmp3, 1);
 7691   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 7692 
 7693   pop(tmp5);
 7694   pop(tmp4);
 7695   pop(tmp3);
 7696   pop(tmp2);
 7697   pop(tmp1);
 7698 }
 7699 
 7700 /**
 7701  * Helper function for mul_add()
 7702  * Multiply the in[] by int k and add to out[] starting at offset offs using
 7703  * 128 bit by 32 bit multiply and return the carry in tmp5.
 7704  * Only quad int aligned length of in[] is operated on in this function.
 7705  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 7706  * This function preserves out, in and k registers.
 7707  * len and offset point to the appropriate index in "in" & "out" correspondingly
 7708  * tmp5 has the carry.
 7709  * other registers are temporary and are modified.
 7710  *
 7711  */
 7712 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 7713   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 7714   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7715 
 7716   Label L_first_loop, L_first_loop_exit;
 7717 
 7718   movl(tmp1, len);
 7719   shrl(tmp1, 2);
 7720 
 7721   bind(L_first_loop);
 7722   subl(tmp1, 1);
 7723   jccb(Assembler::negative, L_first_loop_exit);
 7724 
 7725   subl(len, 4);
 7726   subl(offset, 4);
 7727 
 7728   Register op2 = tmp2;
 7729   const Register sum = tmp3;
 7730   const Register op1 = tmp4;
 7731   const Register carry = tmp5;
 7732 
 7733   if (UseBMI2Instructions) {
 7734     op2 = rdxReg;
 7735   }
 7736 
 7737   movq(op1, Address(in, len, Address::times_4,  8));
 7738   rorq(op1, 32);
 7739   movq(sum, Address(out, offset, Address::times_4,  8));
 7740   rorq(sum, 32);
 7741   if (UseBMI2Instructions) {
 7742     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7743   }
 7744   else {
 7745     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7746   }
 7747   // Store back in big endian from little endian
 7748   rorq(sum, 0x20);
 7749   movq(Address(out, offset, Address::times_4,  8), sum);
 7750 
 7751   movq(op1, Address(in, len, Address::times_4,  0));
 7752   rorq(op1, 32);
 7753   movq(sum, Address(out, offset, Address::times_4,  0));
 7754   rorq(sum, 32);
 7755   if (UseBMI2Instructions) {
 7756     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7757   }
 7758   else {
 7759     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7760   }
 7761   // Store back in big endian from little endian
 7762   rorq(sum, 0x20);
 7763   movq(Address(out, offset, Address::times_4,  0), sum);
 7764 
 7765   jmp(L_first_loop);
 7766   bind(L_first_loop_exit);
 7767 }
 7768 
 7769 /**
 7770  * Code for BigInteger::mulAdd() intrinsic
 7771  *
 7772  * rdi: out
 7773  * rsi: in
 7774  * r11: offs (out.length - offset)
 7775  * rcx: len
 7776  * r8:  k
 7777  * r12: tmp1
 7778  * r13: tmp2
 7779  * r14: tmp3
 7780  * r15: tmp4
 7781  * rbx: tmp5
 7782  * Multiply the in[] by word k and add to out[], return the carry in rax
 7783  */
 7784 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 7785    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 7786    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7787 
 7788   Label L_carry, L_last_in, L_done;
 7789 
 7790 // carry = 0;
 7791 // for (int j=len-1; j >= 0; j--) {
 7792 //    long product = (in[j] & LONG_MASK) * kLong +
 7793 //                   (out[offs] & LONG_MASK) + carry;
 7794 //    out[offs--] = (int)product;
 7795 //    carry = product >>> 32;
 7796 // }
 7797 //
 7798   push(tmp1);
 7799   push(tmp2);
 7800   push(tmp3);
 7801   push(tmp4);
 7802   push(tmp5);
 7803 
 7804   Register op2 = tmp2;
 7805   const Register sum = tmp3;
 7806   const Register op1 = tmp4;
 7807   const Register carry =  tmp5;
 7808 
 7809   if (UseBMI2Instructions) {
 7810     op2 = rdxReg;
 7811     movl(op2, k);
 7812   }
 7813   else {
 7814     movl(op2, k);
 7815   }
 7816 
 7817   xorq(carry, carry);
 7818 
 7819   //First loop
 7820 
 7821   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 7822   //The carry is in tmp5
 7823   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7824 
 7825   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 7826   decrementl(len);
 7827   jccb(Assembler::negative, L_carry);
 7828   decrementl(len);
 7829   jccb(Assembler::negative, L_last_in);
 7830 
 7831   movq(op1, Address(in, len, Address::times_4,  0));
 7832   rorq(op1, 32);
 7833 
 7834   subl(offs, 2);
 7835   movq(sum, Address(out, offs, Address::times_4,  0));
 7836   rorq(sum, 32);
 7837 
 7838   if (UseBMI2Instructions) {
 7839     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7840   }
 7841   else {
 7842     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7843   }
 7844 
 7845   // Store back in big endian from little endian
 7846   rorq(sum, 0x20);
 7847   movq(Address(out, offs, Address::times_4,  0), sum);
 7848 
 7849   testl(len, len);
 7850   jccb(Assembler::zero, L_carry);
 7851 
 7852   //Multiply the last in[] entry, if any
 7853   bind(L_last_in);
 7854   movl(op1, Address(in, 0));
 7855   movl(sum, Address(out, offs, Address::times_4,  -4));
 7856 
 7857   movl(raxReg, k);
 7858   mull(op1); //tmp4 * eax -> edx:eax
 7859   addl(sum, carry);
 7860   adcl(rdxReg, 0);
 7861   addl(sum, raxReg);
 7862   adcl(rdxReg, 0);
 7863   movl(carry, rdxReg);
 7864 
 7865   movl(Address(out, offs, Address::times_4,  -4), sum);
 7866 
 7867   bind(L_carry);
 7868   //return tmp5/carry as carry in rax
 7869   movl(rax, carry);
 7870 
 7871   bind(L_done);
 7872   pop(tmp5);
 7873   pop(tmp4);
 7874   pop(tmp3);
 7875   pop(tmp2);
 7876   pop(tmp1);
 7877 }
 7878 #endif
 7879 
 7880 /**
 7881  * Emits code to update CRC-32 with a byte value according to constants in table
 7882  *
 7883  * @param [in,out]crc   Register containing the crc.
 7884  * @param [in]val       Register containing the byte to fold into the CRC.
 7885  * @param [in]table     Register containing the table of crc constants.
 7886  *
 7887  * uint32_t crc;
 7888  * val = crc_table[(val ^ crc) & 0xFF];
 7889  * crc = val ^ (crc >> 8);
 7890  *
 7891  */
 7892 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 7893   xorl(val, crc);
 7894   andl(val, 0xFF);
 7895   shrl(crc, 8); // unsigned shift
 7896   xorl(crc, Address(table, val, Address::times_4, 0));
 7897 }
 7898 
 7899 /**
 7900  * Fold 128-bit data chunk
 7901  */
 7902 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 7903   if (UseAVX > 0) {
 7904     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 7905     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 7906     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 7907     pxor(xcrc, xtmp);
 7908   } else {
 7909     movdqa(xtmp, xcrc);
 7910     pclmulhdq(xtmp, xK);   // [123:64]
 7911     pclmulldq(xcrc, xK);   // [63:0]
 7912     pxor(xcrc, xtmp);
 7913     movdqu(xtmp, Address(buf, offset));
 7914     pxor(xcrc, xtmp);
 7915   }
 7916 }
 7917 
 7918 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 7919   if (UseAVX > 0) {
 7920     vpclmulhdq(xtmp, xK, xcrc);
 7921     vpclmulldq(xcrc, xK, xcrc);
 7922     pxor(xcrc, xbuf);
 7923     pxor(xcrc, xtmp);
 7924   } else {
 7925     movdqa(xtmp, xcrc);
 7926     pclmulhdq(xtmp, xK);
 7927     pclmulldq(xcrc, xK);
 7928     pxor(xcrc, xbuf);
 7929     pxor(xcrc, xtmp);
 7930   }
 7931 }
 7932 
 7933 /**
 7934  * 8-bit folds to compute 32-bit CRC
 7935  *
 7936  * uint64_t xcrc;
 7937  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 7938  */
 7939 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 7940   movdl(tmp, xcrc);
 7941   andl(tmp, 0xFF);
 7942   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 7943   psrldq(xcrc, 1); // unsigned shift one byte
 7944   pxor(xcrc, xtmp);
 7945 }
 7946 
 7947 /**
 7948  * uint32_t crc;
 7949  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 7950  */
 7951 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 7952   movl(tmp, crc);
 7953   andl(tmp, 0xFF);
 7954   shrl(crc, 8);
 7955   xorl(crc, Address(table, tmp, Address::times_4, 0));
 7956 }
 7957 
 7958 /**
 7959  * @param crc   register containing existing CRC (32-bit)
 7960  * @param buf   register pointing to input byte buffer (byte*)
 7961  * @param len   register containing number of bytes
 7962  * @param table register that will contain address of CRC table
 7963  * @param tmp   scratch register
 7964  */
 7965 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 7966   assert_different_registers(crc, buf, len, table, tmp, rax);
 7967 
 7968   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 7969   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 7970 
 7971   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 7972   // context for the registers used, where all instructions below are using 128-bit mode
 7973   // On EVEX without VL and BW, these instructions will all be AVX.
 7974   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 7975   notl(crc); // ~crc
 7976   cmpl(len, 16);
 7977   jcc(Assembler::less, L_tail);
 7978 
 7979   // Align buffer to 16 bytes
 7980   movl(tmp, buf);
 7981   andl(tmp, 0xF);
 7982   jccb(Assembler::zero, L_aligned);
 7983   subl(tmp,  16);
 7984   addl(len, tmp);
 7985 
 7986   align(4);
 7987   BIND(L_align_loop);
 7988   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 7989   update_byte_crc32(crc, rax, table);
 7990   increment(buf);
 7991   incrementl(tmp);
 7992   jccb(Assembler::less, L_align_loop);
 7993 
 7994   BIND(L_aligned);
 7995   movl(tmp, len); // save
 7996   shrl(len, 4);
 7997   jcc(Assembler::zero, L_tail_restore);
 7998 
 7999   // Fold crc into first bytes of vector
 8000   movdqa(xmm1, Address(buf, 0));
 8001   movdl(rax, xmm1);
 8002   xorl(crc, rax);
 8003   if (VM_Version::supports_sse4_1()) {
 8004     pinsrd(xmm1, crc, 0);
 8005   } else {
 8006     pinsrw(xmm1, crc, 0);
 8007     shrl(crc, 16);
 8008     pinsrw(xmm1, crc, 1);
 8009   }
 8010   addptr(buf, 16);
 8011   subl(len, 4); // len > 0
 8012   jcc(Assembler::less, L_fold_tail);
 8013 
 8014   movdqa(xmm2, Address(buf,  0));
 8015   movdqa(xmm3, Address(buf, 16));
 8016   movdqa(xmm4, Address(buf, 32));
 8017   addptr(buf, 48);
 8018   subl(len, 3);
 8019   jcc(Assembler::lessEqual, L_fold_512b);
 8020 
 8021   // Fold total 512 bits of polynomial on each iteration,
 8022   // 128 bits per each of 4 parallel streams.
 8023   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8024 
 8025   align32();
 8026   BIND(L_fold_512b_loop);
 8027   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8028   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8029   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8030   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8031   addptr(buf, 64);
 8032   subl(len, 4);
 8033   jcc(Assembler::greater, L_fold_512b_loop);
 8034 
 8035   // Fold 512 bits to 128 bits.
 8036   BIND(L_fold_512b);
 8037   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8038   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8039   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8040   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8041 
 8042   // Fold the rest of 128 bits data chunks
 8043   BIND(L_fold_tail);
 8044   addl(len, 3);
 8045   jccb(Assembler::lessEqual, L_fold_128b);
 8046   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8047 
 8048   BIND(L_fold_tail_loop);
 8049   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8050   addptr(buf, 16);
 8051   decrementl(len);
 8052   jccb(Assembler::greater, L_fold_tail_loop);
 8053 
 8054   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8055   BIND(L_fold_128b);
 8056   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8057   if (UseAVX > 0) {
 8058     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8059     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8060     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8061   } else {
 8062     movdqa(xmm2, xmm0);
 8063     pclmulqdq(xmm2, xmm1, 0x1);
 8064     movdqa(xmm3, xmm0);
 8065     pand(xmm3, xmm2);
 8066     pclmulqdq(xmm0, xmm3, 0x1);
 8067   }
 8068   psrldq(xmm1, 8);
 8069   psrldq(xmm2, 4);
 8070   pxor(xmm0, xmm1);
 8071   pxor(xmm0, xmm2);
 8072 
 8073   // 8 8-bit folds to compute 32-bit CRC.
 8074   for (int j = 0; j < 4; j++) {
 8075     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8076   }
 8077   movdl(crc, xmm0); // mov 32 bits to general register
 8078   for (int j = 0; j < 4; j++) {
 8079     fold_8bit_crc32(crc, table, rax);
 8080   }
 8081 
 8082   BIND(L_tail_restore);
 8083   movl(len, tmp); // restore
 8084   BIND(L_tail);
 8085   andl(len, 0xf);
 8086   jccb(Assembler::zero, L_exit);
 8087 
 8088   // Fold the rest of bytes
 8089   align(4);
 8090   BIND(L_tail_loop);
 8091   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8092   update_byte_crc32(crc, rax, table);
 8093   increment(buf);
 8094   decrementl(len);
 8095   jccb(Assembler::greater, L_tail_loop);
 8096 
 8097   BIND(L_exit);
 8098   notl(crc); // ~c
 8099 }
 8100 
 8101 #ifdef _LP64
 8102 // Helper function for AVX 512 CRC32
 8103 // Fold 512-bit data chunks
 8104 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8105                                              Register pos, int offset) {
 8106   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8107   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8108   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8109   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8110   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8111 }
 8112 
 8113 // Helper function for AVX 512 CRC32
 8114 // Compute CRC32 for < 256B buffers
 8115 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8116                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8117                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8118 
 8119   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8120   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8121   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8122 
 8123   // check if there is enough buffer to be able to fold 16B at a time
 8124   cmpl(len, 32);
 8125   jcc(Assembler::less, L_less_than_32);
 8126 
 8127   // if there is, load the constants
 8128   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8129   movdl(xmm0, crc);                        // get the initial crc value
 8130   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8131   pxor(xmm7, xmm0);
 8132 
 8133   // update the buffer pointer
 8134   addl(pos, 16);
 8135   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8136   subl(len, 32);
 8137   jmp(L_16B_reduction_loop);
 8138 
 8139   bind(L_less_than_32);
 8140   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8141   movl(rax, crc);
 8142   testl(len, len);
 8143   jcc(Assembler::equal, L_cleanup);
 8144 
 8145   movdl(xmm0, crc);                        //get the initial crc value
 8146 
 8147   cmpl(len, 16);
 8148   jcc(Assembler::equal, L_exact_16_left);
 8149   jcc(Assembler::less, L_less_than_16_left);
 8150 
 8151   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8152   pxor(xmm7, xmm0);                       //xor the initial crc value
 8153   addl(pos, 16);
 8154   subl(len, 16);
 8155   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8156   jmp(L_get_last_two_xmms);
 8157 
 8158   bind(L_less_than_16_left);
 8159   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8160   pxor(xmm1, xmm1);
 8161   movptr(tmp1, rsp);
 8162   movdqu(Address(tmp1, 0 * 16), xmm1);
 8163 
 8164   cmpl(len, 4);
 8165   jcc(Assembler::less, L_only_less_than_4);
 8166 
 8167   //backup the counter value
 8168   movl(tmp2, len);
 8169   cmpl(len, 8);
 8170   jcc(Assembler::less, L_less_than_8_left);
 8171 
 8172   //load 8 Bytes
 8173   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8174   movq(Address(tmp1, 0 * 16), rax);
 8175   addptr(tmp1, 8);
 8176   subl(len, 8);
 8177   addl(pos, 8);
 8178 
 8179   bind(L_less_than_8_left);
 8180   cmpl(len, 4);
 8181   jcc(Assembler::less, L_less_than_4_left);
 8182 
 8183   //load 4 Bytes
 8184   movl(rax, Address(buf, pos, Address::times_1, 0));
 8185   movl(Address(tmp1, 0 * 16), rax);
 8186   addptr(tmp1, 4);
 8187   subl(len, 4);
 8188   addl(pos, 4);
 8189 
 8190   bind(L_less_than_4_left);
 8191   cmpl(len, 2);
 8192   jcc(Assembler::less, L_less_than_2_left);
 8193 
 8194   // load 2 Bytes
 8195   movw(rax, Address(buf, pos, Address::times_1, 0));
 8196   movl(Address(tmp1, 0 * 16), rax);
 8197   addptr(tmp1, 2);
 8198   subl(len, 2);
 8199   addl(pos, 2);
 8200 
 8201   bind(L_less_than_2_left);
 8202   cmpl(len, 1);
 8203   jcc(Assembler::less, L_zero_left);
 8204 
 8205   // load 1 Byte
 8206   movb(rax, Address(buf, pos, Address::times_1, 0));
 8207   movb(Address(tmp1, 0 * 16), rax);
 8208 
 8209   bind(L_zero_left);
 8210   movdqu(xmm7, Address(rsp, 0));
 8211   pxor(xmm7, xmm0);                       //xor the initial crc value
 8212 
 8213   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8214   movdqu(xmm0, Address(rax, tmp2));
 8215   pshufb(xmm7, xmm0);
 8216   jmp(L_128_done);
 8217 
 8218   bind(L_exact_16_left);
 8219   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8220   pxor(xmm7, xmm0);                       //xor the initial crc value
 8221   jmp(L_128_done);
 8222 
 8223   bind(L_only_less_than_4);
 8224   cmpl(len, 3);
 8225   jcc(Assembler::less, L_only_less_than_3);
 8226 
 8227   // load 3 Bytes
 8228   movb(rax, Address(buf, pos, Address::times_1, 0));
 8229   movb(Address(tmp1, 0), rax);
 8230 
 8231   movb(rax, Address(buf, pos, Address::times_1, 1));
 8232   movb(Address(tmp1, 1), rax);
 8233 
 8234   movb(rax, Address(buf, pos, Address::times_1, 2));
 8235   movb(Address(tmp1, 2), rax);
 8236 
 8237   movdqu(xmm7, Address(rsp, 0));
 8238   pxor(xmm7, xmm0);                     //xor the initial crc value
 8239 
 8240   pslldq(xmm7, 0x5);
 8241   jmp(L_barrett);
 8242   bind(L_only_less_than_3);
 8243   cmpl(len, 2);
 8244   jcc(Assembler::less, L_only_less_than_2);
 8245 
 8246   // load 2 Bytes
 8247   movb(rax, Address(buf, pos, Address::times_1, 0));
 8248   movb(Address(tmp1, 0), rax);
 8249 
 8250   movb(rax, Address(buf, pos, Address::times_1, 1));
 8251   movb(Address(tmp1, 1), rax);
 8252 
 8253   movdqu(xmm7, Address(rsp, 0));
 8254   pxor(xmm7, xmm0);                     //xor the initial crc value
 8255 
 8256   pslldq(xmm7, 0x6);
 8257   jmp(L_barrett);
 8258 
 8259   bind(L_only_less_than_2);
 8260   //load 1 Byte
 8261   movb(rax, Address(buf, pos, Address::times_1, 0));
 8262   movb(Address(tmp1, 0), rax);
 8263 
 8264   movdqu(xmm7, Address(rsp, 0));
 8265   pxor(xmm7, xmm0);                     //xor the initial crc value
 8266 
 8267   pslldq(xmm7, 0x7);
 8268 }
 8269 
 8270 /**
 8271 * Compute CRC32 using AVX512 instructions
 8272 * param crc   register containing existing CRC (32-bit)
 8273 * param buf   register pointing to input byte buffer (byte*)
 8274 * param len   register containing number of bytes
 8275 * param table address of crc or crc32c table
 8276 * param tmp1  scratch register
 8277 * param tmp2  scratch register
 8278 * return rax  result register
 8279 *
 8280 * This routine is identical for crc32c with the exception of the precomputed constant
 8281 * table which will be passed as the table argument.  The calculation steps are
 8282 * the same for both variants.
 8283 */
 8284 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8285   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8286 
 8287   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8288   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8289   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8290   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8291   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8292 
 8293   const Register pos = r12;
 8294   push(r12);
 8295   subptr(rsp, 16 * 2 + 8);
 8296 
 8297   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8298   // context for the registers used, where all instructions below are using 128-bit mode
 8299   // On EVEX without VL and BW, these instructions will all be AVX.
 8300   movl(pos, 0);
 8301 
 8302   // check if smaller than 256B
 8303   cmpl(len, 256);
 8304   jcc(Assembler::less, L_less_than_256);
 8305 
 8306   // load the initial crc value
 8307   movdl(xmm10, crc);
 8308 
 8309   // receive the initial 64B data, xor the initial crc value
 8310   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8311   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8312   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8313   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8314 
 8315   subl(len, 256);
 8316   cmpl(len, 256);
 8317   jcc(Assembler::less, L_fold_128_B_loop);
 8318 
 8319   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8320   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8321   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8322   subl(len, 256);
 8323 
 8324   bind(L_fold_256_B_loop);
 8325   addl(pos, 256);
 8326   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8327   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8328   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8329   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8330 
 8331   subl(len, 256);
 8332   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8333 
 8334   // Fold 256 into 128
 8335   addl(pos, 256);
 8336   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8337   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8338   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8339 
 8340   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8341   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8342   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8343 
 8344   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8345   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8346 
 8347   addl(len, 128);
 8348   jmp(L_fold_128_B_register);
 8349 
 8350   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8351   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8352 
 8353   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8354   bind(L_fold_128_B_loop);
 8355   addl(pos, 128);
 8356   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8357   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8358 
 8359   subl(len, 128);
 8360   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8361 
 8362   addl(pos, 128);
 8363 
 8364   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8365   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8366   bind(L_fold_128_B_register);
 8367   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8368   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8369   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8370   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8371   // save last that has no multiplicand
 8372   vextracti64x2(xmm7, xmm4, 3);
 8373 
 8374   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8375   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8376   // Needed later in reduction loop
 8377   movdqu(xmm10, Address(table, 1 * 16));
 8378   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8379   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8380 
 8381   // Swap 1,0,3,2 - 01 00 11 10
 8382   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8383   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8384   vextracti128(xmm5, xmm8, 1);
 8385   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8386 
 8387   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8388   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8389   addl(len, 128 - 16);
 8390   jcc(Assembler::less, L_final_reduction_for_128);
 8391 
 8392   bind(L_16B_reduction_loop);
 8393   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8394   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8395   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8396   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8397   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8398   addl(pos, 16);
 8399   subl(len, 16);
 8400   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8401 
 8402   bind(L_final_reduction_for_128);
 8403   addl(len, 16);
 8404   jcc(Assembler::equal, L_128_done);
 8405 
 8406   bind(L_get_last_two_xmms);
 8407   movdqu(xmm2, xmm7);
 8408   addl(pos, len);
 8409   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8410   subl(pos, len);
 8411 
 8412   // get rid of the extra data that was loaded before
 8413   // load the shift constant
 8414   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8415   movdqu(xmm0, Address(rax, len));
 8416   addl(rax, len);
 8417 
 8418   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8419   //Change mask to 512
 8420   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8421   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8422 
 8423   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8424   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8425   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8426   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8427   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8428 
 8429   bind(L_128_done);
 8430   // compute crc of a 128-bit value
 8431   movdqu(xmm10, Address(table, 3 * 16));
 8432   movdqu(xmm0, xmm7);
 8433 
 8434   // 64b fold
 8435   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8436   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8437   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8438 
 8439   // 32b fold
 8440   movdqu(xmm0, xmm7);
 8441   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8442   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8443   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8444   jmp(L_barrett);
 8445 
 8446   bind(L_less_than_256);
 8447   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8448 
 8449   //barrett reduction
 8450   bind(L_barrett);
 8451   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8452   movdqu(xmm1, xmm7);
 8453   movdqu(xmm2, xmm7);
 8454   movdqu(xmm10, Address(table, 4 * 16));
 8455 
 8456   pclmulqdq(xmm7, xmm10, 0x0);
 8457   pxor(xmm7, xmm2);
 8458   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8459   movdqu(xmm2, xmm7);
 8460   pclmulqdq(xmm7, xmm10, 0x10);
 8461   pxor(xmm7, xmm2);
 8462   pxor(xmm7, xmm1);
 8463   pextrd(crc, xmm7, 2);
 8464 
 8465   bind(L_cleanup);
 8466   addptr(rsp, 16 * 2 + 8);
 8467   pop(r12);
 8468 }
 8469 
 8470 // S. Gueron / Information Processing Letters 112 (2012) 184
 8471 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8472 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8473 // Output: the 64-bit carry-less product of B * CONST
 8474 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8475                                      Register tmp1, Register tmp2, Register tmp3) {
 8476   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8477   if (n > 0) {
 8478     addq(tmp3, n * 256 * 8);
 8479   }
 8480   //    Q1 = TABLEExt[n][B & 0xFF];
 8481   movl(tmp1, in);
 8482   andl(tmp1, 0x000000FF);
 8483   shll(tmp1, 3);
 8484   addq(tmp1, tmp3);
 8485   movq(tmp1, Address(tmp1, 0));
 8486 
 8487   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8488   movl(tmp2, in);
 8489   shrl(tmp2, 8);
 8490   andl(tmp2, 0x000000FF);
 8491   shll(tmp2, 3);
 8492   addq(tmp2, tmp3);
 8493   movq(tmp2, Address(tmp2, 0));
 8494 
 8495   shlq(tmp2, 8);
 8496   xorq(tmp1, tmp2);
 8497 
 8498   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8499   movl(tmp2, in);
 8500   shrl(tmp2, 16);
 8501   andl(tmp2, 0x000000FF);
 8502   shll(tmp2, 3);
 8503   addq(tmp2, tmp3);
 8504   movq(tmp2, Address(tmp2, 0));
 8505 
 8506   shlq(tmp2, 16);
 8507   xorq(tmp1, tmp2);
 8508 
 8509   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8510   shrl(in, 24);
 8511   andl(in, 0x000000FF);
 8512   shll(in, 3);
 8513   addq(in, tmp3);
 8514   movq(in, Address(in, 0));
 8515 
 8516   shlq(in, 24);
 8517   xorq(in, tmp1);
 8518   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8519 }
 8520 
 8521 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8522                                       Register in_out,
 8523                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8524                                       XMMRegister w_xtmp2,
 8525                                       Register tmp1,
 8526                                       Register n_tmp2, Register n_tmp3) {
 8527   if (is_pclmulqdq_supported) {
 8528     movdl(w_xtmp1, in_out); // modified blindly
 8529 
 8530     movl(tmp1, const_or_pre_comp_const_index);
 8531     movdl(w_xtmp2, tmp1);
 8532     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8533 
 8534     movdq(in_out, w_xtmp1);
 8535   } else {
 8536     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8537   }
 8538 }
 8539 
 8540 // Recombination Alternative 2: No bit-reflections
 8541 // T1 = (CRC_A * U1) << 1
 8542 // T2 = (CRC_B * U2) << 1
 8543 // C1 = T1 >> 32
 8544 // C2 = T2 >> 32
 8545 // T1 = T1 & 0xFFFFFFFF
 8546 // T2 = T2 & 0xFFFFFFFF
 8547 // T1 = CRC32(0, T1)
 8548 // T2 = CRC32(0, T2)
 8549 // C1 = C1 ^ T1
 8550 // C2 = C2 ^ T2
 8551 // CRC = C1 ^ C2 ^ CRC_C
 8552 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8553                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8554                                      Register tmp1, Register tmp2,
 8555                                      Register n_tmp3) {
 8556   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8557   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8558   shlq(in_out, 1);
 8559   movl(tmp1, in_out);
 8560   shrq(in_out, 32);
 8561   xorl(tmp2, tmp2);
 8562   crc32(tmp2, tmp1, 4);
 8563   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8564   shlq(in1, 1);
 8565   movl(tmp1, in1);
 8566   shrq(in1, 32);
 8567   xorl(tmp2, tmp2);
 8568   crc32(tmp2, tmp1, 4);
 8569   xorl(in1, tmp2);
 8570   xorl(in_out, in1);
 8571   xorl(in_out, in2);
 8572 }
 8573 
 8574 // Set N to predefined value
 8575 // Subtract from a length of a buffer
 8576 // execute in a loop:
 8577 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8578 // for i = 1 to N do
 8579 //  CRC_A = CRC32(CRC_A, A[i])
 8580 //  CRC_B = CRC32(CRC_B, B[i])
 8581 //  CRC_C = CRC32(CRC_C, C[i])
 8582 // end for
 8583 // Recombine
 8584 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8585                                        Register in_out1, Register in_out2, Register in_out3,
 8586                                        Register tmp1, Register tmp2, Register tmp3,
 8587                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8588                                        Register tmp4, Register tmp5,
 8589                                        Register n_tmp6) {
 8590   Label L_processPartitions;
 8591   Label L_processPartition;
 8592   Label L_exit;
 8593 
 8594   bind(L_processPartitions);
 8595   cmpl(in_out1, 3 * size);
 8596   jcc(Assembler::less, L_exit);
 8597     xorl(tmp1, tmp1);
 8598     xorl(tmp2, tmp2);
 8599     movq(tmp3, in_out2);
 8600     addq(tmp3, size);
 8601 
 8602     bind(L_processPartition);
 8603       crc32(in_out3, Address(in_out2, 0), 8);
 8604       crc32(tmp1, Address(in_out2, size), 8);
 8605       crc32(tmp2, Address(in_out2, size * 2), 8);
 8606       addq(in_out2, 8);
 8607       cmpq(in_out2, tmp3);
 8608       jcc(Assembler::less, L_processPartition);
 8609     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8610             w_xtmp1, w_xtmp2, w_xtmp3,
 8611             tmp4, tmp5,
 8612             n_tmp6);
 8613     addq(in_out2, 2 * size);
 8614     subl(in_out1, 3 * size);
 8615     jmp(L_processPartitions);
 8616 
 8617   bind(L_exit);
 8618 }
 8619 #else
 8620 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 8621                                      Register tmp1, Register tmp2, Register tmp3,
 8622                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 8623   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8624   if (n > 0) {
 8625     addl(tmp3, n * 256 * 8);
 8626   }
 8627   //    Q1 = TABLEExt[n][B & 0xFF];
 8628   movl(tmp1, in_out);
 8629   andl(tmp1, 0x000000FF);
 8630   shll(tmp1, 3);
 8631   addl(tmp1, tmp3);
 8632   movq(xtmp1, Address(tmp1, 0));
 8633 
 8634   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8635   movl(tmp2, in_out);
 8636   shrl(tmp2, 8);
 8637   andl(tmp2, 0x000000FF);
 8638   shll(tmp2, 3);
 8639   addl(tmp2, tmp3);
 8640   movq(xtmp2, Address(tmp2, 0));
 8641 
 8642   psllq(xtmp2, 8);
 8643   pxor(xtmp1, xtmp2);
 8644 
 8645   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8646   movl(tmp2, in_out);
 8647   shrl(tmp2, 16);
 8648   andl(tmp2, 0x000000FF);
 8649   shll(tmp2, 3);
 8650   addl(tmp2, tmp3);
 8651   movq(xtmp2, Address(tmp2, 0));
 8652 
 8653   psllq(xtmp2, 16);
 8654   pxor(xtmp1, xtmp2);
 8655 
 8656   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8657   shrl(in_out, 24);
 8658   andl(in_out, 0x000000FF);
 8659   shll(in_out, 3);
 8660   addl(in_out, tmp3);
 8661   movq(xtmp2, Address(in_out, 0));
 8662 
 8663   psllq(xtmp2, 24);
 8664   pxor(xtmp1, xtmp2); // Result in CXMM
 8665   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8666 }
 8667 
 8668 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8669                                       Register in_out,
 8670                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8671                                       XMMRegister w_xtmp2,
 8672                                       Register tmp1,
 8673                                       Register n_tmp2, Register n_tmp3) {
 8674   if (is_pclmulqdq_supported) {
 8675     movdl(w_xtmp1, in_out);
 8676 
 8677     movl(tmp1, const_or_pre_comp_const_index);
 8678     movdl(w_xtmp2, tmp1);
 8679     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8680     // Keep result in XMM since GPR is 32 bit in length
 8681   } else {
 8682     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 8683   }
 8684 }
 8685 
 8686 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8687                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8688                                      Register tmp1, Register tmp2,
 8689                                      Register n_tmp3) {
 8690   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8691   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8692 
 8693   psllq(w_xtmp1, 1);
 8694   movdl(tmp1, w_xtmp1);
 8695   psrlq(w_xtmp1, 32);
 8696   movdl(in_out, w_xtmp1);
 8697 
 8698   xorl(tmp2, tmp2);
 8699   crc32(tmp2, tmp1, 4);
 8700   xorl(in_out, tmp2);
 8701 
 8702   psllq(w_xtmp2, 1);
 8703   movdl(tmp1, w_xtmp2);
 8704   psrlq(w_xtmp2, 32);
 8705   movdl(in1, w_xtmp2);
 8706 
 8707   xorl(tmp2, tmp2);
 8708   crc32(tmp2, tmp1, 4);
 8709   xorl(in1, tmp2);
 8710   xorl(in_out, in1);
 8711   xorl(in_out, in2);
 8712 }
 8713 
 8714 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8715                                        Register in_out1, Register in_out2, Register in_out3,
 8716                                        Register tmp1, Register tmp2, Register tmp3,
 8717                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8718                                        Register tmp4, Register tmp5,
 8719                                        Register n_tmp6) {
 8720   Label L_processPartitions;
 8721   Label L_processPartition;
 8722   Label L_exit;
 8723 
 8724   bind(L_processPartitions);
 8725   cmpl(in_out1, 3 * size);
 8726   jcc(Assembler::less, L_exit);
 8727     xorl(tmp1, tmp1);
 8728     xorl(tmp2, tmp2);
 8729     movl(tmp3, in_out2);
 8730     addl(tmp3, size);
 8731 
 8732     bind(L_processPartition);
 8733       crc32(in_out3, Address(in_out2, 0), 4);
 8734       crc32(tmp1, Address(in_out2, size), 4);
 8735       crc32(tmp2, Address(in_out2, size*2), 4);
 8736       crc32(in_out3, Address(in_out2, 0+4), 4);
 8737       crc32(tmp1, Address(in_out2, size+4), 4);
 8738       crc32(tmp2, Address(in_out2, size*2+4), 4);
 8739       addl(in_out2, 8);
 8740       cmpl(in_out2, tmp3);
 8741       jcc(Assembler::less, L_processPartition);
 8742 
 8743         push(tmp3);
 8744         push(in_out1);
 8745         push(in_out2);
 8746         tmp4 = tmp3;
 8747         tmp5 = in_out1;
 8748         n_tmp6 = in_out2;
 8749 
 8750       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8751             w_xtmp1, w_xtmp2, w_xtmp3,
 8752             tmp4, tmp5,
 8753             n_tmp6);
 8754 
 8755         pop(in_out2);
 8756         pop(in_out1);
 8757         pop(tmp3);
 8758 
 8759     addl(in_out2, 2 * size);
 8760     subl(in_out1, 3 * size);
 8761     jmp(L_processPartitions);
 8762 
 8763   bind(L_exit);
 8764 }
 8765 #endif //LP64
 8766 
 8767 #ifdef _LP64
 8768 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 8769 // Input: A buffer I of L bytes.
 8770 // Output: the CRC32C value of the buffer.
 8771 // Notations:
 8772 // Write L = 24N + r, with N = floor (L/24).
 8773 // r = L mod 24 (0 <= r < 24).
 8774 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 8775 // N quadwords, and R consists of r bytes.
 8776 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 8777 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 8778 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 8779 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 8780 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8781                                           Register tmp1, Register tmp2, Register tmp3,
 8782                                           Register tmp4, Register tmp5, Register tmp6,
 8783                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8784                                           bool is_pclmulqdq_supported) {
 8785   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8786   Label L_wordByWord;
 8787   Label L_byteByByteProlog;
 8788   Label L_byteByByte;
 8789   Label L_exit;
 8790 
 8791   if (is_pclmulqdq_supported ) {
 8792     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8793     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
 8794 
 8795     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8796     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8797 
 8798     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8799     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8800     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 8801   } else {
 8802     const_or_pre_comp_const_index[0] = 1;
 8803     const_or_pre_comp_const_index[1] = 0;
 8804 
 8805     const_or_pre_comp_const_index[2] = 3;
 8806     const_or_pre_comp_const_index[3] = 2;
 8807 
 8808     const_or_pre_comp_const_index[4] = 5;
 8809     const_or_pre_comp_const_index[5] = 4;
 8810    }
 8811   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8812                     in2, in1, in_out,
 8813                     tmp1, tmp2, tmp3,
 8814                     w_xtmp1, w_xtmp2, w_xtmp3,
 8815                     tmp4, tmp5,
 8816                     tmp6);
 8817   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8818                     in2, in1, in_out,
 8819                     tmp1, tmp2, tmp3,
 8820                     w_xtmp1, w_xtmp2, w_xtmp3,
 8821                     tmp4, tmp5,
 8822                     tmp6);
 8823   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8824                     in2, in1, in_out,
 8825                     tmp1, tmp2, tmp3,
 8826                     w_xtmp1, w_xtmp2, w_xtmp3,
 8827                     tmp4, tmp5,
 8828                     tmp6);
 8829   movl(tmp1, in2);
 8830   andl(tmp1, 0x00000007);
 8831   negl(tmp1);
 8832   addl(tmp1, in2);
 8833   addq(tmp1, in1);
 8834 
 8835   cmpq(in1, tmp1);
 8836   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 8837   align(16);
 8838   BIND(L_wordByWord);
 8839     crc32(in_out, Address(in1, 0), 8);
 8840     addq(in1, 8);
 8841     cmpq(in1, tmp1);
 8842     jcc(Assembler::less, L_wordByWord);
 8843 
 8844   BIND(L_byteByByteProlog);
 8845   andl(in2, 0x00000007);
 8846   movl(tmp2, 1);
 8847 
 8848   cmpl(tmp2, in2);
 8849   jccb(Assembler::greater, L_exit);
 8850   BIND(L_byteByByte);
 8851     crc32(in_out, Address(in1, 0), 1);
 8852     incq(in1);
 8853     incl(tmp2);
 8854     cmpl(tmp2, in2);
 8855     jcc(Assembler::lessEqual, L_byteByByte);
 8856 
 8857   BIND(L_exit);
 8858 }
 8859 #else
 8860 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8861                                           Register tmp1, Register  tmp2, Register tmp3,
 8862                                           Register tmp4, Register  tmp5, Register tmp6,
 8863                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8864                                           bool is_pclmulqdq_supported) {
 8865   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8866   Label L_wordByWord;
 8867   Label L_byteByByteProlog;
 8868   Label L_byteByByte;
 8869   Label L_exit;
 8870 
 8871   if (is_pclmulqdq_supported) {
 8872     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8873     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
 8874 
 8875     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8876     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8877 
 8878     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8879     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8880   } else {
 8881     const_or_pre_comp_const_index[0] = 1;
 8882     const_or_pre_comp_const_index[1] = 0;
 8883 
 8884     const_or_pre_comp_const_index[2] = 3;
 8885     const_or_pre_comp_const_index[3] = 2;
 8886 
 8887     const_or_pre_comp_const_index[4] = 5;
 8888     const_or_pre_comp_const_index[5] = 4;
 8889   }
 8890   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8891                     in2, in1, in_out,
 8892                     tmp1, tmp2, tmp3,
 8893                     w_xtmp1, w_xtmp2, w_xtmp3,
 8894                     tmp4, tmp5,
 8895                     tmp6);
 8896   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8897                     in2, in1, in_out,
 8898                     tmp1, tmp2, tmp3,
 8899                     w_xtmp1, w_xtmp2, w_xtmp3,
 8900                     tmp4, tmp5,
 8901                     tmp6);
 8902   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8903                     in2, in1, in_out,
 8904                     tmp1, tmp2, tmp3,
 8905                     w_xtmp1, w_xtmp2, w_xtmp3,
 8906                     tmp4, tmp5,
 8907                     tmp6);
 8908   movl(tmp1, in2);
 8909   andl(tmp1, 0x00000007);
 8910   negl(tmp1);
 8911   addl(tmp1, in2);
 8912   addl(tmp1, in1);
 8913 
 8914   BIND(L_wordByWord);
 8915   cmpl(in1, tmp1);
 8916   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 8917     crc32(in_out, Address(in1,0), 4);
 8918     addl(in1, 4);
 8919     jmp(L_wordByWord);
 8920 
 8921   BIND(L_byteByByteProlog);
 8922   andl(in2, 0x00000007);
 8923   movl(tmp2, 1);
 8924 
 8925   BIND(L_byteByByte);
 8926   cmpl(tmp2, in2);
 8927   jccb(Assembler::greater, L_exit);
 8928     movb(tmp1, Address(in1, 0));
 8929     crc32(in_out, tmp1, 1);
 8930     incl(in1);
 8931     incl(tmp2);
 8932     jmp(L_byteByByte);
 8933 
 8934   BIND(L_exit);
 8935 }
 8936 #endif // LP64
 8937 #undef BIND
 8938 #undef BLOCK_COMMENT
 8939 
 8940 // Compress char[] array to byte[].
 8941 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
 8942 // Return the array length if every element in array can be encoded,
 8943 // otherwise, the index of first non-latin1 (> 0xff) character.
 8944 //   @IntrinsicCandidate
 8945 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 8946 //     for (int i = 0; i < len; i++) {
 8947 //       char c = src[srcOff];
 8948 //       if (c > 0xff) {
 8949 //           return i;  // return index of non-latin1 char
 8950 //       }
 8951 //       dst[dstOff] = (byte)c;
 8952 //       srcOff++;
 8953 //       dstOff++;
 8954 //     }
 8955 //     return len;
 8956 //   }
 8957 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 8958   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 8959   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 8960   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 8961   Label copy_chars_loop, done, reset_sp, copy_tail;
 8962 
 8963   // rsi: src
 8964   // rdi: dst
 8965   // rdx: len
 8966   // rcx: tmp5
 8967   // rax: result
 8968 
 8969   // rsi holds start addr of source char[] to be compressed
 8970   // rdi holds start addr of destination byte[]
 8971   // rdx holds length
 8972 
 8973   assert(len != result, "");
 8974 
 8975   // save length for return
 8976   movl(result, len);
 8977 
 8978   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 8979     VM_Version::supports_avx512vlbw() &&
 8980     VM_Version::supports_bmi2()) {
 8981 
 8982     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
 8983 
 8984     // alignment
 8985     Label post_alignment;
 8986 
 8987     // if length of the string is less than 32, handle it the old fashioned way
 8988     testl(len, -32);
 8989     jcc(Assembler::zero, below_threshold);
 8990 
 8991     // First check whether a character is compressible ( <= 0xFF).
 8992     // Create mask to test for Unicode chars inside zmm vector
 8993     movl(tmp5, 0x00FF);
 8994     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
 8995 
 8996     testl(len, -64);
 8997     jccb(Assembler::zero, post_alignment);
 8998 
 8999     movl(tmp5, dst);
 9000     andl(tmp5, (32 - 1));
 9001     negl(tmp5);
 9002     andl(tmp5, (32 - 1));
 9003 
 9004     // bail out when there is nothing to be done
 9005     testl(tmp5, 0xFFFFFFFF);
 9006     jccb(Assembler::zero, post_alignment);
 9007 
 9008     // ~(~0 << len), where len is the # of remaining elements to process
 9009     movl(len, 0xFFFFFFFF);
 9010     shlxl(len, len, tmp5);
 9011     notl(len);
 9012     kmovdl(mask2, len);
 9013     movl(len, result);
 9014 
 9015     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9016     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9017     ktestd(mask1, mask2);
 9018     jcc(Assembler::carryClear, copy_tail);
 9019 
 9020     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9021 
 9022     addptr(src, tmp5);
 9023     addptr(src, tmp5);
 9024     addptr(dst, tmp5);
 9025     subl(len, tmp5);
 9026 
 9027     bind(post_alignment);
 9028     // end of alignment
 9029 
 9030     movl(tmp5, len);
 9031     andl(tmp5, (32 - 1));    // tail count (in chars)
 9032     andl(len, ~(32 - 1));    // vector count (in chars)
 9033     jccb(Assembler::zero, copy_loop_tail);
 9034 
 9035     lea(src, Address(src, len, Address::times_2));
 9036     lea(dst, Address(dst, len, Address::times_1));
 9037     negptr(len);
 9038 
 9039     bind(copy_32_loop);
 9040     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9041     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9042     kortestdl(mask1, mask1);
 9043     jccb(Assembler::carryClear, reset_for_copy_tail);
 9044 
 9045     // All elements in current processed chunk are valid candidates for
 9046     // compression. Write a truncated byte elements to the memory.
 9047     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9048     addptr(len, 32);
 9049     jccb(Assembler::notZero, copy_32_loop);
 9050 
 9051     bind(copy_loop_tail);
 9052     // bail out when there is nothing to be done
 9053     testl(tmp5, 0xFFFFFFFF);
 9054     jcc(Assembler::zero, done);
 9055 
 9056     movl(len, tmp5);
 9057 
 9058     // ~(~0 << len), where len is the # of remaining elements to process
 9059     movl(tmp5, 0xFFFFFFFF);
 9060     shlxl(tmp5, tmp5, len);
 9061     notl(tmp5);
 9062 
 9063     kmovdl(mask2, tmp5);
 9064 
 9065     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9066     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9067     ktestd(mask1, mask2);
 9068     jcc(Assembler::carryClear, copy_tail);
 9069 
 9070     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9071     jmp(done);
 9072 
 9073     bind(reset_for_copy_tail);
 9074     lea(src, Address(src, tmp5, Address::times_2));
 9075     lea(dst, Address(dst, tmp5, Address::times_1));
 9076     subptr(len, tmp5);
 9077     jmp(copy_chars_loop);
 9078 
 9079     bind(below_threshold);
 9080   }
 9081 
 9082   if (UseSSE42Intrinsics) {
 9083     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
 9084 
 9085     // vectored compression
 9086     testl(len, 0xfffffff8);
 9087     jcc(Assembler::zero, copy_tail);
 9088 
 9089     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9090     movdl(tmp1Reg, tmp5);
 9091     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9092 
 9093     andl(len, 0xfffffff0);
 9094     jccb(Assembler::zero, copy_16);
 9095 
 9096     // compress 16 chars per iter
 9097     pxor(tmp4Reg, tmp4Reg);
 9098 
 9099     lea(src, Address(src, len, Address::times_2));
 9100     lea(dst, Address(dst, len, Address::times_1));
 9101     negptr(len);
 9102 
 9103     bind(copy_32_loop);
 9104     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9105     por(tmp4Reg, tmp2Reg);
 9106     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9107     por(tmp4Reg, tmp3Reg);
 9108     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9109     jccb(Assembler::notZero, reset_for_copy_tail);
 9110     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9111     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9112     addptr(len, 16);
 9113     jccb(Assembler::notZero, copy_32_loop);
 9114 
 9115     // compress next vector of 8 chars (if any)
 9116     bind(copy_16);
 9117     // len = 0
 9118     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
 9119     jccb(Assembler::zero, copy_tail_sse);
 9120 
 9121     pxor(tmp3Reg, tmp3Reg);
 9122 
 9123     movdqu(tmp2Reg, Address(src, 0));
 9124     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9125     jccb(Assembler::notZero, reset_for_copy_tail);
 9126     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9127     movq(Address(dst, 0), tmp2Reg);
 9128     addptr(src, 16);
 9129     addptr(dst, 8);
 9130     jmpb(copy_tail_sse);
 9131 
 9132     bind(reset_for_copy_tail);
 9133     movl(tmp5, result);
 9134     andl(tmp5, 0x0000000f);
 9135     lea(src, Address(src, tmp5, Address::times_2));
 9136     lea(dst, Address(dst, tmp5, Address::times_1));
 9137     subptr(len, tmp5);
 9138     jmpb(copy_chars_loop);
 9139 
 9140     bind(copy_tail_sse);
 9141     movl(len, result);
 9142     andl(len, 0x00000007);    // tail count (in chars)
 9143   }
 9144   // compress 1 char per iter
 9145   bind(copy_tail);
 9146   testl(len, len);
 9147   jccb(Assembler::zero, done);
 9148   lea(src, Address(src, len, Address::times_2));
 9149   lea(dst, Address(dst, len, Address::times_1));
 9150   negptr(len);
 9151 
 9152   bind(copy_chars_loop);
 9153   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
 9154   testl(tmp5, 0xff00);      // check if Unicode char
 9155   jccb(Assembler::notZero, reset_sp);
 9156   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
 9157   increment(len);
 9158   jccb(Assembler::notZero, copy_chars_loop);
 9159 
 9160   // add len then return (len will be zero if compress succeeded, otherwise negative)
 9161   bind(reset_sp);
 9162   addl(result, len);
 9163 
 9164   bind(done);
 9165 }
 9166 
 9167 // Inflate byte[] array to char[].
 9168 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9169 //   @IntrinsicCandidate
 9170 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9171 //     for (int i = 0; i < len; i++) {
 9172 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9173 //     }
 9174 //   }
 9175 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9176   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9177   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9178   // rsi: src
 9179   // rdi: dst
 9180   // rdx: len
 9181   // rcx: tmp2
 9182 
 9183   // rsi holds start addr of source byte[] to be inflated
 9184   // rdi holds start addr of destination char[]
 9185   // rdx holds length
 9186   assert_different_registers(src, dst, len, tmp2);
 9187   movl(tmp2, len);
 9188   if ((UseAVX > 2) && // AVX512
 9189     VM_Version::supports_avx512vlbw() &&
 9190     VM_Version::supports_bmi2()) {
 9191 
 9192     Label copy_32_loop, copy_tail;
 9193     Register tmp3_aliased = len;
 9194 
 9195     // if length of the string is less than 16, handle it in an old fashioned way
 9196     testl(len, -16);
 9197     jcc(Assembler::zero, below_threshold);
 9198 
 9199     testl(len, -1 * AVX3Threshold);
 9200     jcc(Assembler::zero, avx3_threshold);
 9201 
 9202     // In order to use only one arithmetic operation for the main loop we use
 9203     // this pre-calculation
 9204     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9205     andl(len, -32);     // vector count
 9206     jccb(Assembler::zero, copy_tail);
 9207 
 9208     lea(src, Address(src, len, Address::times_1));
 9209     lea(dst, Address(dst, len, Address::times_2));
 9210     negptr(len);
 9211 
 9212 
 9213     // inflate 32 chars per iter
 9214     bind(copy_32_loop);
 9215     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9216     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9217     addptr(len, 32);
 9218     jcc(Assembler::notZero, copy_32_loop);
 9219 
 9220     bind(copy_tail);
 9221     // bail out when there is nothing to be done
 9222     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9223     jcc(Assembler::zero, done);
 9224 
 9225     // ~(~0 << length), where length is the # of remaining elements to process
 9226     movl(tmp3_aliased, -1);
 9227     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9228     notl(tmp3_aliased);
 9229     kmovdl(mask, tmp3_aliased);
 9230     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9231     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9232 
 9233     jmp(done);
 9234     bind(avx3_threshold);
 9235   }
 9236   if (UseSSE42Intrinsics) {
 9237     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9238 
 9239     if (UseAVX > 1) {
 9240       andl(tmp2, (16 - 1));
 9241       andl(len, -16);
 9242       jccb(Assembler::zero, copy_new_tail);
 9243     } else {
 9244       andl(tmp2, 0x00000007);   // tail count (in chars)
 9245       andl(len, 0xfffffff8);    // vector count (in chars)
 9246       jccb(Assembler::zero, copy_tail);
 9247     }
 9248 
 9249     // vectored inflation
 9250     lea(src, Address(src, len, Address::times_1));
 9251     lea(dst, Address(dst, len, Address::times_2));
 9252     negptr(len);
 9253 
 9254     if (UseAVX > 1) {
 9255       bind(copy_16_loop);
 9256       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9257       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9258       addptr(len, 16);
 9259       jcc(Assembler::notZero, copy_16_loop);
 9260 
 9261       bind(below_threshold);
 9262       bind(copy_new_tail);
 9263       movl(len, tmp2);
 9264       andl(tmp2, 0x00000007);
 9265       andl(len, 0xFFFFFFF8);
 9266       jccb(Assembler::zero, copy_tail);
 9267 
 9268       pmovzxbw(tmp1, Address(src, 0));
 9269       movdqu(Address(dst, 0), tmp1);
 9270       addptr(src, 8);
 9271       addptr(dst, 2 * 8);
 9272 
 9273       jmp(copy_tail, true);
 9274     }
 9275 
 9276     // inflate 8 chars per iter
 9277     bind(copy_8_loop);
 9278     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9279     movdqu(Address(dst, len, Address::times_2), tmp1);
 9280     addptr(len, 8);
 9281     jcc(Assembler::notZero, copy_8_loop);
 9282 
 9283     bind(copy_tail);
 9284     movl(len, tmp2);
 9285 
 9286     cmpl(len, 4);
 9287     jccb(Assembler::less, copy_bytes);
 9288 
 9289     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9290     pmovzxbw(tmp1, tmp1);
 9291     movq(Address(dst, 0), tmp1);
 9292     subptr(len, 4);
 9293     addptr(src, 4);
 9294     addptr(dst, 8);
 9295 
 9296     bind(copy_bytes);
 9297   } else {
 9298     bind(below_threshold);
 9299   }
 9300 
 9301   testl(len, len);
 9302   jccb(Assembler::zero, done);
 9303   lea(src, Address(src, len, Address::times_1));
 9304   lea(dst, Address(dst, len, Address::times_2));
 9305   negptr(len);
 9306 
 9307   // inflate 1 char per iter
 9308   bind(copy_chars_loop);
 9309   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9310   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9311   increment(len);
 9312   jcc(Assembler::notZero, copy_chars_loop);
 9313 
 9314   bind(done);
 9315 }
 9316 
 9317 
 9318 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9319   switch(type) {
 9320     case T_BYTE:
 9321     case T_BOOLEAN:
 9322       evmovdqub(dst, kmask, src, merge, vector_len);
 9323       break;
 9324     case T_CHAR:
 9325     case T_SHORT:
 9326       evmovdquw(dst, kmask, src, merge, vector_len);
 9327       break;
 9328     case T_INT:
 9329     case T_FLOAT:
 9330       evmovdqul(dst, kmask, src, merge, vector_len);
 9331       break;
 9332     case T_LONG:
 9333     case T_DOUBLE:
 9334       evmovdquq(dst, kmask, src, merge, vector_len);
 9335       break;
 9336     default:
 9337       fatal("Unexpected type argument %s", type2name(type));
 9338       break;
 9339   }
 9340 }
 9341 
 9342 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9343   switch(type) {
 9344     case T_BYTE:
 9345     case T_BOOLEAN:
 9346       evmovdqub(dst, kmask, src, merge, vector_len);
 9347       break;
 9348     case T_CHAR:
 9349     case T_SHORT:
 9350       evmovdquw(dst, kmask, src, merge, vector_len);
 9351       break;
 9352     case T_INT:
 9353     case T_FLOAT:
 9354       evmovdqul(dst, kmask, src, merge, vector_len);
 9355       break;
 9356     case T_LONG:
 9357     case T_DOUBLE:
 9358       evmovdquq(dst, kmask, src, merge, vector_len);
 9359       break;
 9360     default:
 9361       fatal("Unexpected type argument %s", type2name(type));
 9362       break;
 9363   }
 9364 }
 9365 
 9366 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9367   switch(masklen) {
 9368     case 2:
 9369        knotbl(dst, src);
 9370        movl(rtmp, 3);
 9371        kmovbl(ktmp, rtmp);
 9372        kandbl(dst, ktmp, dst);
 9373        break;
 9374     case 4:
 9375        knotbl(dst, src);
 9376        movl(rtmp, 15);
 9377        kmovbl(ktmp, rtmp);
 9378        kandbl(dst, ktmp, dst);
 9379        break;
 9380     case 8:
 9381        knotbl(dst, src);
 9382        break;
 9383     case 16:
 9384        knotwl(dst, src);
 9385        break;
 9386     case 32:
 9387        knotdl(dst, src);
 9388        break;
 9389     case 64:
 9390        knotql(dst, src);
 9391        break;
 9392     default:
 9393       fatal("Unexpected vector length %d", masklen);
 9394       break;
 9395   }
 9396 }
 9397 
 9398 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9399   switch(type) {
 9400     case T_BOOLEAN:
 9401     case T_BYTE:
 9402        kandbl(dst, src1, src2);
 9403        break;
 9404     case T_CHAR:
 9405     case T_SHORT:
 9406        kandwl(dst, src1, src2);
 9407        break;
 9408     case T_INT:
 9409     case T_FLOAT:
 9410        kanddl(dst, src1, src2);
 9411        break;
 9412     case T_LONG:
 9413     case T_DOUBLE:
 9414        kandql(dst, src1, src2);
 9415        break;
 9416     default:
 9417       fatal("Unexpected type argument %s", type2name(type));
 9418       break;
 9419   }
 9420 }
 9421 
 9422 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9423   switch(type) {
 9424     case T_BOOLEAN:
 9425     case T_BYTE:
 9426        korbl(dst, src1, src2);
 9427        break;
 9428     case T_CHAR:
 9429     case T_SHORT:
 9430        korwl(dst, src1, src2);
 9431        break;
 9432     case T_INT:
 9433     case T_FLOAT:
 9434        kordl(dst, src1, src2);
 9435        break;
 9436     case T_LONG:
 9437     case T_DOUBLE:
 9438        korql(dst, src1, src2);
 9439        break;
 9440     default:
 9441       fatal("Unexpected type argument %s", type2name(type));
 9442       break;
 9443   }
 9444 }
 9445 
 9446 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9447   switch(type) {
 9448     case T_BOOLEAN:
 9449     case T_BYTE:
 9450        kxorbl(dst, src1, src2);
 9451        break;
 9452     case T_CHAR:
 9453     case T_SHORT:
 9454        kxorwl(dst, src1, src2);
 9455        break;
 9456     case T_INT:
 9457     case T_FLOAT:
 9458        kxordl(dst, src1, src2);
 9459        break;
 9460     case T_LONG:
 9461     case T_DOUBLE:
 9462        kxorql(dst, src1, src2);
 9463        break;
 9464     default:
 9465       fatal("Unexpected type argument %s", type2name(type));
 9466       break;
 9467   }
 9468 }
 9469 
 9470 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9471   switch(type) {
 9472     case T_BOOLEAN:
 9473     case T_BYTE:
 9474       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9475     case T_CHAR:
 9476     case T_SHORT:
 9477       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9478     case T_INT:
 9479     case T_FLOAT:
 9480       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9481     case T_LONG:
 9482     case T_DOUBLE:
 9483       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9484     default:
 9485       fatal("Unexpected type argument %s", type2name(type)); break;
 9486   }
 9487 }
 9488 
 9489 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9490   switch(type) {
 9491     case T_BOOLEAN:
 9492     case T_BYTE:
 9493       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9494     case T_CHAR:
 9495     case T_SHORT:
 9496       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9497     case T_INT:
 9498     case T_FLOAT:
 9499       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9500     case T_LONG:
 9501     case T_DOUBLE:
 9502       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9503     default:
 9504       fatal("Unexpected type argument %s", type2name(type)); break;
 9505   }
 9506 }
 9507 
 9508 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9509   switch(type) {
 9510     case T_BYTE:
 9511       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9512     case T_SHORT:
 9513       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9514     case T_INT:
 9515       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9516     case T_LONG:
 9517       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9518     default:
 9519       fatal("Unexpected type argument %s", type2name(type)); break;
 9520   }
 9521 }
 9522 
 9523 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9524   switch(type) {
 9525     case T_BYTE:
 9526       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9527     case T_SHORT:
 9528       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9529     case T_INT:
 9530       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9531     case T_LONG:
 9532       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9533     default:
 9534       fatal("Unexpected type argument %s", type2name(type)); break;
 9535   }
 9536 }
 9537 
 9538 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9539   switch(type) {
 9540     case T_BYTE:
 9541       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9542     case T_SHORT:
 9543       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9544     case T_INT:
 9545       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9546     case T_LONG:
 9547       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9548     default:
 9549       fatal("Unexpected type argument %s", type2name(type)); break;
 9550   }
 9551 }
 9552 
 9553 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9554   switch(type) {
 9555     case T_BYTE:
 9556       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9557     case T_SHORT:
 9558       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9559     case T_INT:
 9560       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9561     case T_LONG:
 9562       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9563     default:
 9564       fatal("Unexpected type argument %s", type2name(type)); break;
 9565   }
 9566 }
 9567 
 9568 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9569   switch(type) {
 9570     case T_INT:
 9571       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9572     case T_LONG:
 9573       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9574     default:
 9575       fatal("Unexpected type argument %s", type2name(type)); break;
 9576   }
 9577 }
 9578 
 9579 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9580   switch(type) {
 9581     case T_INT:
 9582       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9583     case T_LONG:
 9584       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9585     default:
 9586       fatal("Unexpected type argument %s", type2name(type)); break;
 9587   }
 9588 }
 9589 
 9590 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9591   switch(type) {
 9592     case T_INT:
 9593       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9594     case T_LONG:
 9595       evporq(dst, mask, nds, src, merge, vector_len); break;
 9596     default:
 9597       fatal("Unexpected type argument %s", type2name(type)); break;
 9598   }
 9599 }
 9600 
 9601 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9602   switch(type) {
 9603     case T_INT:
 9604       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9605     case T_LONG:
 9606       evporq(dst, mask, nds, src, merge, vector_len); break;
 9607     default:
 9608       fatal("Unexpected type argument %s", type2name(type)); break;
 9609   }
 9610 }
 9611 
 9612 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9613   switch(type) {
 9614     case T_INT:
 9615       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9616     case T_LONG:
 9617       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9618     default:
 9619       fatal("Unexpected type argument %s", type2name(type)); break;
 9620   }
 9621 }
 9622 
 9623 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9624   switch(type) {
 9625     case T_INT:
 9626       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9627     case T_LONG:
 9628       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9629     default:
 9630       fatal("Unexpected type argument %s", type2name(type)); break;
 9631   }
 9632 }
 9633 
 9634 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
 9635   switch(masklen) {
 9636     case 8:
 9637        kortestbl(src1, src2);
 9638        break;
 9639     case 16:
 9640        kortestwl(src1, src2);
 9641        break;
 9642     case 32:
 9643        kortestdl(src1, src2);
 9644        break;
 9645     case 64:
 9646        kortestql(src1, src2);
 9647        break;
 9648     default:
 9649       fatal("Unexpected mask length %d", masklen);
 9650       break;
 9651   }
 9652 }
 9653 
 9654 
 9655 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
 9656   switch(masklen)  {
 9657     case 8:
 9658        ktestbl(src1, src2);
 9659        break;
 9660     case 16:
 9661        ktestwl(src1, src2);
 9662        break;
 9663     case 32:
 9664        ktestdl(src1, src2);
 9665        break;
 9666     case 64:
 9667        ktestql(src1, src2);
 9668        break;
 9669     default:
 9670       fatal("Unexpected mask length %d", masklen);
 9671       break;
 9672   }
 9673 }
 9674 
 9675 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9676   switch(type) {
 9677     case T_INT:
 9678       evprold(dst, mask, src, shift, merge, vlen_enc); break;
 9679     case T_LONG:
 9680       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
 9681     default:
 9682       fatal("Unexpected type argument %s", type2name(type)); break;
 9683       break;
 9684   }
 9685 }
 9686 
 9687 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9688   switch(type) {
 9689     case T_INT:
 9690       evprord(dst, mask, src, shift, merge, vlen_enc); break;
 9691     case T_LONG:
 9692       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
 9693     default:
 9694       fatal("Unexpected type argument %s", type2name(type)); break;
 9695   }
 9696 }
 9697 
 9698 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9699   switch(type) {
 9700     case T_INT:
 9701       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9702     case T_LONG:
 9703       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9704     default:
 9705       fatal("Unexpected type argument %s", type2name(type)); break;
 9706   }
 9707 }
 9708 
 9709 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9710   switch(type) {
 9711     case T_INT:
 9712       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9713     case T_LONG:
 9714       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9715     default:
 9716       fatal("Unexpected type argument %s", type2name(type)); break;
 9717   }
 9718 }
 9719 
 9720 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9721   assert(rscratch != noreg || always_reachable(src), "missing");
 9722 
 9723   if (reachable(src)) {
 9724     evpandq(dst, nds, as_Address(src), vector_len);
 9725   } else {
 9726     lea(rscratch, src);
 9727     evpandq(dst, nds, Address(rscratch, 0), vector_len);
 9728   }
 9729 }
 9730 
 9731 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 9732   assert(rscratch != noreg || always_reachable(src), "missing");
 9733 
 9734   if (reachable(src)) {
 9735     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
 9736   } else {
 9737     lea(rscratch, src);
 9738     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 9739   }
 9740 }
 9741 
 9742 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9743   assert(rscratch != noreg || always_reachable(src), "missing");
 9744 
 9745   if (reachable(src)) {
 9746     evporq(dst, nds, as_Address(src), vector_len);
 9747   } else {
 9748     lea(rscratch, src);
 9749     evporq(dst, nds, Address(rscratch, 0), vector_len);
 9750   }
 9751 }
 9752 
 9753 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9754   assert(rscratch != noreg || always_reachable(src), "missing");
 9755 
 9756   if (reachable(src)) {
 9757     vpshufb(dst, nds, as_Address(src), vector_len);
 9758   } else {
 9759     lea(rscratch, src);
 9760     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
 9761   }
 9762 }
 9763 
 9764 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9765   assert(rscratch != noreg || always_reachable(src), "missing");
 9766 
 9767   if (reachable(src)) {
 9768     Assembler::vpor(dst, nds, as_Address(src), vector_len);
 9769   } else {
 9770     lea(rscratch, src);
 9771     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
 9772   }
 9773 }
 9774 
 9775 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
 9776   assert(rscratch != noreg || always_reachable(src3), "missing");
 9777 
 9778   if (reachable(src3)) {
 9779     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
 9780   } else {
 9781     lea(rscratch, src3);
 9782     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
 9783   }
 9784 }
 9785 
 9786 #if COMPILER2_OR_JVMCI
 9787 
 9788 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
 9789                                  Register length, Register temp, int vec_enc) {
 9790   // Computing mask for predicated vector store.
 9791   movptr(temp, -1);
 9792   bzhiq(temp, temp, length);
 9793   kmov(mask, temp);
 9794   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
 9795 }
 9796 
 9797 // Set memory operation for length "less than" 64 bytes.
 9798 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
 9799                                        XMMRegister xmm, KRegister mask, Register length,
 9800                                        Register temp, bool use64byteVector) {
 9801   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9802   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9803   if (!use64byteVector) {
 9804     fill32(dst, disp, xmm);
 9805     subptr(length, 32 >> shift);
 9806     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
 9807   } else {
 9808     assert(MaxVectorSize == 64, "vector length != 64");
 9809     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
 9810   }
 9811 }
 9812 
 9813 
 9814 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
 9815                                        XMMRegister xmm, KRegister mask, Register length,
 9816                                        Register temp) {
 9817   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9818   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9819   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
 9820 }
 9821 
 9822 
 9823 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
 9824   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9825   vmovdqu(dst, xmm);
 9826 }
 9827 
 9828 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
 9829   fill32(Address(dst, disp), xmm);
 9830 }
 9831 
 9832 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
 9833   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9834   if (!use64byteVector) {
 9835     fill32(dst, xmm);
 9836     fill32(dst.plus_disp(32), xmm);
 9837   } else {
 9838     evmovdquq(dst, xmm, Assembler::AVX_512bit);
 9839   }
 9840 }
 9841 
 9842 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
 9843   fill64(Address(dst, disp), xmm, use64byteVector);
 9844 }
 9845 
 9846 #ifdef _LP64
 9847 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
 9848                                         Register count, Register rtmp, XMMRegister xtmp) {
 9849   Label L_exit;
 9850   Label L_fill_start;
 9851   Label L_fill_64_bytes;
 9852   Label L_fill_96_bytes;
 9853   Label L_fill_128_bytes;
 9854   Label L_fill_128_bytes_loop;
 9855   Label L_fill_128_loop_header;
 9856   Label L_fill_128_bytes_loop_header;
 9857   Label L_fill_128_bytes_loop_pre_header;
 9858   Label L_fill_zmm_sequence;
 9859 
 9860   int shift = -1;
 9861   int avx3threshold = VM_Version::avx3_threshold();
 9862   switch(type) {
 9863     case T_BYTE:  shift = 0;
 9864       break;
 9865     case T_SHORT: shift = 1;
 9866       break;
 9867     case T_INT:   shift = 2;
 9868       break;
 9869     /* Uncomment when LONG fill stubs are supported.
 9870     case T_LONG:  shift = 3;
 9871       break;
 9872     */
 9873     default:
 9874       fatal("Unhandled type: %s\n", type2name(type));
 9875   }
 9876 
 9877   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
 9878 
 9879     if (MaxVectorSize == 64) {
 9880       cmpq(count, avx3threshold >> shift);
 9881       jcc(Assembler::greater, L_fill_zmm_sequence);
 9882     }
 9883 
 9884     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
 9885 
 9886     bind(L_fill_start);
 9887 
 9888     cmpq(count, 32 >> shift);
 9889     jccb(Assembler::greater, L_fill_64_bytes);
 9890     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9891     jmp(L_exit);
 9892 
 9893     bind(L_fill_64_bytes);
 9894     cmpq(count, 64 >> shift);
 9895     jccb(Assembler::greater, L_fill_96_bytes);
 9896     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9897     jmp(L_exit);
 9898 
 9899     bind(L_fill_96_bytes);
 9900     cmpq(count, 96 >> shift);
 9901     jccb(Assembler::greater, L_fill_128_bytes);
 9902     fill64(to, 0, xtmp);
 9903     subq(count, 64 >> shift);
 9904     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
 9905     jmp(L_exit);
 9906 
 9907     bind(L_fill_128_bytes);
 9908     cmpq(count, 128 >> shift);
 9909     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
 9910     fill64(to, 0, xtmp);
 9911     fill32(to, 64, xtmp);
 9912     subq(count, 96 >> shift);
 9913     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
 9914     jmp(L_exit);
 9915 
 9916     bind(L_fill_128_bytes_loop_pre_header);
 9917     {
 9918       mov(rtmp, to);
 9919       andq(rtmp, 31);
 9920       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
 9921       negq(rtmp);
 9922       addq(rtmp, 32);
 9923       mov64(r8, -1L);
 9924       bzhiq(r8, r8, rtmp);
 9925       kmovql(k2, r8);
 9926       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
 9927       addq(to, rtmp);
 9928       shrq(rtmp, shift);
 9929       subq(count, rtmp);
 9930     }
 9931 
 9932     cmpq(count, 128 >> shift);
 9933     jcc(Assembler::less, L_fill_start);
 9934 
 9935     bind(L_fill_128_bytes_loop_header);
 9936     subq(count, 128 >> shift);
 9937 
 9938     align32();
 9939     bind(L_fill_128_bytes_loop);
 9940       fill64(to, 0, xtmp);
 9941       fill64(to, 64, xtmp);
 9942       addq(to, 128);
 9943       subq(count, 128 >> shift);
 9944       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
 9945 
 9946     addq(count, 128 >> shift);
 9947     jcc(Assembler::zero, L_exit);
 9948     jmp(L_fill_start);
 9949   }
 9950 
 9951   if (MaxVectorSize == 64) {
 9952     // Sequence using 64 byte ZMM register.
 9953     Label L_fill_128_bytes_zmm;
 9954     Label L_fill_192_bytes_zmm;
 9955     Label L_fill_192_bytes_loop_zmm;
 9956     Label L_fill_192_bytes_loop_header_zmm;
 9957     Label L_fill_192_bytes_loop_pre_header_zmm;
 9958     Label L_fill_start_zmm_sequence;
 9959 
 9960     bind(L_fill_zmm_sequence);
 9961     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
 9962 
 9963     bind(L_fill_start_zmm_sequence);
 9964     cmpq(count, 64 >> shift);
 9965     jccb(Assembler::greater, L_fill_128_bytes_zmm);
 9966     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
 9967     jmp(L_exit);
 9968 
 9969     bind(L_fill_128_bytes_zmm);
 9970     cmpq(count, 128 >> shift);
 9971     jccb(Assembler::greater, L_fill_192_bytes_zmm);
 9972     fill64(to, 0, xtmp, true);
 9973     subq(count, 64 >> shift);
 9974     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
 9975     jmp(L_exit);
 9976 
 9977     bind(L_fill_192_bytes_zmm);
 9978     cmpq(count, 192 >> shift);
 9979     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
 9980     fill64(to, 0, xtmp, true);
 9981     fill64(to, 64, xtmp, true);
 9982     subq(count, 128 >> shift);
 9983     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
 9984     jmp(L_exit);
 9985 
 9986     bind(L_fill_192_bytes_loop_pre_header_zmm);
 9987     {
 9988       movq(rtmp, to);
 9989       andq(rtmp, 63);
 9990       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
 9991       negq(rtmp);
 9992       addq(rtmp, 64);
 9993       mov64(r8, -1L);
 9994       bzhiq(r8, r8, rtmp);
 9995       kmovql(k2, r8);
 9996       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
 9997       addq(to, rtmp);
 9998       shrq(rtmp, shift);
 9999       subq(count, rtmp);
10000     }
10001 
10002     cmpq(count, 192 >> shift);
10003     jcc(Assembler::less, L_fill_start_zmm_sequence);
10004 
10005     bind(L_fill_192_bytes_loop_header_zmm);
10006     subq(count, 192 >> shift);
10007 
10008     align32();
10009     bind(L_fill_192_bytes_loop_zmm);
10010       fill64(to, 0, xtmp, true);
10011       fill64(to, 64, xtmp, true);
10012       fill64(to, 128, xtmp, true);
10013       addq(to, 192);
10014       subq(count, 192 >> shift);
10015       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10016 
10017     addq(count, 192 >> shift);
10018     jcc(Assembler::zero, L_exit);
10019     jmp(L_fill_start_zmm_sequence);
10020   }
10021   bind(L_exit);
10022 }
10023 #endif
10024 #endif //COMPILER2_OR_JVMCI
10025 
10026 
10027 #ifdef _LP64
10028 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10029   Label done;
10030   cvttss2sil(dst, src);
10031   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10032   cmpl(dst, 0x80000000); // float_sign_flip
10033   jccb(Assembler::notEqual, done);
10034   subptr(rsp, 8);
10035   movflt(Address(rsp, 0), src);
10036   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10037   pop(dst);
10038   bind(done);
10039 }
10040 
10041 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10042   Label done;
10043   cvttsd2sil(dst, src);
10044   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10045   cmpl(dst, 0x80000000); // float_sign_flip
10046   jccb(Assembler::notEqual, done);
10047   subptr(rsp, 8);
10048   movdbl(Address(rsp, 0), src);
10049   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10050   pop(dst);
10051   bind(done);
10052 }
10053 
10054 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10055   Label done;
10056   cvttss2siq(dst, src);
10057   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10058   jccb(Assembler::notEqual, done);
10059   subptr(rsp, 8);
10060   movflt(Address(rsp, 0), src);
10061   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10062   pop(dst);
10063   bind(done);
10064 }
10065 
10066 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10067   // Following code is line by line assembly translation rounding algorithm.
10068   // Please refer to java.lang.Math.round(float) algorithm for details.
10069   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10070   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10071   const int32_t FloatConsts_EXP_BIAS = 127;
10072   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10073   const int32_t MINUS_32 = 0xFFFFFFE0;
10074   Label L_special_case, L_block1, L_exit;
10075   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10076   movdl(dst, src);
10077   andl(dst, rtmp);
10078   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10079   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10080   subl(rtmp, dst);
10081   movl(rcx, rtmp);
10082   movl(dst, MINUS_32);
10083   testl(rtmp, dst);
10084   jccb(Assembler::notEqual, L_special_case);
10085   movdl(dst, src);
10086   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10087   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10088   movdl(rtmp, src);
10089   testl(rtmp, rtmp);
10090   jccb(Assembler::greaterEqual, L_block1);
10091   negl(dst);
10092   bind(L_block1);
10093   sarl(dst);
10094   addl(dst, 0x1);
10095   sarl(dst, 0x1);
10096   jmp(L_exit);
10097   bind(L_special_case);
10098   convert_f2i(dst, src);
10099   bind(L_exit);
10100 }
10101 
10102 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10103   // Following code is line by line assembly translation rounding algorithm.
10104   // Please refer to java.lang.Math.round(double) algorithm for details.
10105   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10106   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10107   const int64_t DoubleConsts_EXP_BIAS = 1023;
10108   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10109   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10110   Label L_special_case, L_block1, L_exit;
10111   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10112   movq(dst, src);
10113   andq(dst, rtmp);
10114   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10115   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10116   subq(rtmp, dst);
10117   movq(rcx, rtmp);
10118   mov64(dst, MINUS_64);
10119   testq(rtmp, dst);
10120   jccb(Assembler::notEqual, L_special_case);
10121   movq(dst, src);
10122   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10123   andq(dst, rtmp);
10124   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10125   orq(dst, rtmp);
10126   movq(rtmp, src);
10127   testq(rtmp, rtmp);
10128   jccb(Assembler::greaterEqual, L_block1);
10129   negq(dst);
10130   bind(L_block1);
10131   sarq(dst);
10132   addq(dst, 0x1);
10133   sarq(dst, 0x1);
10134   jmp(L_exit);
10135   bind(L_special_case);
10136   convert_d2l(dst, src);
10137   bind(L_exit);
10138 }
10139 
10140 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10141   Label done;
10142   cvttsd2siq(dst, src);
10143   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10144   jccb(Assembler::notEqual, done);
10145   subptr(rsp, 8);
10146   movdbl(Address(rsp, 0), src);
10147   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10148   pop(dst);
10149   bind(done);
10150 }
10151 
10152 void MacroAssembler::cache_wb(Address line)
10153 {
10154   // 64 bit cpus always support clflush
10155   assert(VM_Version::supports_clflush(), "clflush should be available");
10156   bool optimized = VM_Version::supports_clflushopt();
10157   bool no_evict = VM_Version::supports_clwb();
10158 
10159   // prefer clwb (writeback without evict) otherwise
10160   // prefer clflushopt (potentially parallel writeback with evict)
10161   // otherwise fallback on clflush (serial writeback with evict)
10162 
10163   if (optimized) {
10164     if (no_evict) {
10165       clwb(line);
10166     } else {
10167       clflushopt(line);
10168     }
10169   } else {
10170     // no need for fence when using CLFLUSH
10171     clflush(line);
10172   }
10173 }
10174 
10175 void MacroAssembler::cache_wbsync(bool is_pre)
10176 {
10177   assert(VM_Version::supports_clflush(), "clflush should be available");
10178   bool optimized = VM_Version::supports_clflushopt();
10179   bool no_evict = VM_Version::supports_clwb();
10180 
10181   // pick the correct implementation
10182 
10183   if (!is_pre && (optimized || no_evict)) {
10184     // need an sfence for post flush when using clflushopt or clwb
10185     // otherwise no no need for any synchroniaztion
10186 
10187     sfence();
10188   }
10189 }
10190 
10191 #endif // _LP64
10192 
10193 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10194   switch (cond) {
10195     // Note some conditions are synonyms for others
10196     case Assembler::zero:         return Assembler::notZero;
10197     case Assembler::notZero:      return Assembler::zero;
10198     case Assembler::less:         return Assembler::greaterEqual;
10199     case Assembler::lessEqual:    return Assembler::greater;
10200     case Assembler::greater:      return Assembler::lessEqual;
10201     case Assembler::greaterEqual: return Assembler::less;
10202     case Assembler::below:        return Assembler::aboveEqual;
10203     case Assembler::belowEqual:   return Assembler::above;
10204     case Assembler::above:        return Assembler::belowEqual;
10205     case Assembler::aboveEqual:   return Assembler::below;
10206     case Assembler::overflow:     return Assembler::noOverflow;
10207     case Assembler::noOverflow:   return Assembler::overflow;
10208     case Assembler::negative:     return Assembler::positive;
10209     case Assembler::positive:     return Assembler::negative;
10210     case Assembler::parity:       return Assembler::noParity;
10211     case Assembler::noParity:     return Assembler::parity;
10212   }
10213   ShouldNotReachHere(); return Assembler::overflow;
10214 }
10215 
10216 SkipIfEqual::SkipIfEqual(
10217     MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
10218   _masm = masm;
10219   _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
10220   _masm->jcc(Assembler::equal, _label);
10221 }
10222 
10223 SkipIfEqual::~SkipIfEqual() {
10224   _masm->bind(_label);
10225 }
10226 
10227 // 32-bit Windows has its own fast-path implementation
10228 // of get_thread
10229 #if !defined(WIN32) || defined(_LP64)
10230 
10231 // This is simply a call to Thread::current()
10232 void MacroAssembler::get_thread(Register thread) {
10233   if (thread != rax) {
10234     push(rax);
10235   }
10236   LP64_ONLY(push(rdi);)
10237   LP64_ONLY(push(rsi);)
10238   push(rdx);
10239   push(rcx);
10240 #ifdef _LP64
10241   push(r8);
10242   push(r9);
10243   push(r10);
10244   push(r11);
10245 #endif
10246 
10247   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10248 
10249 #ifdef _LP64
10250   pop(r11);
10251   pop(r10);
10252   pop(r9);
10253   pop(r8);
10254 #endif
10255   pop(rcx);
10256   pop(rdx);
10257   LP64_ONLY(pop(rsi);)
10258   LP64_ONLY(pop(rdi);)
10259   if (thread != rax) {
10260     mov(thread, rax);
10261     pop(rax);
10262   }
10263 }
10264 
10265 
10266 #endif // !WIN32 || _LP64
10267 
10268 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10269   Label L_stack_ok;
10270   if (bias == 0) {
10271     testptr(sp, 2 * wordSize - 1);
10272   } else {
10273     // lea(tmp, Address(rsp, bias);
10274     mov(tmp, sp);
10275     addptr(tmp, bias);
10276     testptr(tmp, 2 * wordSize - 1);
10277   }
10278   jcc(Assembler::equal, L_stack_ok);
10279   block_comment(msg);
10280   stop(msg);
10281   bind(L_stack_ok);
10282 }
10283 
10284 // Implements lightweight-locking.
10285 //
10286 // obj: the object to be locked
10287 // reg_rax: rax
10288 // thread: the thread which attempts to lock obj
10289 // tmp: a temporary register
10290 void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10291   assert(reg_rax == rax, "");
10292   assert_different_registers(obj, reg_rax, thread, tmp);
10293 
10294   Label push;
10295   const Register top = tmp;
10296 
10297   // Preload the markWord. It is important that this is the first
10298   // instruction emitted as it is part of C1's null check semantics.
10299   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10300 
10301   // Load top.
10302   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10303 
10304   // Check if the lock-stack is full.
10305   cmpl(top, LockStack::end_offset());
10306   jcc(Assembler::greaterEqual, slow);
10307 
10308   // Check for recursion.
10309   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10310   jcc(Assembler::equal, push);
10311 
10312   // Check header for monitor (0b10).
10313   testptr(reg_rax, markWord::monitor_value);
10314   jcc(Assembler::notZero, slow);
10315 
10316   // Try to lock. Transition lock bits 0b01 => 0b00
10317   movptr(tmp, reg_rax);
10318   andptr(tmp, ~(int32_t)markWord::unlocked_value);
10319   orptr(reg_rax, markWord::unlocked_value);
10320   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10321   jcc(Assembler::notEqual, slow);
10322 
10323   // Restore top, CAS clobbers register.
10324   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10325 
10326   bind(push);
10327   // After successful lock, push object on lock-stack.
10328   movptr(Address(thread, top), obj);
10329   incrementl(top, oopSize);
10330   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
10331 }
10332 
10333 // Implements lightweight-unlocking.
10334 //
10335 // obj: the object to be unlocked
10336 // reg_rax: rax
10337 // thread: the thread
10338 // tmp: a temporary register
10339 //
10340 // x86_32 Note: reg_rax and thread may alias each other due to limited register
10341 //              availiability.
10342 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
10343   assert(reg_rax == rax, "");
10344   assert_different_registers(obj, reg_rax, tmp);
10345   LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
10346 
10347   Label unlocked, push_and_slow;
10348   const Register top = tmp;
10349 
10350   // Check if obj is top of lock-stack.
10351   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10352   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10353   jcc(Assembler::notEqual, slow);
10354 
10355   // Pop lock-stack.
10356   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
10357   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10358 
10359   // Check if recursive.
10360   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
10361   jcc(Assembler::equal, unlocked);
10362 
10363   // Not recursive. Check header for monitor (0b10).
10364   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10365   testptr(reg_rax, markWord::monitor_value);
10366   jcc(Assembler::notZero, push_and_slow);
10367 
10368 #ifdef ASSERT
10369   // Check header not unlocked (0b01).
10370   Label not_unlocked;
10371   testptr(reg_rax, markWord::unlocked_value);
10372   jcc(Assembler::zero, not_unlocked);
10373   stop("lightweight_unlock already unlocked");
10374   bind(not_unlocked);
10375 #endif
10376 
10377   // Try to unlock. Transition lock bits 0b00 => 0b01
10378   movptr(tmp, reg_rax);
10379   orptr(tmp, markWord::unlocked_value);
10380   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10381   jcc(Assembler::equal, unlocked);
10382 
10383   bind(push_and_slow);
10384   // Restore lock-stack and handle the unlock in runtime.
10385   if (thread == reg_rax) {
10386     // On x86_32 we may lose the thread.
10387     get_thread(thread);
10388   }
10389 #ifdef ASSERT
10390   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10391   movptr(Address(thread, top), obj);
10392 #endif
10393   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10394   jmp(slow);
10395 
10396   bind(unlocked);
10397 }
10398 
10399 #ifdef _LP64
10400 // Saves legacy GPRs state on stack.
10401 void MacroAssembler::save_legacy_gprs() {
10402   subq(rsp, 16 * wordSize);
10403   movq(Address(rsp, 15 * wordSize), rax);
10404   movq(Address(rsp, 14 * wordSize), rcx);
10405   movq(Address(rsp, 13 * wordSize), rdx);
10406   movq(Address(rsp, 12 * wordSize), rbx);
10407   movq(Address(rsp, 10 * wordSize), rbp);
10408   movq(Address(rsp, 9 * wordSize), rsi);
10409   movq(Address(rsp, 8 * wordSize), rdi);
10410   movq(Address(rsp, 7 * wordSize), r8);
10411   movq(Address(rsp, 6 * wordSize), r9);
10412   movq(Address(rsp, 5 * wordSize), r10);
10413   movq(Address(rsp, 4 * wordSize), r11);
10414   movq(Address(rsp, 3 * wordSize), r12);
10415   movq(Address(rsp, 2 * wordSize), r13);
10416   movq(Address(rsp, wordSize), r14);
10417   movq(Address(rsp, 0), r15);
10418 }
10419 
10420 // Resotres back legacy GPRs state from stack.
10421 void MacroAssembler::restore_legacy_gprs() {
10422   movq(r15, Address(rsp, 0));
10423   movq(r14, Address(rsp, wordSize));
10424   movq(r13, Address(rsp, 2 * wordSize));
10425   movq(r12, Address(rsp, 3 * wordSize));
10426   movq(r11, Address(rsp, 4 * wordSize));
10427   movq(r10, Address(rsp, 5 * wordSize));
10428   movq(r9,  Address(rsp, 6 * wordSize));
10429   movq(r8,  Address(rsp, 7 * wordSize));
10430   movq(rdi, Address(rsp, 8 * wordSize));
10431   movq(rsi, Address(rsp, 9 * wordSize));
10432   movq(rbp, Address(rsp, 10 * wordSize));
10433   movq(rbx, Address(rsp, 12 * wordSize));
10434   movq(rdx, Address(rsp, 13 * wordSize));
10435   movq(rcx, Address(rsp, 14 * wordSize));
10436   movq(rax, Address(rsp, 15 * wordSize));
10437   addq(rsp, 16 * wordSize);
10438 }
10439 #endif