1 /*
    2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "asm/assembler.hpp"
   26 #include "asm/assembler.inline.hpp"
   27 #include "code/compiledIC.hpp"
   28 #include "compiler/compiler_globals.hpp"
   29 #include "compiler/disassembler.hpp"
   30 #include "ci/ciInlineKlass.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "interpreter/interpreterRuntime.hpp"
   39 #include "jvm.h"
   40 #include "memory/resourceArea.hpp"
   41 #include "memory/universe.hpp"
   42 #include "oops/accessDecorators.hpp"
   43 #include "oops/compressedKlass.inline.hpp"
   44 #include "oops/compressedOops.inline.hpp"
   45 #include "oops/klass.inline.hpp"
   46 #include "oops/resolvedFieldEntry.hpp"
   47 #include "prims/methodHandles.hpp"
   48 #include "runtime/continuation.hpp"
   49 #include "runtime/interfaceSupport.inline.hpp"
   50 #include "runtime/javaThread.hpp"
   51 #include "runtime/jniHandles.hpp"
   52 #include "runtime/objectMonitor.hpp"
   53 #include "runtime/os.hpp"
   54 #include "runtime/safepoint.hpp"
   55 #include "runtime/safepointMechanism.hpp"
   56 #include "runtime/sharedRuntime.hpp"
   57 #include "runtime/signature_cc.hpp"
   58 #include "runtime/stubRoutines.hpp"
   59 #include "utilities/checkedCast.hpp"
   60 #include "utilities/macros.hpp"
   61 #include "vmreg_x86.inline.hpp"
   62 #ifdef COMPILER2
   63 #include "opto/output.hpp"
   64 #endif
   65 
   66 #ifdef PRODUCT
   67 #define BLOCK_COMMENT(str) /* nothing */
   68 #define STOP(error) stop(error)
   69 #else
   70 #define BLOCK_COMMENT(str) block_comment(str)
   71 #define STOP(error) block_comment(error); stop(error)
   72 #endif
   73 
   74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   75 
   76 #ifdef ASSERT
   77 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   78 #endif
   79 
   80 static const Assembler::Condition reverse[] = {
   81     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   82     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   83     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   84     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   85     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   86     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   87     Assembler::above          /* belowEqual    = 0x6 */ ,
   88     Assembler::belowEqual     /* above         = 0x7 */ ,
   89     Assembler::positive       /* negative      = 0x8 */ ,
   90     Assembler::negative       /* positive      = 0x9 */ ,
   91     Assembler::noParity       /* parity        = 0xa */ ,
   92     Assembler::parity         /* noParity      = 0xb */ ,
   93     Assembler::greaterEqual   /* less          = 0xc */ ,
   94     Assembler::less           /* greaterEqual  = 0xd */ ,
   95     Assembler::greater        /* lessEqual     = 0xe */ ,
   96     Assembler::lessEqual      /* greater       = 0xf, */
   97 
   98 };
   99 
  100 
  101 // Implementation of MacroAssembler
  102 
  103 // First all the versions that have distinct versions depending on 32/64 bit
  104 // Unless the difference is trivial (1 line or so).
  105 
  106 #ifndef _LP64
  107 
  108 // 32bit versions
  109 
  110 Address MacroAssembler::as_Address(AddressLiteral adr) {
  111   return Address(adr.target(), adr.rspec());
  112 }
  113 
  114 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  115   assert(rscratch == noreg, "");
  116   return Address::make_array(adr);
  117 }
  118 
  119 void MacroAssembler::call_VM_leaf_base(address entry_point,
  120                                        int number_of_arguments) {
  121   call(RuntimeAddress(entry_point));
  122   increment(rsp, number_of_arguments * wordSize);
  123 }
  124 
  125 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  126   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  127 }
  128 
  129 
  130 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  131   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  132 }
  133 
  134 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  135   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  136 }
  137 
  138 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  139   assert(rscratch == noreg, "redundant");
  140   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  141 }
  142 
  143 void MacroAssembler::extend_sign(Register hi, Register lo) {
  144   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  145   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  146     cdql();
  147   } else {
  148     movl(hi, lo);
  149     sarl(hi, 31);
  150   }
  151 }
  152 
  153 void MacroAssembler::jC2(Register tmp, Label& L) {
  154   // set parity bit if FPU flag C2 is set (via rax)
  155   save_rax(tmp);
  156   fwait(); fnstsw_ax();
  157   sahf();
  158   restore_rax(tmp);
  159   // branch
  160   jcc(Assembler::parity, L);
  161 }
  162 
  163 void MacroAssembler::jnC2(Register tmp, Label& L) {
  164   // set parity bit if FPU flag C2 is set (via rax)
  165   save_rax(tmp);
  166   fwait(); fnstsw_ax();
  167   sahf();
  168   restore_rax(tmp);
  169   // branch
  170   jcc(Assembler::noParity, L);
  171 }
  172 
  173 // 32bit can do a case table jump in one instruction but we no longer allow the base
  174 // to be installed in the Address class
  175 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  176   assert(rscratch == noreg, "not needed");
  177   jmp(as_Address(entry, noreg));
  178 }
  179 
  180 // Note: y_lo will be destroyed
  181 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  182   // Long compare for Java (semantics as described in JVM spec.)
  183   Label high, low, done;
  184 
  185   cmpl(x_hi, y_hi);
  186   jcc(Assembler::less, low);
  187   jcc(Assembler::greater, high);
  188   // x_hi is the return register
  189   xorl(x_hi, x_hi);
  190   cmpl(x_lo, y_lo);
  191   jcc(Assembler::below, low);
  192   jcc(Assembler::equal, done);
  193 
  194   bind(high);
  195   xorl(x_hi, x_hi);
  196   increment(x_hi);
  197   jmp(done);
  198 
  199   bind(low);
  200   xorl(x_hi, x_hi);
  201   decrementl(x_hi);
  202 
  203   bind(done);
  204 }
  205 
  206 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  207   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  208 }
  209 
  210 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  211   assert(rscratch == noreg, "not needed");
  212 
  213   // leal(dst, as_Address(adr));
  214   // see note in movl as to why we must use a move
  215   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  216 }
  217 
  218 void MacroAssembler::leave() {
  219   mov(rsp, rbp);
  220   pop(rbp);
  221 }
  222 
  223 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  224   // Multiplication of two Java long values stored on the stack
  225   // as illustrated below. Result is in rdx:rax.
  226   //
  227   // rsp ---> [  ??  ] \               \
  228   //            ....    | y_rsp_offset  |
  229   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  230   //          [ y_hi ]                  | (in bytes)
  231   //            ....                    |
  232   //          [ x_lo ]                 /
  233   //          [ x_hi ]
  234   //            ....
  235   //
  236   // Basic idea: lo(result) = lo(x_lo * y_lo)
  237   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  238   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  239   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  240   Label quick;
  241   // load x_hi, y_hi and check if quick
  242   // multiplication is possible
  243   movl(rbx, x_hi);
  244   movl(rcx, y_hi);
  245   movl(rax, rbx);
  246   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  247   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  248   // do full multiplication
  249   // 1st step
  250   mull(y_lo);                                    // x_hi * y_lo
  251   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  252   // 2nd step
  253   movl(rax, x_lo);
  254   mull(rcx);                                     // x_lo * y_hi
  255   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  256   // 3rd step
  257   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  258   movl(rax, x_lo);
  259   mull(y_lo);                                    // x_lo * y_lo
  260   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  261 }
  262 
  263 void MacroAssembler::lneg(Register hi, Register lo) {
  264   negl(lo);
  265   adcl(hi, 0);
  266   negl(hi);
  267 }
  268 
  269 void MacroAssembler::lshl(Register hi, Register lo) {
  270   // Java shift left long support (semantics as described in JVM spec., p.305)
  271   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  272   // shift value is in rcx !
  273   assert(hi != rcx, "must not use rcx");
  274   assert(lo != rcx, "must not use rcx");
  275   const Register s = rcx;                        // shift count
  276   const int      n = BitsPerWord;
  277   Label L;
  278   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  279   cmpl(s, n);                                    // if (s < n)
  280   jcc(Assembler::less, L);                       // else (s >= n)
  281   movl(hi, lo);                                  // x := x << n
  282   xorl(lo, lo);
  283   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  284   bind(L);                                       // s (mod n) < n
  285   shldl(hi, lo);                                 // x := x << s
  286   shll(lo);
  287 }
  288 
  289 
  290 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  291   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  292   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  293   assert(hi != rcx, "must not use rcx");
  294   assert(lo != rcx, "must not use rcx");
  295   const Register s = rcx;                        // shift count
  296   const int      n = BitsPerWord;
  297   Label L;
  298   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  299   cmpl(s, n);                                    // if (s < n)
  300   jcc(Assembler::less, L);                       // else (s >= n)
  301   movl(lo, hi);                                  // x := x >> n
  302   if (sign_extension) sarl(hi, 31);
  303   else                xorl(hi, hi);
  304   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  305   bind(L);                                       // s (mod n) < n
  306   shrdl(lo, hi);                                 // x := x >> s
  307   if (sign_extension) sarl(hi);
  308   else                shrl(hi);
  309 }
  310 
  311 void MacroAssembler::movoop(Register dst, jobject obj) {
  312   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  313 }
  314 
  315 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  316   assert(rscratch == noreg, "redundant");
  317   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  318 }
  319 
  320 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  321   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  322 }
  323 
  324 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  325   assert(rscratch == noreg, "redundant");
  326   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  327 }
  328 
  329 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  330   if (src.is_lval()) {
  331     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  332   } else {
  333     movl(dst, as_Address(src));
  334   }
  335 }
  336 
  337 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  338   assert(rscratch == noreg, "redundant");
  339   movl(as_Address(dst, noreg), src);
  340 }
  341 
  342 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  343   movl(dst, as_Address(src, noreg));
  344 }
  345 
  346 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  347   assert(rscratch == noreg, "redundant");
  348   movl(dst, src);
  349 }
  350 
  351 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  352   assert(rscratch == noreg, "redundant");
  353   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  354 }
  355 
  356 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  357   assert(rscratch == noreg, "redundant");
  358   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  359 }
  360 
  361 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  362   assert(rscratch == noreg, "redundant");
  363   if (src.is_lval()) {
  364     push_literal32((int32_t)src.target(), src.rspec());
  365   } else {
  366     pushl(as_Address(src));
  367   }
  368 }
  369 
  370 static void pass_arg0(MacroAssembler* masm, Register arg) {
  371   masm->push(arg);
  372 }
  373 
  374 static void pass_arg1(MacroAssembler* masm, Register arg) {
  375   masm->push(arg);
  376 }
  377 
  378 static void pass_arg2(MacroAssembler* masm, Register arg) {
  379   masm->push(arg);
  380 }
  381 
  382 static void pass_arg3(MacroAssembler* masm, Register arg) {
  383   masm->push(arg);
  384 }
  385 
  386 #ifndef PRODUCT
  387 extern "C" void findpc(intptr_t x);
  388 #endif
  389 
  390 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  391   // In order to get locks to work, we need to fake a in_VM state
  392   JavaThread* thread = JavaThread::current();
  393   JavaThreadState saved_state = thread->thread_state();
  394   thread->set_thread_state(_thread_in_vm);
  395   if (ShowMessageBoxOnError) {
  396     JavaThread* thread = JavaThread::current();
  397     JavaThreadState saved_state = thread->thread_state();
  398     thread->set_thread_state(_thread_in_vm);
  399     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  400       ttyLocker ttyl;
  401       BytecodeCounter::print();
  402     }
  403     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  404     // This is the value of eip which points to where verify_oop will return.
  405     if (os::message_box(msg, "Execution stopped, print registers?")) {
  406       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  407       BREAKPOINT;
  408     }
  409   }
  410   fatal("DEBUG MESSAGE: %s", msg);
  411 }
  412 
  413 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  414   ttyLocker ttyl;
  415   DebuggingContext debugging{};
  416   tty->print_cr("eip = 0x%08x", eip);
  417 #ifndef PRODUCT
  418   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  419     tty->cr();
  420     findpc(eip);
  421     tty->cr();
  422   }
  423 #endif
  424 #define PRINT_REG(rax) \
  425   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  426   PRINT_REG(rax);
  427   PRINT_REG(rbx);
  428   PRINT_REG(rcx);
  429   PRINT_REG(rdx);
  430   PRINT_REG(rdi);
  431   PRINT_REG(rsi);
  432   PRINT_REG(rbp);
  433   PRINT_REG(rsp);
  434 #undef PRINT_REG
  435   // Print some words near top of staack.
  436   int* dump_sp = (int*) rsp;
  437   for (int col1 = 0; col1 < 8; col1++) {
  438     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  439     os::print_location(tty, *dump_sp++);
  440   }
  441   for (int row = 0; row < 16; row++) {
  442     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  443     for (int col = 0; col < 8; col++) {
  444       tty->print(" 0x%08x", *dump_sp++);
  445     }
  446     tty->cr();
  447   }
  448   // Print some instructions around pc:
  449   Disassembler::decode((address)eip-64, (address)eip);
  450   tty->print_cr("--------");
  451   Disassembler::decode((address)eip, (address)eip+32);
  452 }
  453 
  454 void MacroAssembler::stop(const char* msg) {
  455   // push address of message
  456   ExternalAddress message((address)msg);
  457   pushptr(message.addr(), noreg);
  458   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  459   pusha();                                            // push registers
  460   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  461   hlt();
  462 }
  463 
  464 void MacroAssembler::warn(const char* msg) {
  465   push_CPU_state();
  466 
  467   // push address of message
  468   ExternalAddress message((address)msg);
  469   pushptr(message.addr(), noreg);
  470 
  471   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  472   addl(rsp, wordSize);       // discard argument
  473   pop_CPU_state();
  474 }
  475 
  476 void MacroAssembler::print_state() {
  477   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  478   pusha();                                            // push registers
  479 
  480   push_CPU_state();
  481   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  482   pop_CPU_state();
  483 
  484   popa();
  485   addl(rsp, wordSize);
  486 }
  487 
  488 #else // _LP64
  489 
  490 // 64 bit versions
  491 
  492 Address MacroAssembler::as_Address(AddressLiteral adr) {
  493   // amd64 always does this as a pc-rel
  494   // we can be absolute or disp based on the instruction type
  495   // jmp/call are displacements others are absolute
  496   assert(!adr.is_lval(), "must be rval");
  497   assert(reachable(adr), "must be");
  498   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  499 
  500 }
  501 
  502 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  503   AddressLiteral base = adr.base();
  504   lea(rscratch, base);
  505   Address index = adr.index();
  506   assert(index._disp == 0, "must not have disp"); // maybe it can?
  507   Address array(rscratch, index._index, index._scale, index._disp);
  508   return array;
  509 }
  510 
  511 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  512   Label L, E;
  513 
  514 #ifdef _WIN64
  515   // Windows always allocates space for it's register args
  516   assert(num_args <= 4, "only register arguments supported");
  517   subq(rsp,  frame::arg_reg_save_area_bytes);
  518 #endif
  519 
  520   // Align stack if necessary
  521   testl(rsp, 15);
  522   jcc(Assembler::zero, L);
  523 
  524   subq(rsp, 8);
  525   call(RuntimeAddress(entry_point));
  526   addq(rsp, 8);
  527   jmp(E);
  528 
  529   bind(L);
  530   call(RuntimeAddress(entry_point));
  531 
  532   bind(E);
  533 
  534 #ifdef _WIN64
  535   // restore stack pointer
  536   addq(rsp, frame::arg_reg_save_area_bytes);
  537 #endif
  538 }
  539 
  540 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  541   assert(!src2.is_lval(), "should use cmpptr");
  542   assert(rscratch != noreg || always_reachable(src2), "missing");
  543 
  544   if (reachable(src2)) {
  545     cmpq(src1, as_Address(src2));
  546   } else {
  547     lea(rscratch, src2);
  548     Assembler::cmpq(src1, Address(rscratch, 0));
  549   }
  550 }
  551 
  552 int MacroAssembler::corrected_idivq(Register reg) {
  553   // Full implementation of Java ldiv and lrem; checks for special
  554   // case as described in JVM spec., p.243 & p.271.  The function
  555   // returns the (pc) offset of the idivl instruction - may be needed
  556   // for implicit exceptions.
  557   //
  558   //         normal case                           special case
  559   //
  560   // input : rax: dividend                         min_long
  561   //         reg: divisor   (may not be eax/edx)   -1
  562   //
  563   // output: rax: quotient  (= rax idiv reg)       min_long
  564   //         rdx: remainder (= rax irem reg)       0
  565   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  566   static const int64_t min_long = 0x8000000000000000;
  567   Label normal_case, special_case;
  568 
  569   // check for special case
  570   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  571   jcc(Assembler::notEqual, normal_case);
  572   xorl(rdx, rdx); // prepare rdx for possible special case (where
  573                   // remainder = 0)
  574   cmpq(reg, -1);
  575   jcc(Assembler::equal, special_case);
  576 
  577   // handle normal case
  578   bind(normal_case);
  579   cdqq();
  580   int idivq_offset = offset();
  581   idivq(reg);
  582 
  583   // normal and special case exit
  584   bind(special_case);
  585 
  586   return idivq_offset;
  587 }
  588 
  589 void MacroAssembler::decrementq(Register reg, int value) {
  590   if (value == min_jint) { subq(reg, value); return; }
  591   if (value <  0) { incrementq(reg, -value); return; }
  592   if (value == 0) {                        ; return; }
  593   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  594   /* else */      { subq(reg, value)       ; return; }
  595 }
  596 
  597 void MacroAssembler::decrementq(Address dst, int value) {
  598   if (value == min_jint) { subq(dst, value); return; }
  599   if (value <  0) { incrementq(dst, -value); return; }
  600   if (value == 0) {                        ; return; }
  601   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  602   /* else */      { subq(dst, value)       ; return; }
  603 }
  604 
  605 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  606   assert(rscratch != noreg || always_reachable(dst), "missing");
  607 
  608   if (reachable(dst)) {
  609     incrementq(as_Address(dst));
  610   } else {
  611     lea(rscratch, dst);
  612     incrementq(Address(rscratch, 0));
  613   }
  614 }
  615 
  616 void MacroAssembler::incrementq(Register reg, int value) {
  617   if (value == min_jint) { addq(reg, value); return; }
  618   if (value <  0) { decrementq(reg, -value); return; }
  619   if (value == 0) {                        ; return; }
  620   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  621   /* else */      { addq(reg, value)       ; return; }
  622 }
  623 
  624 void MacroAssembler::incrementq(Address dst, int value) {
  625   if (value == min_jint) { addq(dst, value); return; }
  626   if (value <  0) { decrementq(dst, -value); return; }
  627   if (value == 0) {                        ; return; }
  628   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  629   /* else */      { addq(dst, value)       ; return; }
  630 }
  631 
  632 // 32bit can do a case table jump in one instruction but we no longer allow the base
  633 // to be installed in the Address class
  634 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  635   lea(rscratch, entry.base());
  636   Address dispatch = entry.index();
  637   assert(dispatch._base == noreg, "must be");
  638   dispatch._base = rscratch;
  639   jmp(dispatch);
  640 }
  641 
  642 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  643   ShouldNotReachHere(); // 64bit doesn't use two regs
  644   cmpq(x_lo, y_lo);
  645 }
  646 
  647 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  648   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  649 }
  650 
  651 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  652   lea(rscratch, adr);
  653   movptr(dst, rscratch);
  654 }
  655 
  656 void MacroAssembler::leave() {
  657   // %%% is this really better? Why not on 32bit too?
  658   emit_int8((unsigned char)0xC9); // LEAVE
  659 }
  660 
  661 void MacroAssembler::lneg(Register hi, Register lo) {
  662   ShouldNotReachHere(); // 64bit doesn't use two regs
  663   negq(lo);
  664 }
  665 
  666 void MacroAssembler::movoop(Register dst, jobject obj) {
  667   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  668 }
  669 
  670 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  671   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  672   movq(dst, rscratch);
  673 }
  674 
  675 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  676   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  677 }
  678 
  679 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  680   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  681   movq(dst, rscratch);
  682 }
  683 
  684 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  685   if (src.is_lval()) {
  686     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  687   } else {
  688     if (reachable(src)) {
  689       movq(dst, as_Address(src));
  690     } else {
  691       lea(dst, src);
  692       movq(dst, Address(dst, 0));
  693     }
  694   }
  695 }
  696 
  697 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  698   movq(as_Address(dst, rscratch), src);
  699 }
  700 
  701 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  702   movq(dst, as_Address(src, dst /*rscratch*/));
  703 }
  704 
  705 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  706 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  707   if (is_simm32(src)) {
  708     movptr(dst, checked_cast<int32_t>(src));
  709   } else {
  710     mov64(rscratch, src);
  711     movq(dst, rscratch);
  712   }
  713 }
  714 
  715 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  716   movoop(rscratch, obj);
  717   push(rscratch);
  718 }
  719 
  720 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  721   mov_metadata(rscratch, obj);
  722   push(rscratch);
  723 }
  724 
  725 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  726   lea(rscratch, src);
  727   if (src.is_lval()) {
  728     push(rscratch);
  729   } else {
  730     pushq(Address(rscratch, 0));
  731   }
  732 }
  733 
  734 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  735   reset_last_Java_frame(r15_thread, clear_fp);
  736 }
  737 
  738 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  739                                          Register last_java_fp,
  740                                          address  last_java_pc,
  741                                          Register rscratch) {
  742   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  743 }
  744 
  745 static void pass_arg0(MacroAssembler* masm, Register arg) {
  746   if (c_rarg0 != arg ) {
  747     masm->mov(c_rarg0, arg);
  748   }
  749 }
  750 
  751 static void pass_arg1(MacroAssembler* masm, Register arg) {
  752   if (c_rarg1 != arg ) {
  753     masm->mov(c_rarg1, arg);
  754   }
  755 }
  756 
  757 static void pass_arg2(MacroAssembler* masm, Register arg) {
  758   if (c_rarg2 != arg ) {
  759     masm->mov(c_rarg2, arg);
  760   }
  761 }
  762 
  763 static void pass_arg3(MacroAssembler* masm, Register arg) {
  764   if (c_rarg3 != arg ) {
  765     masm->mov(c_rarg3, arg);
  766   }
  767 }
  768 
  769 void MacroAssembler::stop(const char* msg) {
  770   if (ShowMessageBoxOnError) {
  771     address rip = pc();
  772     pusha(); // get regs on stack
  773     lea(c_rarg1, InternalAddress(rip));
  774     movq(c_rarg2, rsp); // pass pointer to regs array
  775   }
  776   lea(c_rarg0, ExternalAddress((address) msg));
  777   andq(rsp, -16); // align stack as required by ABI
  778   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  779   hlt();
  780 }
  781 
  782 void MacroAssembler::warn(const char* msg) {
  783   push(rbp);
  784   movq(rbp, rsp);
  785   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  786   push_CPU_state();   // keeps alignment at 16 bytes
  787 
  788 #ifdef _WIN64
  789   // Windows always allocates space for its register args
  790   subq(rsp,  frame::arg_reg_save_area_bytes);
  791 #endif
  792   lea(c_rarg0, ExternalAddress((address) msg));
  793   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  794 
  795 #ifdef _WIN64
  796   // restore stack pointer
  797   addq(rsp, frame::arg_reg_save_area_bytes);
  798 #endif
  799   pop_CPU_state();
  800   mov(rsp, rbp);
  801   pop(rbp);
  802 }
  803 
  804 void MacroAssembler::print_state() {
  805   address rip = pc();
  806   pusha();            // get regs on stack
  807   push(rbp);
  808   movq(rbp, rsp);
  809   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  810   push_CPU_state();   // keeps alignment at 16 bytes
  811 
  812   lea(c_rarg0, InternalAddress(rip));
  813   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  814   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  815 
  816   pop_CPU_state();
  817   mov(rsp, rbp);
  818   pop(rbp);
  819   popa();
  820 }
  821 
  822 #ifndef PRODUCT
  823 extern "C" void findpc(intptr_t x);
  824 #endif
  825 
  826 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  827   // In order to get locks to work, we need to fake a in_VM state
  828   if (ShowMessageBoxOnError) {
  829     JavaThread* thread = JavaThread::current();
  830     JavaThreadState saved_state = thread->thread_state();
  831     thread->set_thread_state(_thread_in_vm);
  832 #ifndef PRODUCT
  833     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  834       ttyLocker ttyl;
  835       BytecodeCounter::print();
  836     }
  837 #endif
  838     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  839     // XXX correct this offset for amd64
  840     // This is the value of eip which points to where verify_oop will return.
  841     if (os::message_box(msg, "Execution stopped, print registers?")) {
  842       print_state64(pc, regs);
  843       BREAKPOINT;
  844     }
  845   }
  846   fatal("DEBUG MESSAGE: %s", msg);
  847 }
  848 
  849 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  850   ttyLocker ttyl;
  851   DebuggingContext debugging{};
  852   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  853 #ifndef PRODUCT
  854   tty->cr();
  855   findpc(pc);
  856   tty->cr();
  857 #endif
  858 #define PRINT_REG(rax, value) \
  859   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  860   PRINT_REG(rax, regs[15]);
  861   PRINT_REG(rbx, regs[12]);
  862   PRINT_REG(rcx, regs[14]);
  863   PRINT_REG(rdx, regs[13]);
  864   PRINT_REG(rdi, regs[8]);
  865   PRINT_REG(rsi, regs[9]);
  866   PRINT_REG(rbp, regs[10]);
  867   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  868   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  869   PRINT_REG(r8 , regs[7]);
  870   PRINT_REG(r9 , regs[6]);
  871   PRINT_REG(r10, regs[5]);
  872   PRINT_REG(r11, regs[4]);
  873   PRINT_REG(r12, regs[3]);
  874   PRINT_REG(r13, regs[2]);
  875   PRINT_REG(r14, regs[1]);
  876   PRINT_REG(r15, regs[0]);
  877 #undef PRINT_REG
  878   // Print some words near the top of the stack.
  879   int64_t* rsp = &regs[16];
  880   int64_t* dump_sp = rsp;
  881   for (int col1 = 0; col1 < 8; col1++) {
  882     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  883     os::print_location(tty, *dump_sp++);
  884   }
  885   for (int row = 0; row < 25; row++) {
  886     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  887     for (int col = 0; col < 4; col++) {
  888       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  889     }
  890     tty->cr();
  891   }
  892   // Print some instructions around pc:
  893   Disassembler::decode((address)pc-64, (address)pc);
  894   tty->print_cr("--------");
  895   Disassembler::decode((address)pc, (address)pc+32);
  896 }
  897 
  898 // The java_calling_convention describes stack locations as ideal slots on
  899 // a frame with no abi restrictions. Since we must observe abi restrictions
  900 // (like the placement of the register window) the slots must be biased by
  901 // the following value.
  902 static int reg2offset_in(VMReg r) {
  903   // Account for saved rbp and return address
  904   // This should really be in_preserve_stack_slots
  905   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  906 }
  907 
  908 static int reg2offset_out(VMReg r) {
  909   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  910 }
  911 
  912 // A long move
  913 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  914 
  915   // The calling conventions assures us that each VMregpair is either
  916   // all really one physical register or adjacent stack slots.
  917 
  918   if (src.is_single_phys_reg() ) {
  919     if (dst.is_single_phys_reg()) {
  920       if (dst.first() != src.first()) {
  921         mov(dst.first()->as_Register(), src.first()->as_Register());
  922       }
  923     } else {
  924       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  925              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  926       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  927     }
  928   } else if (dst.is_single_phys_reg()) {
  929     assert(src.is_single_reg(),  "not a stack pair");
  930     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  931   } else {
  932     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  933     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  934     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  935   }
  936 }
  937 
  938 // A double move
  939 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  940 
  941   // The calling conventions assures us that each VMregpair is either
  942   // all really one physical register or adjacent stack slots.
  943 
  944   if (src.is_single_phys_reg() ) {
  945     if (dst.is_single_phys_reg()) {
  946       // In theory these overlap but the ordering is such that this is likely a nop
  947       if ( src.first() != dst.first()) {
  948         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  949       }
  950     } else {
  951       assert(dst.is_single_reg(), "not a stack pair");
  952       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  953     }
  954   } else if (dst.is_single_phys_reg()) {
  955     assert(src.is_single_reg(),  "not a stack pair");
  956     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  957   } else {
  958     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  959     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  960     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  961   }
  962 }
  963 
  964 
  965 // A float arg may have to do float reg int reg conversion
  966 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  967   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  968 
  969   // The calling conventions assures us that each VMregpair is either
  970   // all really one physical register or adjacent stack slots.
  971 
  972   if (src.first()->is_stack()) {
  973     if (dst.first()->is_stack()) {
  974       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  975       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  976     } else {
  977       // stack to reg
  978       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  979       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  980     }
  981   } else if (dst.first()->is_stack()) {
  982     // reg to stack
  983     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  984     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  985   } else {
  986     // reg to reg
  987     // In theory these overlap but the ordering is such that this is likely a nop
  988     if ( src.first() != dst.first()) {
  989       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  990     }
  991   }
  992 }
  993 
  994 // On 64 bit we will store integer like items to the stack as
  995 // 64 bits items (x86_32/64 abi) even though java would only store
  996 // 32bits for a parameter. On 32bit it will simply be 32 bits
  997 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  998 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  999   if (src.first()->is_stack()) {
 1000     if (dst.first()->is_stack()) {
 1001       // stack to stack
 1002       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 1003       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 1004     } else {
 1005       // stack to reg
 1006       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 1007     }
 1008   } else if (dst.first()->is_stack()) {
 1009     // reg to stack
 1010     // Do we really have to sign extend???
 1011     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 1012     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 1013   } else {
 1014     // Do we really have to sign extend???
 1015     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1016     if (dst.first() != src.first()) {
 1017       movq(dst.first()->as_Register(), src.first()->as_Register());
 1018     }
 1019   }
 1020 }
 1021 
 1022 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1023   if (src.first()->is_stack()) {
 1024     if (dst.first()->is_stack()) {
 1025       // stack to stack
 1026       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1027       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1028     } else {
 1029       // stack to reg
 1030       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1031     }
 1032   } else if (dst.first()->is_stack()) {
 1033     // reg to stack
 1034     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1035   } else {
 1036     if (dst.first() != src.first()) {
 1037       movq(dst.first()->as_Register(), src.first()->as_Register());
 1038     }
 1039   }
 1040 }
 1041 
 1042 // An oop arg. Must pass a handle not the oop itself
 1043 void MacroAssembler::object_move(OopMap* map,
 1044                         int oop_handle_offset,
 1045                         int framesize_in_slots,
 1046                         VMRegPair src,
 1047                         VMRegPair dst,
 1048                         bool is_receiver,
 1049                         int* receiver_offset) {
 1050 
 1051   // must pass a handle. First figure out the location we use as a handle
 1052 
 1053   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1054 
 1055   // See if oop is null if it is we need no handle
 1056 
 1057   if (src.first()->is_stack()) {
 1058 
 1059     // Oop is already on the stack as an argument
 1060     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1061     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1062     if (is_receiver) {
 1063       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1064     }
 1065 
 1066     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1067     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1068     // conditionally move a null
 1069     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1070   } else {
 1071 
 1072     // Oop is in a register we must store it to the space we reserve
 1073     // on the stack for oop_handles and pass a handle if oop is non-null
 1074 
 1075     const Register rOop = src.first()->as_Register();
 1076     int oop_slot;
 1077     if (rOop == j_rarg0)
 1078       oop_slot = 0;
 1079     else if (rOop == j_rarg1)
 1080       oop_slot = 1;
 1081     else if (rOop == j_rarg2)
 1082       oop_slot = 2;
 1083     else if (rOop == j_rarg3)
 1084       oop_slot = 3;
 1085     else if (rOop == j_rarg4)
 1086       oop_slot = 4;
 1087     else {
 1088       assert(rOop == j_rarg5, "wrong register");
 1089       oop_slot = 5;
 1090     }
 1091 
 1092     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1093     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1094 
 1095     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1096     // Store oop in handle area, may be null
 1097     movptr(Address(rsp, offset), rOop);
 1098     if (is_receiver) {
 1099       *receiver_offset = offset;
 1100     }
 1101 
 1102     cmpptr(rOop, NULL_WORD);
 1103     lea(rHandle, Address(rsp, offset));
 1104     // conditionally move a null from the handle area where it was just stored
 1105     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1106   }
 1107 
 1108   // If arg is on the stack then place it otherwise it is already in correct reg.
 1109   if (dst.first()->is_stack()) {
 1110     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1111   }
 1112 }
 1113 
 1114 #endif // _LP64
 1115 
 1116 // Now versions that are common to 32/64 bit
 1117 
 1118 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1119   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1120 }
 1121 
 1122 void MacroAssembler::addptr(Register dst, Register src) {
 1123   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1124 }
 1125 
 1126 void MacroAssembler::addptr(Address dst, Register src) {
 1127   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1128 }
 1129 
 1130 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1131   assert(rscratch != noreg || always_reachable(src), "missing");
 1132 
 1133   if (reachable(src)) {
 1134     Assembler::addsd(dst, as_Address(src));
 1135   } else {
 1136     lea(rscratch, src);
 1137     Assembler::addsd(dst, Address(rscratch, 0));
 1138   }
 1139 }
 1140 
 1141 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1142   assert(rscratch != noreg || always_reachable(src), "missing");
 1143 
 1144   if (reachable(src)) {
 1145     addss(dst, as_Address(src));
 1146   } else {
 1147     lea(rscratch, src);
 1148     addss(dst, Address(rscratch, 0));
 1149   }
 1150 }
 1151 
 1152 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1153   assert(rscratch != noreg || always_reachable(src), "missing");
 1154 
 1155   if (reachable(src)) {
 1156     Assembler::addpd(dst, as_Address(src));
 1157   } else {
 1158     lea(rscratch, src);
 1159     Assembler::addpd(dst, Address(rscratch, 0));
 1160   }
 1161 }
 1162 
 1163 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1164 // Stub code is generated once and never copied.
 1165 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1166 void MacroAssembler::align64() {
 1167   align(64, (uint)(uintptr_t)pc());
 1168 }
 1169 
 1170 void MacroAssembler::align32() {
 1171   align(32, (uint)(uintptr_t)pc());
 1172 }
 1173 
 1174 void MacroAssembler::align(uint modulus) {
 1175   // 8273459: Ensure alignment is possible with current segment alignment
 1176   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1177   align(modulus, offset());
 1178 }
 1179 
 1180 void MacroAssembler::align(uint modulus, uint target) {
 1181   if (target % modulus != 0) {
 1182     nop(modulus - (target % modulus));
 1183   }
 1184 }
 1185 
 1186 void MacroAssembler::push_f(XMMRegister r) {
 1187   subptr(rsp, wordSize);
 1188   movflt(Address(rsp, 0), r);
 1189 }
 1190 
 1191 void MacroAssembler::pop_f(XMMRegister r) {
 1192   movflt(r, Address(rsp, 0));
 1193   addptr(rsp, wordSize);
 1194 }
 1195 
 1196 void MacroAssembler::push_d(XMMRegister r) {
 1197   subptr(rsp, 2 * wordSize);
 1198   movdbl(Address(rsp, 0), r);
 1199 }
 1200 
 1201 void MacroAssembler::pop_d(XMMRegister r) {
 1202   movdbl(r, Address(rsp, 0));
 1203   addptr(rsp, 2 * Interpreter::stackElementSize);
 1204 }
 1205 
 1206 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1207   // Used in sign-masking with aligned address.
 1208   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1209   assert(rscratch != noreg || always_reachable(src), "missing");
 1210 
 1211   if (UseAVX > 2 &&
 1212       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 1213       (dst->encoding() >= 16)) {
 1214     vpand(dst, dst, src, AVX_512bit, rscratch);
 1215   } else if (reachable(src)) {
 1216     Assembler::andpd(dst, as_Address(src));
 1217   } else {
 1218     lea(rscratch, src);
 1219     Assembler::andpd(dst, Address(rscratch, 0));
 1220   }
 1221 }
 1222 
 1223 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1224   // Used in sign-masking with aligned address.
 1225   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1226   assert(rscratch != noreg || always_reachable(src), "missing");
 1227 
 1228   if (reachable(src)) {
 1229     Assembler::andps(dst, as_Address(src));
 1230   } else {
 1231     lea(rscratch, src);
 1232     Assembler::andps(dst, Address(rscratch, 0));
 1233   }
 1234 }
 1235 
 1236 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1237   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1238 }
 1239 
 1240 #ifdef _LP64
 1241 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1242   assert(rscratch != noreg || always_reachable(src), "missing");
 1243 
 1244   if (reachable(src)) {
 1245     andq(dst, as_Address(src));
 1246   } else {
 1247     lea(rscratch, src);
 1248     andq(dst, Address(rscratch, 0));
 1249   }
 1250 }
 1251 #endif
 1252 
 1253 void MacroAssembler::atomic_incl(Address counter_addr) {
 1254   lock();
 1255   incrementl(counter_addr);
 1256 }
 1257 
 1258 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1259   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1260 
 1261   if (reachable(counter_addr)) {
 1262     atomic_incl(as_Address(counter_addr));
 1263   } else {
 1264     lea(rscratch, counter_addr);
 1265     atomic_incl(Address(rscratch, 0));
 1266   }
 1267 }
 1268 
 1269 #ifdef _LP64
 1270 void MacroAssembler::atomic_incq(Address counter_addr) {
 1271   lock();
 1272   incrementq(counter_addr);
 1273 }
 1274 
 1275 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1276   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1277 
 1278   if (reachable(counter_addr)) {
 1279     atomic_incq(as_Address(counter_addr));
 1280   } else {
 1281     lea(rscratch, counter_addr);
 1282     atomic_incq(Address(rscratch, 0));
 1283   }
 1284 }
 1285 #endif
 1286 
 1287 // Writes to stack successive pages until offset reached to check for
 1288 // stack overflow + shadow pages.  This clobbers tmp.
 1289 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1290   movptr(tmp, rsp);
 1291   // Bang stack for total size given plus shadow page size.
 1292   // Bang one page at a time because large size can bang beyond yellow and
 1293   // red zones.
 1294   Label loop;
 1295   bind(loop);
 1296   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1297   subptr(tmp, (int)os::vm_page_size());
 1298   subl(size, (int)os::vm_page_size());
 1299   jcc(Assembler::greater, loop);
 1300 
 1301   // Bang down shadow pages too.
 1302   // At this point, (tmp-0) is the last address touched, so don't
 1303   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1304   // was post-decremented.)  Skip this address by starting at i=1, and
 1305   // touch a few more pages below.  N.B.  It is important to touch all
 1306   // the way down including all pages in the shadow zone.
 1307   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1308     // this could be any sized move but this is can be a debugging crumb
 1309     // so the bigger the better.
 1310     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1311   }
 1312 }
 1313 
 1314 void MacroAssembler::reserved_stack_check() {
 1315   // testing if reserved zone needs to be enabled
 1316   Label no_reserved_zone_enabling;
 1317   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1318   NOT_LP64(get_thread(rsi);)
 1319 
 1320   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1321   jcc(Assembler::below, no_reserved_zone_enabling);
 1322 
 1323   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1324   jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
 1325   should_not_reach_here();
 1326 
 1327   bind(no_reserved_zone_enabling);
 1328 }
 1329 
 1330 void MacroAssembler::c2bool(Register x) {
 1331   // implements x == 0 ? 0 : 1
 1332   // note: must only look at least-significant byte of x
 1333   //       since C-style booleans are stored in one byte
 1334   //       only! (was bug)
 1335   andl(x, 0xFF);
 1336   setb(Assembler::notZero, x);
 1337 }
 1338 
 1339 // Wouldn't need if AddressLiteral version had new name
 1340 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1341   Assembler::call(L, rtype);
 1342 }
 1343 
 1344 void MacroAssembler::call(Register entry) {
 1345   Assembler::call(entry);
 1346 }
 1347 
 1348 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1349   assert(rscratch != noreg || always_reachable(entry), "missing");
 1350 
 1351   if (reachable(entry)) {
 1352     Assembler::call_literal(entry.target(), entry.rspec());
 1353   } else {
 1354     lea(rscratch, entry);
 1355     Assembler::call(rscratch);
 1356   }
 1357 }
 1358 
 1359 void MacroAssembler::ic_call(address entry, jint method_index) {
 1360   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1361 #ifdef _LP64
 1362   // Needs full 64-bit immediate for later patching.
 1363   mov64(rax, (int64_t)Universe::non_oop_word());
 1364 #else
 1365   movptr(rax, (intptr_t)Universe::non_oop_word());
 1366 #endif
 1367   call(AddressLiteral(entry, rh));
 1368 }
 1369 
 1370 int MacroAssembler::ic_check_size() {
 1371   return
 1372       LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12);
 1373 }
 1374 
 1375 int MacroAssembler::ic_check(int end_alignment) {
 1376   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1377   Register data = rax;
 1378   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1379 
 1380   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1381   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1382   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1383   // before the inline cache check here, and not after
 1384   align(end_alignment, offset() + ic_check_size());
 1385 
 1386   int uep_offset = offset();
 1387 
 1388 #ifdef _LP64
 1389   if (UseCompactObjectHeaders) {
 1390     load_narrow_klass_compact(temp, receiver);
 1391     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1392   } else
 1393 #endif
 1394   if (UseCompressedClassPointers) {
 1395     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1396     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1397   } else {
 1398     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1399     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1400   }
 1401 
 1402   // if inline cache check fails, then jump to runtime routine
 1403   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1404   assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
 1405 
 1406   return uep_offset;
 1407 }
 1408 
 1409 void MacroAssembler::emit_static_call_stub() {
 1410   // Static stub relocation also tags the Method* in the code-stream.
 1411   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1412   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1413   jump(RuntimeAddress(pc()));
 1414 }
 1415 
 1416 // Implementation of call_VM versions
 1417 
 1418 void MacroAssembler::call_VM(Register oop_result,
 1419                              address entry_point,
 1420                              bool check_exceptions) {
 1421   Label C, E;
 1422   call(C, relocInfo::none);
 1423   jmp(E);
 1424 
 1425   bind(C);
 1426   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1427   ret(0);
 1428 
 1429   bind(E);
 1430 }
 1431 
 1432 void MacroAssembler::call_VM(Register oop_result,
 1433                              address entry_point,
 1434                              Register arg_1,
 1435                              bool check_exceptions) {
 1436   Label C, E;
 1437   call(C, relocInfo::none);
 1438   jmp(E);
 1439 
 1440   bind(C);
 1441   pass_arg1(this, arg_1);
 1442   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1443   ret(0);
 1444 
 1445   bind(E);
 1446 }
 1447 
 1448 void MacroAssembler::call_VM(Register oop_result,
 1449                              address entry_point,
 1450                              Register arg_1,
 1451                              Register arg_2,
 1452                              bool check_exceptions) {
 1453   Label C, E;
 1454   call(C, relocInfo::none);
 1455   jmp(E);
 1456 
 1457   bind(C);
 1458 
 1459   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1460 
 1461   pass_arg2(this, arg_2);
 1462   pass_arg1(this, arg_1);
 1463   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1464   ret(0);
 1465 
 1466   bind(E);
 1467 }
 1468 
 1469 void MacroAssembler::call_VM(Register oop_result,
 1470                              address entry_point,
 1471                              Register arg_1,
 1472                              Register arg_2,
 1473                              Register arg_3,
 1474                              bool check_exceptions) {
 1475   Label C, E;
 1476   call(C, relocInfo::none);
 1477   jmp(E);
 1478 
 1479   bind(C);
 1480 
 1481   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1482   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1483   pass_arg3(this, arg_3);
 1484   pass_arg2(this, arg_2);
 1485   pass_arg1(this, arg_1);
 1486   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1487   ret(0);
 1488 
 1489   bind(E);
 1490 }
 1491 
 1492 void MacroAssembler::call_VM(Register oop_result,
 1493                              Register last_java_sp,
 1494                              address entry_point,
 1495                              int number_of_arguments,
 1496                              bool check_exceptions) {
 1497   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1498   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1499 }
 1500 
 1501 void MacroAssembler::call_VM(Register oop_result,
 1502                              Register last_java_sp,
 1503                              address entry_point,
 1504                              Register arg_1,
 1505                              bool check_exceptions) {
 1506   pass_arg1(this, arg_1);
 1507   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1508 }
 1509 
 1510 void MacroAssembler::call_VM(Register oop_result,
 1511                              Register last_java_sp,
 1512                              address entry_point,
 1513                              Register arg_1,
 1514                              Register arg_2,
 1515                              bool check_exceptions) {
 1516 
 1517   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1518   pass_arg2(this, arg_2);
 1519   pass_arg1(this, arg_1);
 1520   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1521 }
 1522 
 1523 void MacroAssembler::call_VM(Register oop_result,
 1524                              Register last_java_sp,
 1525                              address entry_point,
 1526                              Register arg_1,
 1527                              Register arg_2,
 1528                              Register arg_3,
 1529                              bool check_exceptions) {
 1530   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1531   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1532   pass_arg3(this, arg_3);
 1533   pass_arg2(this, arg_2);
 1534   pass_arg1(this, arg_1);
 1535   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1536 }
 1537 
 1538 void MacroAssembler::super_call_VM(Register oop_result,
 1539                                    Register last_java_sp,
 1540                                    address entry_point,
 1541                                    int number_of_arguments,
 1542                                    bool check_exceptions) {
 1543   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1544   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1545 }
 1546 
 1547 void MacroAssembler::super_call_VM(Register oop_result,
 1548                                    Register last_java_sp,
 1549                                    address entry_point,
 1550                                    Register arg_1,
 1551                                    bool check_exceptions) {
 1552   pass_arg1(this, arg_1);
 1553   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1554 }
 1555 
 1556 void MacroAssembler::super_call_VM(Register oop_result,
 1557                                    Register last_java_sp,
 1558                                    address entry_point,
 1559                                    Register arg_1,
 1560                                    Register arg_2,
 1561                                    bool check_exceptions) {
 1562 
 1563   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1564   pass_arg2(this, arg_2);
 1565   pass_arg1(this, arg_1);
 1566   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1567 }
 1568 
 1569 void MacroAssembler::super_call_VM(Register oop_result,
 1570                                    Register last_java_sp,
 1571                                    address entry_point,
 1572                                    Register arg_1,
 1573                                    Register arg_2,
 1574                                    Register arg_3,
 1575                                    bool check_exceptions) {
 1576   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1577   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1578   pass_arg3(this, arg_3);
 1579   pass_arg2(this, arg_2);
 1580   pass_arg1(this, arg_1);
 1581   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1582 }
 1583 
 1584 void MacroAssembler::call_VM_base(Register oop_result,
 1585                                   Register java_thread,
 1586                                   Register last_java_sp,
 1587                                   address  entry_point,
 1588                                   int      number_of_arguments,
 1589                                   bool     check_exceptions) {
 1590   // determine java_thread register
 1591   if (!java_thread->is_valid()) {
 1592 #ifdef _LP64
 1593     java_thread = r15_thread;
 1594 #else
 1595     java_thread = rdi;
 1596     get_thread(java_thread);
 1597 #endif // LP64
 1598   }
 1599   // determine last_java_sp register
 1600   if (!last_java_sp->is_valid()) {
 1601     last_java_sp = rsp;
 1602   }
 1603   // debugging support
 1604   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1605   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1606 #ifdef ASSERT
 1607   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1608   // r12 is the heapbase.
 1609   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1610 #endif // ASSERT
 1611 
 1612   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1613   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1614 
 1615   // push java thread (becomes first argument of C function)
 1616 
 1617   NOT_LP64(push(java_thread); number_of_arguments++);
 1618   LP64_ONLY(mov(c_rarg0, r15_thread));
 1619 
 1620   // set last Java frame before call
 1621   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1622 
 1623   // Only interpreter should have to set fp
 1624   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1625 
 1626   // do the call, remove parameters
 1627   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1628 
 1629   // restore the thread (cannot use the pushed argument since arguments
 1630   // may be overwritten by C code generated by an optimizing compiler);
 1631   // however can use the register value directly if it is callee saved.
 1632   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1633     // rdi & rsi (also r15) are callee saved -> nothing to do
 1634 #ifdef ASSERT
 1635     guarantee(java_thread != rax, "change this code");
 1636     push(rax);
 1637     { Label L;
 1638       get_thread(rax);
 1639       cmpptr(java_thread, rax);
 1640       jcc(Assembler::equal, L);
 1641       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1642       bind(L);
 1643     }
 1644     pop(rax);
 1645 #endif
 1646   } else {
 1647     get_thread(java_thread);
 1648   }
 1649   // reset last Java frame
 1650   // Only interpreter should have to clear fp
 1651   reset_last_Java_frame(java_thread, true);
 1652 
 1653    // C++ interp handles this in the interpreter
 1654   check_and_handle_popframe(java_thread);
 1655   check_and_handle_earlyret(java_thread);
 1656 
 1657   if (check_exceptions) {
 1658     // check for pending exceptions (java_thread is set upon return)
 1659     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1660 #ifndef _LP64
 1661     jump_cc(Assembler::notEqual,
 1662             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1663 #else
 1664     // This used to conditionally jump to forward_exception however it is
 1665     // possible if we relocate that the branch will not reach. So we must jump
 1666     // around so we can always reach
 1667 
 1668     Label ok;
 1669     jcc(Assembler::equal, ok);
 1670     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1671     bind(ok);
 1672 #endif // LP64
 1673   }
 1674 
 1675   // get oop result if there is one and reset the value in the thread
 1676   if (oop_result->is_valid()) {
 1677     get_vm_result(oop_result, java_thread);
 1678   }
 1679 }
 1680 
 1681 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1682 
 1683   // Calculate the value for last_Java_sp
 1684   // somewhat subtle. call_VM does an intermediate call
 1685   // which places a return address on the stack just under the
 1686   // stack pointer as the user finished with it. This allows
 1687   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1688   // On 32bit we then have to push additional args on the stack to accomplish
 1689   // the actual requested call. On 64bit call_VM only can use register args
 1690   // so the only extra space is the return address that call_VM created.
 1691   // This hopefully explains the calculations here.
 1692 
 1693 #ifdef _LP64
 1694   // We've pushed one address, correct last_Java_sp
 1695   lea(rax, Address(rsp, wordSize));
 1696 #else
 1697   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1698 #endif // LP64
 1699 
 1700   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1701 
 1702 }
 1703 
 1704 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1705 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1706   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1707 }
 1708 
 1709 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1710   call_VM_leaf_base(entry_point, number_of_arguments);
 1711 }
 1712 
 1713 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1714   pass_arg0(this, arg_0);
 1715   call_VM_leaf(entry_point, 1);
 1716 }
 1717 
 1718 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1719 
 1720   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1721   pass_arg1(this, arg_1);
 1722   pass_arg0(this, arg_0);
 1723   call_VM_leaf(entry_point, 2);
 1724 }
 1725 
 1726 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1727   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1728   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1729   pass_arg2(this, arg_2);
 1730   pass_arg1(this, arg_1);
 1731   pass_arg0(this, arg_0);
 1732   call_VM_leaf(entry_point, 3);
 1733 }
 1734 
 1735 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1736   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1737   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1738   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1739   pass_arg3(this, arg_3);
 1740   pass_arg2(this, arg_2);
 1741   pass_arg1(this, arg_1);
 1742   pass_arg0(this, arg_0);
 1743   call_VM_leaf(entry_point, 3);
 1744 }
 1745 
 1746 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 1747   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1748 }
 1749 
 1750 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1751   pass_arg0(this, arg_0);
 1752   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1753 }
 1754 
 1755 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1756   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1757   pass_arg1(this, arg_1);
 1758   pass_arg0(this, arg_0);
 1759   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1760 }
 1761 
 1762 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1763   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1764   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1765   pass_arg2(this, arg_2);
 1766   pass_arg1(this, arg_1);
 1767   pass_arg0(this, arg_0);
 1768   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1769 }
 1770 
 1771 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1772   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1773   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1774   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1775   pass_arg3(this, arg_3);
 1776   pass_arg2(this, arg_2);
 1777   pass_arg1(this, arg_1);
 1778   pass_arg0(this, arg_0);
 1779   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1780 }
 1781 
 1782 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1783   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1784   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1785   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1786 }
 1787 
 1788 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1789   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1790   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1791 }
 1792 
 1793 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1794 }
 1795 
 1796 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1797 }
 1798 
 1799 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1800   assert(rscratch != noreg || always_reachable(src1), "missing");
 1801 
 1802   if (reachable(src1)) {
 1803     cmpl(as_Address(src1), imm);
 1804   } else {
 1805     lea(rscratch, src1);
 1806     cmpl(Address(rscratch, 0), imm);
 1807   }
 1808 }
 1809 
 1810 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1811   assert(!src2.is_lval(), "use cmpptr");
 1812   assert(rscratch != noreg || always_reachable(src2), "missing");
 1813 
 1814   if (reachable(src2)) {
 1815     cmpl(src1, as_Address(src2));
 1816   } else {
 1817     lea(rscratch, src2);
 1818     cmpl(src1, Address(rscratch, 0));
 1819   }
 1820 }
 1821 
 1822 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1823   Assembler::cmpl(src1, imm);
 1824 }
 1825 
 1826 void MacroAssembler::cmp32(Register src1, Address src2) {
 1827   Assembler::cmpl(src1, src2);
 1828 }
 1829 
 1830 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1831   ucomisd(opr1, opr2);
 1832 
 1833   Label L;
 1834   if (unordered_is_less) {
 1835     movl(dst, -1);
 1836     jcc(Assembler::parity, L);
 1837     jcc(Assembler::below , L);
 1838     movl(dst, 0);
 1839     jcc(Assembler::equal , L);
 1840     increment(dst);
 1841   } else { // unordered is greater
 1842     movl(dst, 1);
 1843     jcc(Assembler::parity, L);
 1844     jcc(Assembler::above , L);
 1845     movl(dst, 0);
 1846     jcc(Assembler::equal , L);
 1847     decrementl(dst);
 1848   }
 1849   bind(L);
 1850 }
 1851 
 1852 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1853   ucomiss(opr1, opr2);
 1854 
 1855   Label L;
 1856   if (unordered_is_less) {
 1857     movl(dst, -1);
 1858     jcc(Assembler::parity, L);
 1859     jcc(Assembler::below , L);
 1860     movl(dst, 0);
 1861     jcc(Assembler::equal , L);
 1862     increment(dst);
 1863   } else { // unordered is greater
 1864     movl(dst, 1);
 1865     jcc(Assembler::parity, L);
 1866     jcc(Assembler::above , L);
 1867     movl(dst, 0);
 1868     jcc(Assembler::equal , L);
 1869     decrementl(dst);
 1870   }
 1871   bind(L);
 1872 }
 1873 
 1874 
 1875 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1876   assert(rscratch != noreg || always_reachable(src1), "missing");
 1877 
 1878   if (reachable(src1)) {
 1879     cmpb(as_Address(src1), imm);
 1880   } else {
 1881     lea(rscratch, src1);
 1882     cmpb(Address(rscratch, 0), imm);
 1883   }
 1884 }
 1885 
 1886 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1887 #ifdef _LP64
 1888   assert(rscratch != noreg || always_reachable(src2), "missing");
 1889 
 1890   if (src2.is_lval()) {
 1891     movptr(rscratch, src2);
 1892     Assembler::cmpq(src1, rscratch);
 1893   } else if (reachable(src2)) {
 1894     cmpq(src1, as_Address(src2));
 1895   } else {
 1896     lea(rscratch, src2);
 1897     Assembler::cmpq(src1, Address(rscratch, 0));
 1898   }
 1899 #else
 1900   assert(rscratch == noreg, "not needed");
 1901   if (src2.is_lval()) {
 1902     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1903   } else {
 1904     cmpl(src1, as_Address(src2));
 1905   }
 1906 #endif // _LP64
 1907 }
 1908 
 1909 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1910   assert(src2.is_lval(), "not a mem-mem compare");
 1911 #ifdef _LP64
 1912   // moves src2's literal address
 1913   movptr(rscratch, src2);
 1914   Assembler::cmpq(src1, rscratch);
 1915 #else
 1916   assert(rscratch == noreg, "not needed");
 1917   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1918 #endif // _LP64
 1919 }
 1920 
 1921 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1922   cmpptr(src1, src2);
 1923 }
 1924 
 1925 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1926   cmpptr(src1, src2);
 1927 }
 1928 
 1929 #ifdef _LP64
 1930 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1931   movoop(rscratch, src2);
 1932   cmpptr(src1, rscratch);
 1933 }
 1934 #endif
 1935 
 1936 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1937   assert(rscratch != noreg || always_reachable(adr), "missing");
 1938 
 1939   if (reachable(adr)) {
 1940     lock();
 1941     cmpxchgptr(reg, as_Address(adr));
 1942   } else {
 1943     lea(rscratch, adr);
 1944     lock();
 1945     cmpxchgptr(reg, Address(rscratch, 0));
 1946   }
 1947 }
 1948 
 1949 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1950   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1951 }
 1952 
 1953 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1954   assert(rscratch != noreg || always_reachable(src), "missing");
 1955 
 1956   if (reachable(src)) {
 1957     Assembler::comisd(dst, as_Address(src));
 1958   } else {
 1959     lea(rscratch, src);
 1960     Assembler::comisd(dst, Address(rscratch, 0));
 1961   }
 1962 }
 1963 
 1964 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1965   assert(rscratch != noreg || always_reachable(src), "missing");
 1966 
 1967   if (reachable(src)) {
 1968     Assembler::comiss(dst, as_Address(src));
 1969   } else {
 1970     lea(rscratch, src);
 1971     Assembler::comiss(dst, Address(rscratch, 0));
 1972   }
 1973 }
 1974 
 1975 
 1976 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1977   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1978 
 1979   Condition negated_cond = negate_condition(cond);
 1980   Label L;
 1981   jcc(negated_cond, L);
 1982   pushf(); // Preserve flags
 1983   atomic_incl(counter_addr, rscratch);
 1984   popf();
 1985   bind(L);
 1986 }
 1987 
 1988 int MacroAssembler::corrected_idivl(Register reg) {
 1989   // Full implementation of Java idiv and irem; checks for
 1990   // special case as described in JVM spec., p.243 & p.271.
 1991   // The function returns the (pc) offset of the idivl
 1992   // instruction - may be needed for implicit exceptions.
 1993   //
 1994   //         normal case                           special case
 1995   //
 1996   // input : rax,: dividend                         min_int
 1997   //         reg: divisor   (may not be rax,/rdx)   -1
 1998   //
 1999   // output: rax,: quotient  (= rax, idiv reg)       min_int
 2000   //         rdx: remainder (= rax, irem reg)       0
 2001   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 2002   const int min_int = 0x80000000;
 2003   Label normal_case, special_case;
 2004 
 2005   // check for special case
 2006   cmpl(rax, min_int);
 2007   jcc(Assembler::notEqual, normal_case);
 2008   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 2009   cmpl(reg, -1);
 2010   jcc(Assembler::equal, special_case);
 2011 
 2012   // handle normal case
 2013   bind(normal_case);
 2014   cdql();
 2015   int idivl_offset = offset();
 2016   idivl(reg);
 2017 
 2018   // normal and special case exit
 2019   bind(special_case);
 2020 
 2021   return idivl_offset;
 2022 }
 2023 
 2024 
 2025 
 2026 void MacroAssembler::decrementl(Register reg, int value) {
 2027   if (value == min_jint) {subl(reg, value) ; return; }
 2028   if (value <  0) { incrementl(reg, -value); return; }
 2029   if (value == 0) {                        ; return; }
 2030   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2031   /* else */      { subl(reg, value)       ; return; }
 2032 }
 2033 
 2034 void MacroAssembler::decrementl(Address dst, int value) {
 2035   if (value == min_jint) {subl(dst, value) ; return; }
 2036   if (value <  0) { incrementl(dst, -value); return; }
 2037   if (value == 0) {                        ; return; }
 2038   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2039   /* else */      { subl(dst, value)       ; return; }
 2040 }
 2041 
 2042 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2043   assert(shift_value > 0, "illegal shift value");
 2044   Label _is_positive;
 2045   testl (reg, reg);
 2046   jcc (Assembler::positive, _is_positive);
 2047   int offset = (1 << shift_value) - 1 ;
 2048 
 2049   if (offset == 1) {
 2050     incrementl(reg);
 2051   } else {
 2052     addl(reg, offset);
 2053   }
 2054 
 2055   bind (_is_positive);
 2056   sarl(reg, shift_value);
 2057 }
 2058 
 2059 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2060   assert(rscratch != noreg || always_reachable(src), "missing");
 2061 
 2062   if (reachable(src)) {
 2063     Assembler::divsd(dst, as_Address(src));
 2064   } else {
 2065     lea(rscratch, src);
 2066     Assembler::divsd(dst, Address(rscratch, 0));
 2067   }
 2068 }
 2069 
 2070 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2071   assert(rscratch != noreg || always_reachable(src), "missing");
 2072 
 2073   if (reachable(src)) {
 2074     Assembler::divss(dst, as_Address(src));
 2075   } else {
 2076     lea(rscratch, src);
 2077     Assembler::divss(dst, Address(rscratch, 0));
 2078   }
 2079 }
 2080 
 2081 void MacroAssembler::enter() {
 2082   push(rbp);
 2083   mov(rbp, rsp);
 2084 }
 2085 
 2086 void MacroAssembler::post_call_nop() {
 2087   if (!Continuations::enabled()) {
 2088     return;
 2089   }
 2090   InstructionMark im(this);
 2091   relocate(post_call_nop_Relocation::spec());
 2092   InlineSkippedInstructionsCounter skipCounter(this);
 2093   emit_int8((uint8_t)0x0f);
 2094   emit_int8((uint8_t)0x1f);
 2095   emit_int8((uint8_t)0x84);
 2096   emit_int8((uint8_t)0x00);
 2097   emit_int32(0x00);
 2098 }
 2099 
 2100 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2101 void MacroAssembler::fat_nop() {
 2102   if (UseAddressNop) {
 2103     addr_nop_5();
 2104   } else {
 2105     emit_int8((uint8_t)0x26); // es:
 2106     emit_int8((uint8_t)0x2e); // cs:
 2107     emit_int8((uint8_t)0x64); // fs:
 2108     emit_int8((uint8_t)0x65); // gs:
 2109     emit_int8((uint8_t)0x90);
 2110   }
 2111 }
 2112 
 2113 #ifndef _LP64
 2114 void MacroAssembler::fcmp(Register tmp) {
 2115   fcmp(tmp, 1, true, true);
 2116 }
 2117 
 2118 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2119   assert(!pop_right || pop_left, "usage error");
 2120   if (VM_Version::supports_cmov()) {
 2121     assert(tmp == noreg, "unneeded temp");
 2122     if (pop_left) {
 2123       fucomip(index);
 2124     } else {
 2125       fucomi(index);
 2126     }
 2127     if (pop_right) {
 2128       fpop();
 2129     }
 2130   } else {
 2131     assert(tmp != noreg, "need temp");
 2132     if (pop_left) {
 2133       if (pop_right) {
 2134         fcompp();
 2135       } else {
 2136         fcomp(index);
 2137       }
 2138     } else {
 2139       fcom(index);
 2140     }
 2141     // convert FPU condition into eflags condition via rax,
 2142     save_rax(tmp);
 2143     fwait(); fnstsw_ax();
 2144     sahf();
 2145     restore_rax(tmp);
 2146   }
 2147   // condition codes set as follows:
 2148   //
 2149   // CF (corresponds to C0) if x < y
 2150   // PF (corresponds to C2) if unordered
 2151   // ZF (corresponds to C3) if x = y
 2152 }
 2153 
 2154 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2155   fcmp2int(dst, unordered_is_less, 1, true, true);
 2156 }
 2157 
 2158 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2159   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2160   Label L;
 2161   if (unordered_is_less) {
 2162     movl(dst, -1);
 2163     jcc(Assembler::parity, L);
 2164     jcc(Assembler::below , L);
 2165     movl(dst, 0);
 2166     jcc(Assembler::equal , L);
 2167     increment(dst);
 2168   } else { // unordered is greater
 2169     movl(dst, 1);
 2170     jcc(Assembler::parity, L);
 2171     jcc(Assembler::above , L);
 2172     movl(dst, 0);
 2173     jcc(Assembler::equal , L);
 2174     decrementl(dst);
 2175   }
 2176   bind(L);
 2177 }
 2178 
 2179 void MacroAssembler::fld_d(AddressLiteral src) {
 2180   fld_d(as_Address(src));
 2181 }
 2182 
 2183 void MacroAssembler::fld_s(AddressLiteral src) {
 2184   fld_s(as_Address(src));
 2185 }
 2186 
 2187 void MacroAssembler::fldcw(AddressLiteral src) {
 2188   fldcw(as_Address(src));
 2189 }
 2190 
 2191 void MacroAssembler::fpop() {
 2192   ffree();
 2193   fincstp();
 2194 }
 2195 
 2196 void MacroAssembler::fremr(Register tmp) {
 2197   save_rax(tmp);
 2198   { Label L;
 2199     bind(L);
 2200     fprem();
 2201     fwait(); fnstsw_ax();
 2202     sahf();
 2203     jcc(Assembler::parity, L);
 2204   }
 2205   restore_rax(tmp);
 2206   // Result is in ST0.
 2207   // Note: fxch & fpop to get rid of ST1
 2208   // (otherwise FPU stack could overflow eventually)
 2209   fxch(1);
 2210   fpop();
 2211 }
 2212 
 2213 void MacroAssembler::empty_FPU_stack() {
 2214   if (VM_Version::supports_mmx()) {
 2215     emms();
 2216   } else {
 2217     for (int i = 8; i-- > 0; ) ffree(i);
 2218   }
 2219 }
 2220 #endif // !LP64
 2221 
 2222 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2223   assert(rscratch != noreg || always_reachable(src), "missing");
 2224   if (reachable(src)) {
 2225     Assembler::mulpd(dst, as_Address(src));
 2226   } else {
 2227     lea(rscratch, src);
 2228     Assembler::mulpd(dst, Address(rscratch, 0));
 2229   }
 2230 }
 2231 
 2232 void MacroAssembler::load_float(Address src) {
 2233 #ifdef _LP64
 2234   movflt(xmm0, src);
 2235 #else
 2236   if (UseSSE >= 1) {
 2237     movflt(xmm0, src);
 2238   } else {
 2239     fld_s(src);
 2240   }
 2241 #endif // LP64
 2242 }
 2243 
 2244 void MacroAssembler::store_float(Address dst) {
 2245 #ifdef _LP64
 2246   movflt(dst, xmm0);
 2247 #else
 2248   if (UseSSE >= 1) {
 2249     movflt(dst, xmm0);
 2250   } else {
 2251     fstp_s(dst);
 2252   }
 2253 #endif // LP64
 2254 }
 2255 
 2256 void MacroAssembler::load_double(Address src) {
 2257 #ifdef _LP64
 2258   movdbl(xmm0, src);
 2259 #else
 2260   if (UseSSE >= 2) {
 2261     movdbl(xmm0, src);
 2262   } else {
 2263     fld_d(src);
 2264   }
 2265 #endif // LP64
 2266 }
 2267 
 2268 void MacroAssembler::store_double(Address dst) {
 2269 #ifdef _LP64
 2270   movdbl(dst, xmm0);
 2271 #else
 2272   if (UseSSE >= 2) {
 2273     movdbl(dst, xmm0);
 2274   } else {
 2275     fstp_d(dst);
 2276   }
 2277 #endif // LP64
 2278 }
 2279 
 2280 // dst = c = a * b + c
 2281 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2282   Assembler::vfmadd231sd(c, a, b);
 2283   if (dst != c) {
 2284     movdbl(dst, c);
 2285   }
 2286 }
 2287 
 2288 // dst = c = a * b + c
 2289 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2290   Assembler::vfmadd231ss(c, a, b);
 2291   if (dst != c) {
 2292     movflt(dst, c);
 2293   }
 2294 }
 2295 
 2296 // dst = c = a * b + c
 2297 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2298   Assembler::vfmadd231pd(c, a, b, vector_len);
 2299   if (dst != c) {
 2300     vmovdqu(dst, c);
 2301   }
 2302 }
 2303 
 2304 // dst = c = a * b + c
 2305 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2306   Assembler::vfmadd231ps(c, a, b, vector_len);
 2307   if (dst != c) {
 2308     vmovdqu(dst, c);
 2309   }
 2310 }
 2311 
 2312 // dst = c = a * b + c
 2313 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2314   Assembler::vfmadd231pd(c, a, b, vector_len);
 2315   if (dst != c) {
 2316     vmovdqu(dst, c);
 2317   }
 2318 }
 2319 
 2320 // dst = c = a * b + c
 2321 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2322   Assembler::vfmadd231ps(c, a, b, vector_len);
 2323   if (dst != c) {
 2324     vmovdqu(dst, c);
 2325   }
 2326 }
 2327 
 2328 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2329   assert(rscratch != noreg || always_reachable(dst), "missing");
 2330 
 2331   if (reachable(dst)) {
 2332     incrementl(as_Address(dst));
 2333   } else {
 2334     lea(rscratch, dst);
 2335     incrementl(Address(rscratch, 0));
 2336   }
 2337 }
 2338 
 2339 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2340   incrementl(as_Address(dst, rscratch));
 2341 }
 2342 
 2343 void MacroAssembler::incrementl(Register reg, int value) {
 2344   if (value == min_jint) {addl(reg, value) ; return; }
 2345   if (value <  0) { decrementl(reg, -value); return; }
 2346   if (value == 0) {                        ; return; }
 2347   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2348   /* else */      { addl(reg, value)       ; return; }
 2349 }
 2350 
 2351 void MacroAssembler::incrementl(Address dst, int value) {
 2352   if (value == min_jint) {addl(dst, value) ; return; }
 2353   if (value <  0) { decrementl(dst, -value); return; }
 2354   if (value == 0) {                        ; return; }
 2355   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2356   /* else */      { addl(dst, value)       ; return; }
 2357 }
 2358 
 2359 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2360   assert(rscratch != noreg || always_reachable(dst), "missing");
 2361   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump");
 2362   if (reachable(dst)) {
 2363     jmp_literal(dst.target(), dst.rspec());
 2364   } else {
 2365     lea(rscratch, dst);
 2366     jmp(rscratch);
 2367   }
 2368 }
 2369 
 2370 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2371   assert(rscratch != noreg || always_reachable(dst), "missing");
 2372   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc");
 2373   if (reachable(dst)) {
 2374     InstructionMark im(this);
 2375     relocate(dst.reloc());
 2376     const int short_size = 2;
 2377     const int long_size = 6;
 2378     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2379     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2380       // 0111 tttn #8-bit disp
 2381       emit_int8(0x70 | cc);
 2382       emit_int8((offs - short_size) & 0xFF);
 2383     } else {
 2384       // 0000 1111 1000 tttn #32-bit disp
 2385       emit_int8(0x0F);
 2386       emit_int8((unsigned char)(0x80 | cc));
 2387       emit_int32(offs - long_size);
 2388     }
 2389   } else {
 2390 #ifdef ASSERT
 2391     warning("reversing conditional branch");
 2392 #endif /* ASSERT */
 2393     Label skip;
 2394     jccb(reverse[cc], skip);
 2395     lea(rscratch, dst);
 2396     Assembler::jmp(rscratch);
 2397     bind(skip);
 2398   }
 2399 }
 2400 
 2401 void MacroAssembler::cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch) {
 2402   ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
 2403   assert(rscratch != noreg || always_reachable(mxcsr_std), "missing");
 2404 
 2405   stmxcsr(mxcsr_save);
 2406   movl(tmp, mxcsr_save);
 2407   if (EnableX86ECoreOpts) {
 2408     // The mxcsr_std has status bits set for performance on ECore
 2409     orl(tmp, 0x003f);
 2410   } else {
 2411     // Mask out status bits (only check control and mask bits)
 2412     andl(tmp, 0xFFC0);
 2413   }
 2414   cmp32(tmp, mxcsr_std, rscratch);
 2415 }
 2416 
 2417 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2418   assert(rscratch != noreg || always_reachable(src), "missing");
 2419 
 2420   if (reachable(src)) {
 2421     Assembler::ldmxcsr(as_Address(src));
 2422   } else {
 2423     lea(rscratch, src);
 2424     Assembler::ldmxcsr(Address(rscratch, 0));
 2425   }
 2426 }
 2427 
 2428 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2429   int off;
 2430   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2431     off = offset();
 2432     movsbl(dst, src); // movsxb
 2433   } else {
 2434     off = load_unsigned_byte(dst, src);
 2435     shll(dst, 24);
 2436     sarl(dst, 24);
 2437   }
 2438   return off;
 2439 }
 2440 
 2441 // Note: load_signed_short used to be called load_signed_word.
 2442 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2443 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2444 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2445 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2446   int off;
 2447   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2448     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2449     // version but this is what 64bit has always done. This seems to imply
 2450     // that users are only using 32bits worth.
 2451     off = offset();
 2452     movswl(dst, src); // movsxw
 2453   } else {
 2454     off = load_unsigned_short(dst, src);
 2455     shll(dst, 16);
 2456     sarl(dst, 16);
 2457   }
 2458   return off;
 2459 }
 2460 
 2461 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2462   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2463   // and "3.9 Partial Register Penalties", p. 22).
 2464   int off;
 2465   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2466     off = offset();
 2467     movzbl(dst, src); // movzxb
 2468   } else {
 2469     xorl(dst, dst);
 2470     off = offset();
 2471     movb(dst, src);
 2472   }
 2473   return off;
 2474 }
 2475 
 2476 // Note: load_unsigned_short used to be called load_unsigned_word.
 2477 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2478   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2479   // and "3.9 Partial Register Penalties", p. 22).
 2480   int off;
 2481   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2482     off = offset();
 2483     movzwl(dst, src); // movzxw
 2484   } else {
 2485     xorl(dst, dst);
 2486     off = offset();
 2487     movw(dst, src);
 2488   }
 2489   return off;
 2490 }
 2491 
 2492 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2493   switch (size_in_bytes) {
 2494 #ifndef _LP64
 2495   case  8:
 2496     assert(dst2 != noreg, "second dest register required");
 2497     movl(dst,  src);
 2498     movl(dst2, src.plus_disp(BytesPerInt));
 2499     break;
 2500 #else
 2501   case  8:  movq(dst, src); break;
 2502 #endif
 2503   case  4:  movl(dst, src); break;
 2504   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2505   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2506   default:  ShouldNotReachHere();
 2507   }
 2508 }
 2509 
 2510 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2511   switch (size_in_bytes) {
 2512 #ifndef _LP64
 2513   case  8:
 2514     assert(src2 != noreg, "second source register required");
 2515     movl(dst,                        src);
 2516     movl(dst.plus_disp(BytesPerInt), src2);
 2517     break;
 2518 #else
 2519   case  8:  movq(dst, src); break;
 2520 #endif
 2521   case  4:  movl(dst, src); break;
 2522   case  2:  movw(dst, src); break;
 2523   case  1:  movb(dst, src); break;
 2524   default:  ShouldNotReachHere();
 2525   }
 2526 }
 2527 
 2528 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2529   assert(rscratch != noreg || always_reachable(dst), "missing");
 2530 
 2531   if (reachable(dst)) {
 2532     movl(as_Address(dst), src);
 2533   } else {
 2534     lea(rscratch, dst);
 2535     movl(Address(rscratch, 0), src);
 2536   }
 2537 }
 2538 
 2539 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2540   if (reachable(src)) {
 2541     movl(dst, as_Address(src));
 2542   } else {
 2543     lea(dst, src);
 2544     movl(dst, Address(dst, 0));
 2545   }
 2546 }
 2547 
 2548 // C++ bool manipulation
 2549 
 2550 void MacroAssembler::movbool(Register dst, Address src) {
 2551   if(sizeof(bool) == 1)
 2552     movb(dst, src);
 2553   else if(sizeof(bool) == 2)
 2554     movw(dst, src);
 2555   else if(sizeof(bool) == 4)
 2556     movl(dst, src);
 2557   else
 2558     // unsupported
 2559     ShouldNotReachHere();
 2560 }
 2561 
 2562 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2563   if(sizeof(bool) == 1)
 2564     movb(dst, (int) boolconst);
 2565   else if(sizeof(bool) == 2)
 2566     movw(dst, (int) boolconst);
 2567   else if(sizeof(bool) == 4)
 2568     movl(dst, (int) boolconst);
 2569   else
 2570     // unsupported
 2571     ShouldNotReachHere();
 2572 }
 2573 
 2574 void MacroAssembler::movbool(Address dst, Register src) {
 2575   if(sizeof(bool) == 1)
 2576     movb(dst, src);
 2577   else if(sizeof(bool) == 2)
 2578     movw(dst, src);
 2579   else if(sizeof(bool) == 4)
 2580     movl(dst, src);
 2581   else
 2582     // unsupported
 2583     ShouldNotReachHere();
 2584 }
 2585 
 2586 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2587   assert(rscratch != noreg || always_reachable(src), "missing");
 2588 
 2589   if (reachable(src)) {
 2590     movdl(dst, as_Address(src));
 2591   } else {
 2592     lea(rscratch, src);
 2593     movdl(dst, Address(rscratch, 0));
 2594   }
 2595 }
 2596 
 2597 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2598   assert(rscratch != noreg || always_reachable(src), "missing");
 2599 
 2600   if (reachable(src)) {
 2601     movq(dst, as_Address(src));
 2602   } else {
 2603     lea(rscratch, src);
 2604     movq(dst, Address(rscratch, 0));
 2605   }
 2606 }
 2607 
 2608 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2609   assert(rscratch != noreg || always_reachable(src), "missing");
 2610 
 2611   if (reachable(src)) {
 2612     if (UseXmmLoadAndClearUpper) {
 2613       movsd (dst, as_Address(src));
 2614     } else {
 2615       movlpd(dst, as_Address(src));
 2616     }
 2617   } else {
 2618     lea(rscratch, src);
 2619     if (UseXmmLoadAndClearUpper) {
 2620       movsd (dst, Address(rscratch, 0));
 2621     } else {
 2622       movlpd(dst, Address(rscratch, 0));
 2623     }
 2624   }
 2625 }
 2626 
 2627 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2628   assert(rscratch != noreg || always_reachable(src), "missing");
 2629 
 2630   if (reachable(src)) {
 2631     movss(dst, as_Address(src));
 2632   } else {
 2633     lea(rscratch, src);
 2634     movss(dst, Address(rscratch, 0));
 2635   }
 2636 }
 2637 
 2638 void MacroAssembler::movptr(Register dst, Register src) {
 2639   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2640 }
 2641 
 2642 void MacroAssembler::movptr(Register dst, Address src) {
 2643   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2644 }
 2645 
 2646 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2647 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2648 #ifdef _LP64
 2649   if (is_uimm32(src)) {
 2650     movl(dst, checked_cast<uint32_t>(src));
 2651   } else if (is_simm32(src)) {
 2652     movq(dst, checked_cast<int32_t>(src));
 2653   } else {
 2654     mov64(dst, src);
 2655   }
 2656 #else
 2657   movl(dst, src);
 2658 #endif
 2659 }
 2660 
 2661 void MacroAssembler::movptr(Address dst, Register src) {
 2662   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2663 }
 2664 
 2665 void MacroAssembler::movptr(Address dst, int32_t src) {
 2666   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2667 }
 2668 
 2669 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2670   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2671   Assembler::movdqu(dst, src);
 2672 }
 2673 
 2674 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2675   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2676   Assembler::movdqu(dst, src);
 2677 }
 2678 
 2679 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2680   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2681   Assembler::movdqu(dst, src);
 2682 }
 2683 
 2684 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2685   assert(rscratch != noreg || always_reachable(src), "missing");
 2686 
 2687   if (reachable(src)) {
 2688     movdqu(dst, as_Address(src));
 2689   } else {
 2690     lea(rscratch, src);
 2691     movdqu(dst, Address(rscratch, 0));
 2692   }
 2693 }
 2694 
 2695 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2696   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2697   Assembler::vmovdqu(dst, src);
 2698 }
 2699 
 2700 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2701   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2702   Assembler::vmovdqu(dst, src);
 2703 }
 2704 
 2705 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2706   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2707   Assembler::vmovdqu(dst, src);
 2708 }
 2709 
 2710 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2711   assert(rscratch != noreg || always_reachable(src), "missing");
 2712 
 2713   if (reachable(src)) {
 2714     vmovdqu(dst, as_Address(src));
 2715   }
 2716   else {
 2717     lea(rscratch, src);
 2718     vmovdqu(dst, Address(rscratch, 0));
 2719   }
 2720 }
 2721 
 2722 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2723   assert(rscratch != noreg || always_reachable(src), "missing");
 2724 
 2725   if (vector_len == AVX_512bit) {
 2726     evmovdquq(dst, src, AVX_512bit, rscratch);
 2727   } else if (vector_len == AVX_256bit) {
 2728     vmovdqu(dst, src, rscratch);
 2729   } else {
 2730     movdqu(dst, src, rscratch);
 2731   }
 2732 }
 2733 
 2734 void MacroAssembler::kmov(KRegister dst, Address src) {
 2735   if (VM_Version::supports_avx512bw()) {
 2736     kmovql(dst, src);
 2737   } else {
 2738     assert(VM_Version::supports_evex(), "");
 2739     kmovwl(dst, src);
 2740   }
 2741 }
 2742 
 2743 void MacroAssembler::kmov(Address dst, KRegister src) {
 2744   if (VM_Version::supports_avx512bw()) {
 2745     kmovql(dst, src);
 2746   } else {
 2747     assert(VM_Version::supports_evex(), "");
 2748     kmovwl(dst, src);
 2749   }
 2750 }
 2751 
 2752 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2753   if (VM_Version::supports_avx512bw()) {
 2754     kmovql(dst, src);
 2755   } else {
 2756     assert(VM_Version::supports_evex(), "");
 2757     kmovwl(dst, src);
 2758   }
 2759 }
 2760 
 2761 void MacroAssembler::kmov(Register dst, KRegister src) {
 2762   if (VM_Version::supports_avx512bw()) {
 2763     kmovql(dst, src);
 2764   } else {
 2765     assert(VM_Version::supports_evex(), "");
 2766     kmovwl(dst, src);
 2767   }
 2768 }
 2769 
 2770 void MacroAssembler::kmov(KRegister dst, Register src) {
 2771   if (VM_Version::supports_avx512bw()) {
 2772     kmovql(dst, src);
 2773   } else {
 2774     assert(VM_Version::supports_evex(), "");
 2775     kmovwl(dst, src);
 2776   }
 2777 }
 2778 
 2779 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2780   assert(rscratch != noreg || always_reachable(src), "missing");
 2781 
 2782   if (reachable(src)) {
 2783     kmovql(dst, as_Address(src));
 2784   } else {
 2785     lea(rscratch, src);
 2786     kmovql(dst, Address(rscratch, 0));
 2787   }
 2788 }
 2789 
 2790 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2791   assert(rscratch != noreg || always_reachable(src), "missing");
 2792 
 2793   if (reachable(src)) {
 2794     kmovwl(dst, as_Address(src));
 2795   } else {
 2796     lea(rscratch, src);
 2797     kmovwl(dst, Address(rscratch, 0));
 2798   }
 2799 }
 2800 
 2801 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2802                                int vector_len, Register rscratch) {
 2803   assert(rscratch != noreg || always_reachable(src), "missing");
 2804 
 2805   if (reachable(src)) {
 2806     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2807   } else {
 2808     lea(rscratch, src);
 2809     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2810   }
 2811 }
 2812 
 2813 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2814                                int vector_len, Register rscratch) {
 2815   assert(rscratch != noreg || always_reachable(src), "missing");
 2816 
 2817   if (reachable(src)) {
 2818     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2819   } else {
 2820     lea(rscratch, src);
 2821     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2822   }
 2823 }
 2824 
 2825 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2826   assert(rscratch != noreg || always_reachable(src), "missing");
 2827 
 2828   if (reachable(src)) {
 2829     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2830   } else {
 2831     lea(rscratch, src);
 2832     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2833   }
 2834 }
 2835 
 2836 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2837   assert(rscratch != noreg || always_reachable(src), "missing");
 2838 
 2839   if (reachable(src)) {
 2840     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2841   } else {
 2842     lea(rscratch, src);
 2843     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2844   }
 2845 }
 2846 
 2847 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2848   assert(rscratch != noreg || always_reachable(src), "missing");
 2849 
 2850   if (reachable(src)) {
 2851     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2852   } else {
 2853     lea(rscratch, src);
 2854     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2855   }
 2856 }
 2857 
 2858 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2859   assert(rscratch != noreg || always_reachable(src), "missing");
 2860 
 2861   if (reachable(src)) {
 2862     Assembler::movdqa(dst, as_Address(src));
 2863   } else {
 2864     lea(rscratch, src);
 2865     Assembler::movdqa(dst, Address(rscratch, 0));
 2866   }
 2867 }
 2868 
 2869 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2870   assert(rscratch != noreg || always_reachable(src), "missing");
 2871 
 2872   if (reachable(src)) {
 2873     Assembler::movsd(dst, as_Address(src));
 2874   } else {
 2875     lea(rscratch, src);
 2876     Assembler::movsd(dst, Address(rscratch, 0));
 2877   }
 2878 }
 2879 
 2880 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2881   assert(rscratch != noreg || always_reachable(src), "missing");
 2882 
 2883   if (reachable(src)) {
 2884     Assembler::movss(dst, as_Address(src));
 2885   } else {
 2886     lea(rscratch, src);
 2887     Assembler::movss(dst, Address(rscratch, 0));
 2888   }
 2889 }
 2890 
 2891 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2892   assert(rscratch != noreg || always_reachable(src), "missing");
 2893 
 2894   if (reachable(src)) {
 2895     Assembler::movddup(dst, as_Address(src));
 2896   } else {
 2897     lea(rscratch, src);
 2898     Assembler::movddup(dst, Address(rscratch, 0));
 2899   }
 2900 }
 2901 
 2902 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2903   assert(rscratch != noreg || always_reachable(src), "missing");
 2904 
 2905   if (reachable(src)) {
 2906     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2907   } else {
 2908     lea(rscratch, src);
 2909     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2910   }
 2911 }
 2912 
 2913 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2914   assert(rscratch != noreg || always_reachable(src), "missing");
 2915 
 2916   if (reachable(src)) {
 2917     Assembler::mulsd(dst, as_Address(src));
 2918   } else {
 2919     lea(rscratch, src);
 2920     Assembler::mulsd(dst, Address(rscratch, 0));
 2921   }
 2922 }
 2923 
 2924 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2925   assert(rscratch != noreg || always_reachable(src), "missing");
 2926 
 2927   if (reachable(src)) {
 2928     Assembler::mulss(dst, as_Address(src));
 2929   } else {
 2930     lea(rscratch, src);
 2931     Assembler::mulss(dst, Address(rscratch, 0));
 2932   }
 2933 }
 2934 
 2935 void MacroAssembler::null_check(Register reg, int offset) {
 2936   if (needs_explicit_null_check(offset)) {
 2937     // provoke OS null exception if reg is null by
 2938     // accessing M[reg] w/o changing any (non-CC) registers
 2939     // NOTE: cmpl is plenty here to provoke a segv
 2940     cmpptr(rax, Address(reg, 0));
 2941     // Note: should probably use testl(rax, Address(reg, 0));
 2942     //       may be shorter code (however, this version of
 2943     //       testl needs to be implemented first)
 2944   } else {
 2945     // nothing to do, (later) access of M[reg + offset]
 2946     // will provoke OS null exception if reg is null
 2947   }
 2948 }
 2949 
 2950 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
 2951   andptr(markword, markWord::inline_type_mask_in_place);
 2952   cmpptr(markword, markWord::inline_type_pattern);
 2953   jcc(Assembler::equal, is_inline_type);
 2954 }
 2955 
 2956 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
 2957   load_unsigned_short(temp_reg, Address(klass, Klass::access_flags_offset()));
 2958   testl(temp_reg, JVM_ACC_IDENTITY);
 2959   jcc(Assembler::zero, is_inline_type);
 2960 }
 2961 
 2962 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
 2963   testptr(object, object);
 2964   jcc(Assembler::zero, not_inline_type);
 2965   const int is_inline_type_mask = markWord::inline_type_pattern;
 2966   movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
 2967   andptr(tmp, is_inline_type_mask);
 2968   cmpptr(tmp, is_inline_type_mask);
 2969   jcc(Assembler::notEqual, not_inline_type);
 2970 }
 2971 
 2972 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
 2973 #ifdef ASSERT
 2974   {
 2975     Label done_check;
 2976     test_klass_is_inline_type(klass, temp_reg, done_check);
 2977     stop("test_klass_is_empty_inline_type with non inline type klass");
 2978     bind(done_check);
 2979   }
 2980 #endif
 2981   movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
 2982   testl(temp_reg, InstanceKlassFlags::is_empty_inline_type_value());
 2983   jcc(Assembler::notZero, is_empty_inline_type);
 2984 }
 2985 
 2986 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
 2987   movl(temp_reg, flags);
 2988   testl(temp_reg, 1 << ResolvedFieldEntry::is_null_free_inline_type_shift);
 2989   jcc(Assembler::notEqual, is_null_free_inline_type);
 2990 }
 2991 
 2992 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
 2993   movl(temp_reg, flags);
 2994   testl(temp_reg, 1 << ResolvedFieldEntry::is_null_free_inline_type_shift);
 2995   jcc(Assembler::equal, not_null_free_inline_type);
 2996 }
 2997 
 2998 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
 2999   movl(temp_reg, flags);
 3000   testl(temp_reg, 1 << ResolvedFieldEntry::is_flat_shift);
 3001   jcc(Assembler::notEqual, is_flat);
 3002 }
 3003 
 3004 void MacroAssembler::test_field_has_null_marker(Register flags, Register temp_reg, Label& has_null_marker) {
 3005   movl(temp_reg, flags);
 3006   testl(temp_reg, 1 << ResolvedFieldEntry::has_null_marker_shift);
 3007   jcc(Assembler::notEqual, has_null_marker);
 3008 }
 3009 
 3010 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
 3011   Label test_mark_word;
 3012   // load mark word
 3013   movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
 3014   // check displaced
 3015   testl(temp_reg, markWord::unlocked_value);
 3016   jccb(Assembler::notZero, test_mark_word);
 3017   // slow path use klass prototype
 3018   push(rscratch1);
 3019   load_prototype_header(temp_reg, oop, rscratch1);
 3020   pop(rscratch1);
 3021 
 3022   bind(test_mark_word);
 3023   testl(temp_reg, test_bit);
 3024   jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
 3025 }
 3026 
 3027 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
 3028                                          Label& is_flat_array) {
 3029 #ifdef _LP64
 3030   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
 3031 #else
 3032   load_klass(temp_reg, oop, noreg);
 3033   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3034   test_flat_array_layout(temp_reg, is_flat_array);
 3035 #endif
 3036 }
 3037 
 3038 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg,
 3039                                              Label& is_non_flat_array) {
 3040 #ifdef _LP64
 3041   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
 3042 #else
 3043   load_klass(temp_reg, oop, noreg);
 3044   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3045   test_non_flat_array_layout(temp_reg, is_non_flat_array);
 3046 #endif
 3047 }
 3048 
 3049 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
 3050 #ifdef _LP64
 3051   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
 3052 #else
 3053   Unimplemented();
 3054 #endif
 3055 }
 3056 
 3057 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
 3058 #ifdef _LP64
 3059   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
 3060 #else
 3061   Unimplemented();
 3062 #endif
 3063 }
 3064 
 3065 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
 3066   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3067   jcc(Assembler::notZero, is_flat_array);
 3068 }
 3069 
 3070 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) {
 3071   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3072   jcc(Assembler::zero, is_non_flat_array);
 3073 }
 3074 
 3075 void MacroAssembler::os_breakpoint() {
 3076   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 3077   // (e.g., MSVC can't call ps() otherwise)
 3078   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 3079 }
 3080 
 3081 void MacroAssembler::unimplemented(const char* what) {
 3082   const char* buf = nullptr;
 3083   {
 3084     ResourceMark rm;
 3085     stringStream ss;
 3086     ss.print("unimplemented: %s", what);
 3087     buf = code_string(ss.as_string());
 3088   }
 3089   stop(buf);
 3090 }
 3091 
 3092 #ifdef _LP64
 3093 #define XSTATE_BV 0x200
 3094 #endif
 3095 
 3096 void MacroAssembler::pop_CPU_state() {
 3097   pop_FPU_state();
 3098   pop_IU_state();
 3099 }
 3100 
 3101 void MacroAssembler::pop_FPU_state() {
 3102 #ifndef _LP64
 3103   frstor(Address(rsp, 0));
 3104 #else
 3105   fxrstor(Address(rsp, 0));
 3106 #endif
 3107   addptr(rsp, FPUStateSizeInWords * wordSize);
 3108 }
 3109 
 3110 void MacroAssembler::pop_IU_state() {
 3111   popa();
 3112   LP64_ONLY(addq(rsp, 8));
 3113   popf();
 3114 }
 3115 
 3116 // Save Integer and Float state
 3117 // Warning: Stack must be 16 byte aligned (64bit)
 3118 void MacroAssembler::push_CPU_state() {
 3119   push_IU_state();
 3120   push_FPU_state();
 3121 }
 3122 
 3123 void MacroAssembler::push_FPU_state() {
 3124   subptr(rsp, FPUStateSizeInWords * wordSize);
 3125 #ifndef _LP64
 3126   fnsave(Address(rsp, 0));
 3127   fwait();
 3128 #else
 3129   fxsave(Address(rsp, 0));
 3130 #endif // LP64
 3131 }
 3132 
 3133 void MacroAssembler::push_IU_state() {
 3134   // Push flags first because pusha kills them
 3135   pushf();
 3136   // Make sure rsp stays 16-byte aligned
 3137   LP64_ONLY(subq(rsp, 8));
 3138   pusha();
 3139 }
 3140 
 3141 void MacroAssembler::push_cont_fastpath() {
 3142   if (!Continuations::enabled()) return;
 3143 
 3144 #ifndef _LP64
 3145   Register rthread = rax;
 3146   Register rrealsp = rbx;
 3147   push(rthread);
 3148   push(rrealsp);
 3149 
 3150   get_thread(rthread);
 3151 
 3152   // The code below wants the original RSP.
 3153   // Move it back after the pushes above.
 3154   movptr(rrealsp, rsp);
 3155   addptr(rrealsp, 2*wordSize);
 3156 #else
 3157   Register rthread = r15_thread;
 3158   Register rrealsp = rsp;
 3159 #endif
 3160 
 3161   Label done;
 3162   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3163   jccb(Assembler::belowEqual, done);
 3164   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3165   bind(done);
 3166 
 3167 #ifndef _LP64
 3168   pop(rrealsp);
 3169   pop(rthread);
 3170 #endif
 3171 }
 3172 
 3173 void MacroAssembler::pop_cont_fastpath() {
 3174   if (!Continuations::enabled()) return;
 3175 
 3176 #ifndef _LP64
 3177   Register rthread = rax;
 3178   Register rrealsp = rbx;
 3179   push(rthread);
 3180   push(rrealsp);
 3181 
 3182   get_thread(rthread);
 3183 
 3184   // The code below wants the original RSP.
 3185   // Move it back after the pushes above.
 3186   movptr(rrealsp, rsp);
 3187   addptr(rrealsp, 2*wordSize);
 3188 #else
 3189   Register rthread = r15_thread;
 3190   Register rrealsp = rsp;
 3191 #endif
 3192 
 3193   Label done;
 3194   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3195   jccb(Assembler::below, done);
 3196   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3197   bind(done);
 3198 
 3199 #ifndef _LP64
 3200   pop(rrealsp);
 3201   pop(rthread);
 3202 #endif
 3203 }
 3204 
 3205 void MacroAssembler::inc_held_monitor_count() {
 3206 #ifdef _LP64
 3207   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3208 #endif
 3209 }
 3210 
 3211 void MacroAssembler::dec_held_monitor_count() {
 3212 #ifdef _LP64
 3213   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3214 #endif
 3215 }
 3216 
 3217 #ifdef ASSERT
 3218 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3219 #ifdef _LP64
 3220   Label no_cont;
 3221   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3222   testl(cont, cont);
 3223   jcc(Assembler::zero, no_cont);
 3224   stop(name);
 3225   bind(no_cont);
 3226 #else
 3227   Unimplemented();
 3228 #endif
 3229 }
 3230 #endif
 3231 
 3232 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3233   if (!java_thread->is_valid()) {
 3234     java_thread = rdi;
 3235     get_thread(java_thread);
 3236   }
 3237   // we must set sp to zero to clear frame
 3238   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3239   // must clear fp, so that compiled frames are not confused; it is
 3240   // possible that we need it only for debugging
 3241   if (clear_fp) {
 3242     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3243   }
 3244   // Always clear the pc because it could have been set by make_walkable()
 3245   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3246   vzeroupper();
 3247 }
 3248 
 3249 void MacroAssembler::restore_rax(Register tmp) {
 3250   if (tmp == noreg) pop(rax);
 3251   else if (tmp != rax) mov(rax, tmp);
 3252 }
 3253 
 3254 void MacroAssembler::round_to(Register reg, int modulus) {
 3255   addptr(reg, modulus - 1);
 3256   andptr(reg, -modulus);
 3257 }
 3258 
 3259 void MacroAssembler::save_rax(Register tmp) {
 3260   if (tmp == noreg) push(rax);
 3261   else if (tmp != rax) mov(tmp, rax);
 3262 }
 3263 
 3264 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3265   if (at_return) {
 3266     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3267     // we may safely use rsp instead to perform the stack watermark check.
 3268     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3269     jcc(Assembler::above, slow_path);
 3270     return;
 3271   }
 3272   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3273   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3274 }
 3275 
 3276 // Calls to C land
 3277 //
 3278 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3279 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3280 // has to be reset to 0. This is required to allow proper stack traversal.
 3281 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3282                                          Register last_java_sp,
 3283                                          Register last_java_fp,
 3284                                          address  last_java_pc,
 3285                                          Register rscratch) {
 3286   vzeroupper();
 3287   // determine java_thread register
 3288   if (!java_thread->is_valid()) {
 3289     java_thread = rdi;
 3290     get_thread(java_thread);
 3291   }
 3292   // determine last_java_sp register
 3293   if (!last_java_sp->is_valid()) {
 3294     last_java_sp = rsp;
 3295   }
 3296   // last_java_fp is optional
 3297   if (last_java_fp->is_valid()) {
 3298     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3299   }
 3300   // last_java_pc is optional
 3301   if (last_java_pc != nullptr) {
 3302     Address java_pc(java_thread,
 3303                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3304     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3305   }
 3306   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3307 }
 3308 
 3309 #ifdef _LP64
 3310 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 3311                                          Register last_java_fp,
 3312                                          Label &L,
 3313                                          Register scratch) {
 3314   lea(scratch, L);
 3315   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch);
 3316   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, nullptr, scratch);
 3317 }
 3318 #endif
 3319 
 3320 void MacroAssembler::shlptr(Register dst, int imm8) {
 3321   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3322 }
 3323 
 3324 void MacroAssembler::shrptr(Register dst, int imm8) {
 3325   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3326 }
 3327 
 3328 void MacroAssembler::sign_extend_byte(Register reg) {
 3329   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3330     movsbl(reg, reg); // movsxb
 3331   } else {
 3332     shll(reg, 24);
 3333     sarl(reg, 24);
 3334   }
 3335 }
 3336 
 3337 void MacroAssembler::sign_extend_short(Register reg) {
 3338   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3339     movswl(reg, reg); // movsxw
 3340   } else {
 3341     shll(reg, 16);
 3342     sarl(reg, 16);
 3343   }
 3344 }
 3345 
 3346 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3347   if (imm32 >= 0 && is8bit(imm32)) {
 3348     testb(dst, imm32);
 3349   } else {
 3350     Assembler::testl(dst, imm32);
 3351   }
 3352 }
 3353 
 3354 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3355   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3356     testb(dst, imm32);
 3357   } else {
 3358     Assembler::testl(dst, imm32);
 3359   }
 3360 }
 3361 
 3362 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3363   assert(always_reachable(src), "Address should be reachable");
 3364   testl(dst, as_Address(src));
 3365 }
 3366 
 3367 #ifdef _LP64
 3368 
 3369 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3370   if (imm32 >= 0) {
 3371     testl(dst, imm32);
 3372   } else {
 3373     Assembler::testq(dst, imm32);
 3374   }
 3375 }
 3376 
 3377 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3378   if (imm32 >= 0) {
 3379     testl(dst, imm32);
 3380   } else {
 3381     Assembler::testq(dst, imm32);
 3382   }
 3383 }
 3384 
 3385 #endif
 3386 
 3387 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3388   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3389   Assembler::pcmpeqb(dst, src);
 3390 }
 3391 
 3392 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3393   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3394   Assembler::pcmpeqw(dst, src);
 3395 }
 3396 
 3397 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3398   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3399   Assembler::pcmpestri(dst, src, imm8);
 3400 }
 3401 
 3402 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3403   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3404   Assembler::pcmpestri(dst, src, imm8);
 3405 }
 3406 
 3407 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3408   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3409   Assembler::pmovzxbw(dst, src);
 3410 }
 3411 
 3412 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3413   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3414   Assembler::pmovzxbw(dst, src);
 3415 }
 3416 
 3417 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3418   assert((src->encoding() < 16),"XMM register should be 0-15");
 3419   Assembler::pmovmskb(dst, src);
 3420 }
 3421 
 3422 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3423   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3424   Assembler::ptest(dst, src);
 3425 }
 3426 
 3427 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3428   assert(rscratch != noreg || always_reachable(src), "missing");
 3429 
 3430   if (reachable(src)) {
 3431     Assembler::sqrtss(dst, as_Address(src));
 3432   } else {
 3433     lea(rscratch, src);
 3434     Assembler::sqrtss(dst, Address(rscratch, 0));
 3435   }
 3436 }
 3437 
 3438 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3439   assert(rscratch != noreg || always_reachable(src), "missing");
 3440 
 3441   if (reachable(src)) {
 3442     Assembler::subsd(dst, as_Address(src));
 3443   } else {
 3444     lea(rscratch, src);
 3445     Assembler::subsd(dst, Address(rscratch, 0));
 3446   }
 3447 }
 3448 
 3449 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3450   assert(rscratch != noreg || always_reachable(src), "missing");
 3451 
 3452   if (reachable(src)) {
 3453     Assembler::roundsd(dst, as_Address(src), rmode);
 3454   } else {
 3455     lea(rscratch, src);
 3456     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3457   }
 3458 }
 3459 
 3460 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3461   assert(rscratch != noreg || always_reachable(src), "missing");
 3462 
 3463   if (reachable(src)) {
 3464     Assembler::subss(dst, as_Address(src));
 3465   } else {
 3466     lea(rscratch, src);
 3467     Assembler::subss(dst, Address(rscratch, 0));
 3468   }
 3469 }
 3470 
 3471 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3472   assert(rscratch != noreg || always_reachable(src), "missing");
 3473 
 3474   if (reachable(src)) {
 3475     Assembler::ucomisd(dst, as_Address(src));
 3476   } else {
 3477     lea(rscratch, src);
 3478     Assembler::ucomisd(dst, Address(rscratch, 0));
 3479   }
 3480 }
 3481 
 3482 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3483   assert(rscratch != noreg || always_reachable(src), "missing");
 3484 
 3485   if (reachable(src)) {
 3486     Assembler::ucomiss(dst, as_Address(src));
 3487   } else {
 3488     lea(rscratch, src);
 3489     Assembler::ucomiss(dst, Address(rscratch, 0));
 3490   }
 3491 }
 3492 
 3493 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3494   assert(rscratch != noreg || always_reachable(src), "missing");
 3495 
 3496   // Used in sign-bit flipping with aligned address.
 3497   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3498 
 3499   if (UseAVX > 2 &&
 3500       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3501       (dst->encoding() >= 16)) {
 3502     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3503   } else if (reachable(src)) {
 3504     Assembler::xorpd(dst, as_Address(src));
 3505   } else {
 3506     lea(rscratch, src);
 3507     Assembler::xorpd(dst, Address(rscratch, 0));
 3508   }
 3509 }
 3510 
 3511 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3512   if (UseAVX > 2 &&
 3513       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3514       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3515     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3516   } else {
 3517     Assembler::xorpd(dst, src);
 3518   }
 3519 }
 3520 
 3521 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3522   if (UseAVX > 2 &&
 3523       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3524       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3525     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3526   } else {
 3527     Assembler::xorps(dst, src);
 3528   }
 3529 }
 3530 
 3531 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3532   assert(rscratch != noreg || always_reachable(src), "missing");
 3533 
 3534   // Used in sign-bit flipping with aligned address.
 3535   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3536 
 3537   if (UseAVX > 2 &&
 3538       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3539       (dst->encoding() >= 16)) {
 3540     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3541   } else if (reachable(src)) {
 3542     Assembler::xorps(dst, as_Address(src));
 3543   } else {
 3544     lea(rscratch, src);
 3545     Assembler::xorps(dst, Address(rscratch, 0));
 3546   }
 3547 }
 3548 
 3549 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3550   assert(rscratch != noreg || always_reachable(src), "missing");
 3551 
 3552   // Used in sign-bit flipping with aligned address.
 3553   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3554   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3555   if (reachable(src)) {
 3556     Assembler::pshufb(dst, as_Address(src));
 3557   } else {
 3558     lea(rscratch, src);
 3559     Assembler::pshufb(dst, Address(rscratch, 0));
 3560   }
 3561 }
 3562 
 3563 // AVX 3-operands instructions
 3564 
 3565 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3566   assert(rscratch != noreg || always_reachable(src), "missing");
 3567 
 3568   if (reachable(src)) {
 3569     vaddsd(dst, nds, as_Address(src));
 3570   } else {
 3571     lea(rscratch, src);
 3572     vaddsd(dst, nds, Address(rscratch, 0));
 3573   }
 3574 }
 3575 
 3576 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3577   assert(rscratch != noreg || always_reachable(src), "missing");
 3578 
 3579   if (reachable(src)) {
 3580     vaddss(dst, nds, as_Address(src));
 3581   } else {
 3582     lea(rscratch, src);
 3583     vaddss(dst, nds, Address(rscratch, 0));
 3584   }
 3585 }
 3586 
 3587 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3588   assert(UseAVX > 0, "requires some form of AVX");
 3589   assert(rscratch != noreg || always_reachable(src), "missing");
 3590 
 3591   if (reachable(src)) {
 3592     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3593   } else {
 3594     lea(rscratch, src);
 3595     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3596   }
 3597 }
 3598 
 3599 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3600   assert(UseAVX > 0, "requires some form of AVX");
 3601   assert(rscratch != noreg || always_reachable(src), "missing");
 3602 
 3603   if (reachable(src)) {
 3604     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3605   } else {
 3606     lea(rscratch, src);
 3607     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3608   }
 3609 }
 3610 
 3611 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3612   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3613   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3614 
 3615   vandps(dst, nds, negate_field, vector_len, rscratch);
 3616 }
 3617 
 3618 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3619   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3620   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3621 
 3622   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3623 }
 3624 
 3625 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3626   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3627   Assembler::vpaddb(dst, nds, src, vector_len);
 3628 }
 3629 
 3630 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3631   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3632   Assembler::vpaddb(dst, nds, src, vector_len);
 3633 }
 3634 
 3635 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3636   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3637   Assembler::vpaddw(dst, nds, src, vector_len);
 3638 }
 3639 
 3640 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3641   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3642   Assembler::vpaddw(dst, nds, src, vector_len);
 3643 }
 3644 
 3645 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3646   assert(rscratch != noreg || always_reachable(src), "missing");
 3647 
 3648   if (reachable(src)) {
 3649     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3650   } else {
 3651     lea(rscratch, src);
 3652     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3653   }
 3654 }
 3655 
 3656 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3657   assert(rscratch != noreg || always_reachable(src), "missing");
 3658 
 3659   if (reachable(src)) {
 3660     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3661   } else {
 3662     lea(rscratch, src);
 3663     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3664   }
 3665 }
 3666 
 3667 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3668   assert(rscratch != noreg || always_reachable(src), "missing");
 3669 
 3670   if (reachable(src)) {
 3671     Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
 3672   } else {
 3673     lea(rscratch, src);
 3674     Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
 3675   }
 3676 }
 3677 
 3678 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3679   assert(rscratch != noreg || always_reachable(src), "missing");
 3680 
 3681   if (reachable(src)) {
 3682     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3683   } else {
 3684     lea(rscratch, src);
 3685     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3686   }
 3687 }
 3688 
 3689 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3690   assert(rscratch != noreg || always_reachable(src), "missing");
 3691 
 3692   if (reachable(src)) {
 3693     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3694   } else {
 3695     lea(rscratch, src);
 3696     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3697   }
 3698 }
 3699 
 3700 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3701   assert(rscratch != noreg || always_reachable(src), "missing");
 3702 
 3703   if (reachable(src)) {
 3704     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3705   } else {
 3706     lea(rscratch, src);
 3707     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3708   }
 3709 }
 3710 
 3711 // Vector float blend
 3712 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3713 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3714   // WARN: Allow dst == (src1|src2), mask == scratch
 3715   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3716   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3717   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3718   if (blend_emulation && scratch_available && dst_available) {
 3719     if (compute_mask) {
 3720       vpsrad(scratch, mask, 32, vector_len);
 3721       mask = scratch;
 3722     }
 3723     if (dst == src1) {
 3724       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3725       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3726     } else {
 3727       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3728       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3729     }
 3730     vpor(dst, dst, scratch, vector_len);
 3731   } else {
 3732     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3733   }
 3734 }
 3735 
 3736 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3737 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3738   // WARN: Allow dst == (src1|src2), mask == scratch
 3739   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3740   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3741   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3742   if (blend_emulation && scratch_available && dst_available) {
 3743     if (compute_mask) {
 3744       vpxor(scratch, scratch, scratch, vector_len);
 3745       vpcmpgtq(scratch, scratch, mask, vector_len);
 3746       mask = scratch;
 3747     }
 3748     if (dst == src1) {
 3749       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3750       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3751     } else {
 3752       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3753       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3754     }
 3755     vpor(dst, dst, scratch, vector_len);
 3756   } else {
 3757     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3758   }
 3759 }
 3760 
 3761 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3762   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3763   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3764 }
 3765 
 3766 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 3767   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3768   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
 3769 }
 3770 
 3771 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3772   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3773   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3774 }
 3775 
 3776 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3777   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3778   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3779 }
 3780 
 3781 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3782   assert(rscratch != noreg || always_reachable(src), "missing");
 3783 
 3784   if (reachable(src)) {
 3785     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3786   } else {
 3787     lea(rscratch, src);
 3788     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3789   }
 3790 }
 3791 
 3792 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3793                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3794   assert(rscratch != noreg || always_reachable(src), "missing");
 3795 
 3796   if (reachable(src)) {
 3797     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3798   } else {
 3799     lea(rscratch, src);
 3800     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3801   }
 3802 }
 3803 
 3804 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3805                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3806   assert(rscratch != noreg || always_reachable(src), "missing");
 3807 
 3808   if (reachable(src)) {
 3809     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3810   } else {
 3811     lea(rscratch, src);
 3812     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3813   }
 3814 }
 3815 
 3816 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3817                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3818   assert(rscratch != noreg || always_reachable(src), "missing");
 3819 
 3820   if (reachable(src)) {
 3821     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3822   } else {
 3823     lea(rscratch, src);
 3824     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3825   }
 3826 }
 3827 
 3828 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3829                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3830   assert(rscratch != noreg || always_reachable(src), "missing");
 3831 
 3832   if (reachable(src)) {
 3833     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3834   } else {
 3835     lea(rscratch, src);
 3836     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3837   }
 3838 }
 3839 
 3840 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3841   if (width == Assembler::Q) {
 3842     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3843   } else {
 3844     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3845   }
 3846 }
 3847 
 3848 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3849   int eq_cond_enc = 0x29;
 3850   int gt_cond_enc = 0x37;
 3851   if (width != Assembler::Q) {
 3852     eq_cond_enc = 0x74 + width;
 3853     gt_cond_enc = 0x64 + width;
 3854   }
 3855   switch (cond) {
 3856   case eq:
 3857     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3858     break;
 3859   case neq:
 3860     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3861     vallones(xtmp, vector_len);
 3862     vpxor(dst, xtmp, dst, vector_len);
 3863     break;
 3864   case le:
 3865     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3866     vallones(xtmp, vector_len);
 3867     vpxor(dst, xtmp, dst, vector_len);
 3868     break;
 3869   case nlt:
 3870     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3871     vallones(xtmp, vector_len);
 3872     vpxor(dst, xtmp, dst, vector_len);
 3873     break;
 3874   case lt:
 3875     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3876     break;
 3877   case nle:
 3878     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3879     break;
 3880   default:
 3881     assert(false, "Should not reach here");
 3882   }
 3883 }
 3884 
 3885 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3886   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3887   Assembler::vpmovzxbw(dst, src, vector_len);
 3888 }
 3889 
 3890 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3891   assert((src->encoding() < 16),"XMM register should be 0-15");
 3892   Assembler::vpmovmskb(dst, src, vector_len);
 3893 }
 3894 
 3895 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3896   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3897   Assembler::vpmullw(dst, nds, src, vector_len);
 3898 }
 3899 
 3900 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3901   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3902   Assembler::vpmullw(dst, nds, src, vector_len);
 3903 }
 3904 
 3905 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3906   assert((UseAVX > 0), "AVX support is needed");
 3907   assert(rscratch != noreg || always_reachable(src), "missing");
 3908 
 3909   if (reachable(src)) {
 3910     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3911   } else {
 3912     lea(rscratch, src);
 3913     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3914   }
 3915 }
 3916 
 3917 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3918   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3919   Assembler::vpsubb(dst, nds, src, vector_len);
 3920 }
 3921 
 3922 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3923   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3924   Assembler::vpsubb(dst, nds, src, vector_len);
 3925 }
 3926 
 3927 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3928   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3929   Assembler::vpsubw(dst, nds, src, vector_len);
 3930 }
 3931 
 3932 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3933   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3934   Assembler::vpsubw(dst, nds, src, vector_len);
 3935 }
 3936 
 3937 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3938   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3939   Assembler::vpsraw(dst, nds, shift, vector_len);
 3940 }
 3941 
 3942 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3943   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3944   Assembler::vpsraw(dst, nds, shift, vector_len);
 3945 }
 3946 
 3947 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3948   assert(UseAVX > 2,"");
 3949   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3950      vector_len = 2;
 3951   }
 3952   Assembler::evpsraq(dst, nds, shift, vector_len);
 3953 }
 3954 
 3955 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3956   assert(UseAVX > 2,"");
 3957   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3958      vector_len = 2;
 3959   }
 3960   Assembler::evpsraq(dst, nds, shift, vector_len);
 3961 }
 3962 
 3963 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3964   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3965   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3966 }
 3967 
 3968 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3969   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3970   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3971 }
 3972 
 3973 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3974   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3975   Assembler::vpsllw(dst, nds, shift, vector_len);
 3976 }
 3977 
 3978 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3979   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3980   Assembler::vpsllw(dst, nds, shift, vector_len);
 3981 }
 3982 
 3983 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3984   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3985   Assembler::vptest(dst, src);
 3986 }
 3987 
 3988 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3989   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3990   Assembler::punpcklbw(dst, src);
 3991 }
 3992 
 3993 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3994   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3995   Assembler::pshufd(dst, src, mode);
 3996 }
 3997 
 3998 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3999   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 4000   Assembler::pshuflw(dst, src, mode);
 4001 }
 4002 
 4003 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4004   assert(rscratch != noreg || always_reachable(src), "missing");
 4005 
 4006   if (reachable(src)) {
 4007     vandpd(dst, nds, as_Address(src), vector_len);
 4008   } else {
 4009     lea(rscratch, src);
 4010     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 4011   }
 4012 }
 4013 
 4014 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4015   assert(rscratch != noreg || always_reachable(src), "missing");
 4016 
 4017   if (reachable(src)) {
 4018     vandps(dst, nds, as_Address(src), vector_len);
 4019   } else {
 4020     lea(rscratch, src);
 4021     vandps(dst, nds, Address(rscratch, 0), vector_len);
 4022   }
 4023 }
 4024 
 4025 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 4026                             bool merge, int vector_len, Register rscratch) {
 4027   assert(rscratch != noreg || always_reachable(src), "missing");
 4028 
 4029   if (reachable(src)) {
 4030     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 4031   } else {
 4032     lea(rscratch, src);
 4033     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 4034   }
 4035 }
 4036 
 4037 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4038   assert(rscratch != noreg || always_reachable(src), "missing");
 4039 
 4040   if (reachable(src)) {
 4041     vdivsd(dst, nds, as_Address(src));
 4042   } else {
 4043     lea(rscratch, src);
 4044     vdivsd(dst, nds, Address(rscratch, 0));
 4045   }
 4046 }
 4047 
 4048 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4049   assert(rscratch != noreg || always_reachable(src), "missing");
 4050 
 4051   if (reachable(src)) {
 4052     vdivss(dst, nds, as_Address(src));
 4053   } else {
 4054     lea(rscratch, src);
 4055     vdivss(dst, nds, Address(rscratch, 0));
 4056   }
 4057 }
 4058 
 4059 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4060   assert(rscratch != noreg || always_reachable(src), "missing");
 4061 
 4062   if (reachable(src)) {
 4063     vmulsd(dst, nds, as_Address(src));
 4064   } else {
 4065     lea(rscratch, src);
 4066     vmulsd(dst, nds, Address(rscratch, 0));
 4067   }
 4068 }
 4069 
 4070 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4071   assert(rscratch != noreg || always_reachable(src), "missing");
 4072 
 4073   if (reachable(src)) {
 4074     vmulss(dst, nds, as_Address(src));
 4075   } else {
 4076     lea(rscratch, src);
 4077     vmulss(dst, nds, Address(rscratch, 0));
 4078   }
 4079 }
 4080 
 4081 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4082   assert(rscratch != noreg || always_reachable(src), "missing");
 4083 
 4084   if (reachable(src)) {
 4085     vsubsd(dst, nds, as_Address(src));
 4086   } else {
 4087     lea(rscratch, src);
 4088     vsubsd(dst, nds, Address(rscratch, 0));
 4089   }
 4090 }
 4091 
 4092 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4093   assert(rscratch != noreg || always_reachable(src), "missing");
 4094 
 4095   if (reachable(src)) {
 4096     vsubss(dst, nds, as_Address(src));
 4097   } else {
 4098     lea(rscratch, src);
 4099     vsubss(dst, nds, Address(rscratch, 0));
 4100   }
 4101 }
 4102 
 4103 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4104   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 4105   assert(rscratch != noreg || always_reachable(src), "missing");
 4106 
 4107   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 4108 }
 4109 
 4110 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4111   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 4112   assert(rscratch != noreg || always_reachable(src), "missing");
 4113 
 4114   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 4115 }
 4116 
 4117 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4118   assert(rscratch != noreg || always_reachable(src), "missing");
 4119 
 4120   if (reachable(src)) {
 4121     vxorpd(dst, nds, as_Address(src), vector_len);
 4122   } else {
 4123     lea(rscratch, src);
 4124     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 4125   }
 4126 }
 4127 
 4128 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4129   assert(rscratch != noreg || always_reachable(src), "missing");
 4130 
 4131   if (reachable(src)) {
 4132     vxorps(dst, nds, as_Address(src), vector_len);
 4133   } else {
 4134     lea(rscratch, src);
 4135     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 4136   }
 4137 }
 4138 
 4139 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4140   assert(rscratch != noreg || always_reachable(src), "missing");
 4141 
 4142   if (UseAVX > 1 || (vector_len < 1)) {
 4143     if (reachable(src)) {
 4144       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 4145     } else {
 4146       lea(rscratch, src);
 4147       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 4148     }
 4149   } else {
 4150     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 4151   }
 4152 }
 4153 
 4154 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4155   assert(rscratch != noreg || always_reachable(src), "missing");
 4156 
 4157   if (reachable(src)) {
 4158     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 4159   } else {
 4160     lea(rscratch, src);
 4161     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 4162   }
 4163 }
 4164 
 4165 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 4166   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 4167   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 4168   // The inverted mask is sign-extended
 4169   andptr(possibly_non_local, inverted_mask);
 4170 }
 4171 
 4172 void MacroAssembler::resolve_jobject(Register value,
 4173                                      Register thread,
 4174                                      Register tmp) {
 4175   assert_different_registers(value, thread, tmp);
 4176   Label done, tagged, weak_tagged;
 4177   testptr(value, value);
 4178   jcc(Assembler::zero, done);           // Use null as-is.
 4179   testptr(value, JNIHandles::tag_mask); // Test for tag.
 4180   jcc(Assembler::notZero, tagged);
 4181 
 4182   // Resolve local handle
 4183   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4184   verify_oop(value);
 4185   jmp(done);
 4186 
 4187   bind(tagged);
 4188   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4189   jcc(Assembler::notZero, weak_tagged);
 4190 
 4191   // Resolve global handle
 4192   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4193   verify_oop(value);
 4194   jmp(done);
 4195 
 4196   bind(weak_tagged);
 4197   // Resolve jweak.
 4198   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4199                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4200   verify_oop(value);
 4201 
 4202   bind(done);
 4203 }
 4204 
 4205 void MacroAssembler::resolve_global_jobject(Register value,
 4206                                             Register thread,
 4207                                             Register tmp) {
 4208   assert_different_registers(value, thread, tmp);
 4209   Label done;
 4210 
 4211   testptr(value, value);
 4212   jcc(Assembler::zero, done);           // Use null as-is.
 4213 
 4214 #ifdef ASSERT
 4215   {
 4216     Label valid_global_tag;
 4217     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4218     jcc(Assembler::notZero, valid_global_tag);
 4219     stop("non global jobject using resolve_global_jobject");
 4220     bind(valid_global_tag);
 4221   }
 4222 #endif
 4223 
 4224   // Resolve global handle
 4225   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4226   verify_oop(value);
 4227 
 4228   bind(done);
 4229 }
 4230 
 4231 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4232   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4233 }
 4234 
 4235 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4236 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4237   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4238 }
 4239 
 4240 void MacroAssembler::subptr(Register dst, Register src) {
 4241   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4242 }
 4243 
 4244 // C++ bool manipulation
 4245 void MacroAssembler::testbool(Register dst) {
 4246   if(sizeof(bool) == 1)
 4247     testb(dst, 0xff);
 4248   else if(sizeof(bool) == 2) {
 4249     // testw implementation needed for two byte bools
 4250     ShouldNotReachHere();
 4251   } else if(sizeof(bool) == 4)
 4252     testl(dst, dst);
 4253   else
 4254     // unsupported
 4255     ShouldNotReachHere();
 4256 }
 4257 
 4258 void MacroAssembler::testptr(Register dst, Register src) {
 4259   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4260 }
 4261 
 4262 // Object / value buffer allocation...
 4263 //
 4264 // Kills klass and rsi on LP64
 4265 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
 4266                                        Register t1, Register t2,
 4267                                        bool clear_fields, Label& alloc_failed)
 4268 {
 4269   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
 4270   Register layout_size = t1;
 4271   assert(new_obj == rax, "needs to be rax");
 4272   assert_different_registers(klass, new_obj, t1, t2);
 4273 
 4274   // get instance_size in InstanceKlass (scaled to a count of bytes)
 4275   movl(layout_size, Address(klass, Klass::layout_helper_offset()));
 4276   // test to see if it is malformed in some way
 4277   testl(layout_size, Klass::_lh_instance_slow_path_bit);
 4278   jcc(Assembler::notZero, slow_case_no_pop);
 4279 
 4280   // Allocate the instance:
 4281   //  If TLAB is enabled:
 4282   //    Try to allocate in the TLAB.
 4283   //    If fails, go to the slow path.
 4284   //  Else If inline contiguous allocations are enabled:
 4285   //    Try to allocate in eden.
 4286   //    If fails due to heap end, go to slow path.
 4287   //
 4288   //  If TLAB is enabled OR inline contiguous is enabled:
 4289   //    Initialize the allocation.
 4290   //    Exit.
 4291   //
 4292   //  Go to slow path.
 4293 
 4294   push(klass);
 4295   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
 4296 #ifndef _LP64
 4297   if (UseTLAB) {
 4298     get_thread(thread);
 4299   }
 4300 #endif // _LP64
 4301 
 4302   if (UseTLAB) {
 4303     tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
 4304     if (ZeroTLAB || (!clear_fields)) {
 4305       // the fields have been already cleared
 4306       jmp(initialize_header);
 4307     } else {
 4308       // initialize both the header and fields
 4309       jmp(initialize_object);
 4310     }
 4311   } else {
 4312     jmp(slow_case);
 4313   }
 4314 
 4315   // If UseTLAB is true, the object is created above and there is an initialize need.
 4316   // Otherwise, skip and go to the slow path.
 4317   if (UseTLAB) {
 4318     if (clear_fields) {
 4319       // The object is initialized before the header.  If the object size is
 4320       // zero, go directly to the header initialization.
 4321       bind(initialize_object);
 4322       if (UseCompactObjectHeaders) {
 4323         assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
 4324         decrement(layout_size, oopDesc::base_offset_in_bytes());
 4325       } else {
 4326         decrement(layout_size, sizeof(oopDesc));
 4327       }
 4328       jcc(Assembler::zero, initialize_header);
 4329 
 4330       // Initialize topmost object field, divide size by 8, check if odd and
 4331       // test if zero.
 4332       Register zero = klass;
 4333       xorl(zero, zero);    // use zero reg to clear memory (shorter code)
 4334       shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
 4335 
 4336   #ifdef ASSERT
 4337       // make sure instance_size was multiple of 8
 4338       Label L;
 4339       // Ignore partial flag stall after shrl() since it is debug VM
 4340       jcc(Assembler::carryClear, L);
 4341       stop("object size is not multiple of 2 - adjust this code");
 4342       bind(L);
 4343       // must be > 0, no extra check needed here
 4344   #endif
 4345 
 4346       // initialize remaining object fields: instance_size was a multiple of 8
 4347       {
 4348         Label loop;
 4349         bind(loop);
 4350         int header_size_bytes = oopDesc::header_size() * HeapWordSize;
 4351         assert(is_aligned(header_size_bytes, BytesPerLong), "oop header size must be 8-byte-aligned");
 4352         movptr(Address(new_obj, layout_size, Address::times_8, header_size_bytes - 1*oopSize), zero);
 4353         NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, header_size_bytes - 2*oopSize), zero));
 4354         decrement(layout_size);
 4355         jcc(Assembler::notZero, loop);
 4356       }
 4357     } // clear_fields
 4358 
 4359     // initialize object header only.
 4360     bind(initialize_header);
 4361     if (UseCompactObjectHeaders || EnableValhalla) {
 4362       pop(klass);
 4363       Register mark_word = t2;
 4364       movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
 4365       movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 4366     } else {
 4367      movptr(Address(new_obj, oopDesc::mark_offset_in_bytes()),
 4368             (intptr_t)markWord::prototype().value()); // header
 4369      pop(klass);   // get saved klass back in the register.
 4370     }
 4371     if (!UseCompactObjectHeaders) {
 4372 #ifdef _LP64
 4373       xorl(rsi, rsi);                 // use zero reg to clear memory (shorter code)
 4374       store_klass_gap(new_obj, rsi);  // zero klass gap for compressed oops
 4375 #endif
 4376       movptr(t2, klass);         // preserve klass
 4377       store_klass(new_obj, t2, rscratch1);  // src klass reg is potentially compressed
 4378     }
 4379     jmp(done);
 4380   }
 4381 
 4382   bind(slow_case);
 4383   pop(klass);
 4384   bind(slow_case_no_pop);
 4385   jmp(alloc_failed);
 4386 
 4387   bind(done);
 4388 }
 4389 
 4390 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4391 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4392                                    Register var_size_in_bytes,
 4393                                    int con_size_in_bytes,
 4394                                    Register t1,
 4395                                    Register t2,
 4396                                    Label& slow_case) {
 4397   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4398   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4399 }
 4400 
 4401 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4402   RegSet regs;
 4403 #ifdef _LP64
 4404   regs += RegSet::of(rax, rcx, rdx);
 4405 #ifndef _WINDOWS
 4406   regs += RegSet::of(rsi, rdi);
 4407 #endif
 4408   regs += RegSet::range(r8, r11);
 4409 #else
 4410   regs += RegSet::of(rax, rcx, rdx);
 4411 #endif
 4412 #ifdef _LP64
 4413   if (UseAPX) {
 4414     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
 4415   }
 4416 #endif
 4417   return regs;
 4418 }
 4419 
 4420 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4421   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4422 #if defined(_WINDOWS)
 4423   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4424   if (num_xmm_registers > 16) {
 4425      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4426   }
 4427   return result;
 4428 #else
 4429   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4430 #endif
 4431 }
 4432 
 4433 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4434 
 4435 #ifndef _LP64
 4436 static bool use_x87_registers() { return UseSSE < 2; }
 4437 #endif
 4438 static bool use_xmm_registers() { return UseSSE >= 1; }
 4439 
 4440 // C1 only ever uses the first double/float of the XMM register.
 4441 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4442 
 4443 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4444   if (UseSSE == 1) {
 4445     masm->movflt(Address(rsp, offset), reg);
 4446   } else {
 4447     masm->movdbl(Address(rsp, offset), reg);
 4448   }
 4449 }
 4450 
 4451 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4452   if (UseSSE == 1) {
 4453     masm->movflt(reg, Address(rsp, offset));
 4454   } else {
 4455     masm->movdbl(reg, Address(rsp, offset));
 4456   }
 4457 }
 4458 
 4459 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4460                                   bool save_fpu, int& gp_area_size,
 4461                                   int& fp_area_size, int& xmm_area_size) {
 4462 
 4463   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4464                          StackAlignmentInBytes);
 4465 #ifdef _LP64
 4466   fp_area_size = 0;
 4467 #else
 4468   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4469 #endif
 4470   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4471 
 4472   return gp_area_size + fp_area_size + xmm_area_size;
 4473 }
 4474 
 4475 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4476   block_comment("push_call_clobbered_registers start");
 4477   // Regular registers
 4478   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4479 
 4480   int gp_area_size;
 4481   int fp_area_size;
 4482   int xmm_area_size;
 4483   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4484                                                gp_area_size, fp_area_size, xmm_area_size);
 4485   subptr(rsp, total_save_size);
 4486 
 4487   push_set(gp_registers_to_push, 0);
 4488 
 4489 #ifndef _LP64
 4490   if (save_fpu && use_x87_registers()) {
 4491     fnsave(Address(rsp, gp_area_size));
 4492     fwait();
 4493   }
 4494 #endif
 4495   if (save_fpu && use_xmm_registers()) {
 4496     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4497   }
 4498 
 4499   block_comment("push_call_clobbered_registers end");
 4500 }
 4501 
 4502 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4503   block_comment("pop_call_clobbered_registers start");
 4504 
 4505   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4506 
 4507   int gp_area_size;
 4508   int fp_area_size;
 4509   int xmm_area_size;
 4510   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4511                                                gp_area_size, fp_area_size, xmm_area_size);
 4512 
 4513   if (restore_fpu && use_xmm_registers()) {
 4514     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4515   }
 4516 #ifndef _LP64
 4517   if (restore_fpu && use_x87_registers()) {
 4518     frstor(Address(rsp, gp_area_size));
 4519   }
 4520 #endif
 4521 
 4522   pop_set(gp_registers_to_pop, 0);
 4523 
 4524   addptr(rsp, total_save_size);
 4525 
 4526   vzeroupper();
 4527 
 4528   block_comment("pop_call_clobbered_registers end");
 4529 }
 4530 
 4531 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4532   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4533   int spill_offset = offset;
 4534 
 4535   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4536     save_xmm_register(this, spill_offset, *it);
 4537     spill_offset += xmm_save_size();
 4538   }
 4539 }
 4540 
 4541 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4542   int restore_size = set.size() * xmm_save_size();
 4543   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4544 
 4545   int restore_offset = offset + restore_size - xmm_save_size();
 4546 
 4547   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4548     restore_xmm_register(this, restore_offset, *it);
 4549     restore_offset -= xmm_save_size();
 4550   }
 4551 }
 4552 
 4553 void MacroAssembler::push_set(RegSet set, int offset) {
 4554   int spill_offset;
 4555   if (offset == -1) {
 4556     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4557     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4558     subptr(rsp, aligned_size);
 4559     spill_offset = 0;
 4560   } else {
 4561     spill_offset = offset;
 4562   }
 4563 
 4564   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4565     movptr(Address(rsp, spill_offset), *it);
 4566     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4567   }
 4568 }
 4569 
 4570 void MacroAssembler::pop_set(RegSet set, int offset) {
 4571 
 4572   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4573   int restore_size = set.size() * gp_reg_size;
 4574   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4575 
 4576   int restore_offset;
 4577   if (offset == -1) {
 4578     restore_offset = restore_size - gp_reg_size;
 4579   } else {
 4580     restore_offset = offset + restore_size - gp_reg_size;
 4581   }
 4582   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4583     movptr(*it, Address(rsp, restore_offset));
 4584     restore_offset -= gp_reg_size;
 4585   }
 4586 
 4587   if (offset == -1) {
 4588     addptr(rsp, aligned_size);
 4589   }
 4590 }
 4591 
 4592 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4593 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4594   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4595   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4596   Label done;
 4597 
 4598   testptr(length_in_bytes, length_in_bytes);
 4599   jcc(Assembler::zero, done);
 4600 
 4601   // initialize topmost word, divide index by 2, check if odd and test if zero
 4602   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4603 #ifdef ASSERT
 4604   {
 4605     Label L;
 4606     testptr(length_in_bytes, BytesPerWord - 1);
 4607     jcc(Assembler::zero, L);
 4608     stop("length must be a multiple of BytesPerWord");
 4609     bind(L);
 4610   }
 4611 #endif
 4612   Register index = length_in_bytes;
 4613   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4614   if (UseIncDec) {
 4615     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4616   } else {
 4617     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4618     shrptr(index, 1);
 4619   }
 4620 #ifndef _LP64
 4621   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4622   {
 4623     Label even;
 4624     // note: if index was a multiple of 8, then it cannot
 4625     //       be 0 now otherwise it must have been 0 before
 4626     //       => if it is even, we don't need to check for 0 again
 4627     jcc(Assembler::carryClear, even);
 4628     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4629     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4630     // index could be 0 now, must check again
 4631     jcc(Assembler::zero, done);
 4632     bind(even);
 4633   }
 4634 #endif // !_LP64
 4635   // initialize remaining object fields: index is a multiple of 2 now
 4636   {
 4637     Label loop;
 4638     bind(loop);
 4639     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4640     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4641     decrement(index);
 4642     jcc(Assembler::notZero, loop);
 4643   }
 4644 
 4645   bind(done);
 4646 }
 4647 
 4648 void MacroAssembler::get_inline_type_field_klass(Register holder_klass, Register index, Register inline_klass) {
 4649   inline_layout_info(holder_klass, index, inline_klass);
 4650   movptr(inline_klass, Address(inline_klass, InlineLayoutInfo::klass_offset()));
 4651 }
 4652 
 4653 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
 4654   movptr(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset()));
 4655 #ifdef ASSERT
 4656   {
 4657     Label done;
 4658     cmpptr(layout_info, 0);
 4659     jcc(Assembler::notEqual, done);
 4660     stop("inline_layout_info_array is null");
 4661     bind(done);
 4662   }
 4663 #endif
 4664 
 4665   InlineLayoutInfo array[2];
 4666   int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
 4667   if (is_power_of_2(size)) {
 4668     shll(index, log2i_exact(size)); // Scale index by power of 2
 4669   } else {
 4670     imull(index, index, size); // Scale the index to be the entry index * array_element_size
 4671   }
 4672   lea(layout_info, Address(layout_info, index, Address::times_1, Array<InlineLayoutInfo>::base_offset_in_bytes()));
 4673 }
 4674 
 4675 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 4676 #ifdef ASSERT
 4677   {
 4678     Label done_check;
 4679     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 4680     stop("get_default_value_oop from non inline type klass");
 4681     bind(done_check);
 4682   }
 4683 #endif
 4684   Register offset = temp_reg;
 4685   // Getting the offset of the pre-allocated default value
 4686   movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 4687   movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 4688 
 4689   // Getting the mirror
 4690   movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 4691   resolve_oop_handle(obj, inline_klass);
 4692 
 4693   // Getting the pre-allocated default value from the mirror
 4694   Address field(obj, offset, Address::times_1);
 4695   load_heap_oop(obj, field);
 4696 }
 4697 
 4698 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 4699 #ifdef ASSERT
 4700   {
 4701     Label done_check;
 4702     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 4703     stop("get_empty_value from non-empty inline klass");
 4704     bind(done_check);
 4705   }
 4706 #endif
 4707   get_default_value_oop(inline_klass, temp_reg, obj);
 4708 }
 4709 
 4710 
 4711 // Look up the method for a megamorphic invokeinterface call.
 4712 // The target method is determined by <intf_klass, itable_index>.
 4713 // The receiver klass is in recv_klass.
 4714 // On success, the result will be in method_result, and execution falls through.
 4715 // On failure, execution transfers to the given label.
 4716 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4717                                              Register intf_klass,
 4718                                              RegisterOrConstant itable_index,
 4719                                              Register method_result,
 4720                                              Register scan_temp,
 4721                                              Label& L_no_such_interface,
 4722                                              bool return_method) {
 4723   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4724   assert_different_registers(method_result, intf_klass, scan_temp);
 4725   assert(recv_klass != method_result || !return_method,
 4726          "recv_klass can be destroyed when method isn't needed");
 4727 
 4728   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4729          "caller must use same register for non-constant itable index as for method");
 4730 
 4731   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4732   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4733   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4734   int scan_step   = itableOffsetEntry::size() * wordSize;
 4735   int vte_size    = vtableEntry::size_in_bytes();
 4736   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4737   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4738 
 4739   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4740 
 4741   // Could store the aligned, prescaled offset in the klass.
 4742   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4743 
 4744   if (return_method) {
 4745     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4746     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4747     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4748   }
 4749 
 4750   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4751   //   if (scan->interface() == intf) {
 4752   //     result = (klass + scan->offset() + itable_index);
 4753   //   }
 4754   // }
 4755   Label search, found_method;
 4756 
 4757   for (int peel = 1; peel >= 0; peel--) {
 4758     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4759     cmpptr(intf_klass, method_result);
 4760 
 4761     if (peel) {
 4762       jccb(Assembler::equal, found_method);
 4763     } else {
 4764       jccb(Assembler::notEqual, search);
 4765       // (invert the test to fall through to found_method...)
 4766     }
 4767 
 4768     if (!peel)  break;
 4769 
 4770     bind(search);
 4771 
 4772     // Check that the previous entry is non-null.  A null entry means that
 4773     // the receiver class doesn't implement the interface, and wasn't the
 4774     // same as when the caller was compiled.
 4775     testptr(method_result, method_result);
 4776     jcc(Assembler::zero, L_no_such_interface);
 4777     addptr(scan_temp, scan_step);
 4778   }
 4779 
 4780   bind(found_method);
 4781 
 4782   if (return_method) {
 4783     // Got a hit.
 4784     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4785     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4786   }
 4787 }
 4788 
 4789 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4790 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4791 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4792 // The target method is determined by <holder_klass, itable_index>.
 4793 // The receiver klass is in recv_klass.
 4794 // On success, the result will be in method_result, and execution falls through.
 4795 // On failure, execution transfers to the given label.
 4796 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4797                                                   Register holder_klass,
 4798                                                   Register resolved_klass,
 4799                                                   Register method_result,
 4800                                                   Register scan_temp,
 4801                                                   Register temp_reg2,
 4802                                                   Register receiver,
 4803                                                   int itable_index,
 4804                                                   Label& L_no_such_interface) {
 4805   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4806   Register temp_itbl_klass = method_result;
 4807   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4808 
 4809   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4810   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4811   int scan_step = itableOffsetEntry::size() * wordSize;
 4812   int vte_size = vtableEntry::size_in_bytes();
 4813   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4814   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4815   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4816   assert(vte_size == wordSize, "adjust times_vte_scale");
 4817 
 4818   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4819 
 4820   // temp_itbl_klass = recv_klass.itable[0]
 4821   // scan_temp = &recv_klass.itable[0] + step
 4822   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4823   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4824   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4825   xorptr(temp_reg, temp_reg);
 4826 
 4827   // Initial checks:
 4828   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4829   //   - if (itable[0] == 0), no such interface
 4830   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4831   cmpptr(holder_klass, resolved_klass);
 4832   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4833   testptr(temp_itbl_klass, temp_itbl_klass);
 4834   jccb(Assembler::zero, L_no_such_interface);
 4835   cmpptr(holder_klass, temp_itbl_klass);
 4836   jccb(Assembler::equal, L_holder_found);
 4837 
 4838   // Loop: Look for holder_klass record in itable
 4839   //   do {
 4840   //     tmp = itable[index];
 4841   //     index += step;
 4842   //     if (tmp == holder_klass) {
 4843   //       goto L_holder_found; // Found!
 4844   //     }
 4845   //   } while (tmp != 0);
 4846   //   goto L_no_such_interface // Not found.
 4847   Label L_scan_holder;
 4848   bind(L_scan_holder);
 4849     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4850     addptr(scan_temp, scan_step);
 4851     cmpptr(holder_klass, temp_itbl_klass);
 4852     jccb(Assembler::equal, L_holder_found);
 4853     testptr(temp_itbl_klass, temp_itbl_klass);
 4854     jccb(Assembler::notZero, L_scan_holder);
 4855 
 4856   jmpb(L_no_such_interface);
 4857 
 4858   // Loop: Look for resolved_class record in itable
 4859   //   do {
 4860   //     tmp = itable[index];
 4861   //     index += step;
 4862   //     if (tmp == holder_klass) {
 4863   //        // Also check if we have met a holder klass
 4864   //        holder_tmp = itable[index-step-ioffset];
 4865   //     }
 4866   //     if (tmp == resolved_klass) {
 4867   //        goto L_resolved_found;  // Found!
 4868   //     }
 4869   //   } while (tmp != 0);
 4870   //   goto L_no_such_interface // Not found.
 4871   //
 4872   Label L_loop_scan_resolved;
 4873   bind(L_loop_scan_resolved);
 4874     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4875     addptr(scan_temp, scan_step);
 4876     bind(L_loop_scan_resolved_entry);
 4877     cmpptr(holder_klass, temp_itbl_klass);
 4878     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4879     cmpptr(resolved_klass, temp_itbl_klass);
 4880     jccb(Assembler::equal, L_resolved_found);
 4881     testptr(temp_itbl_klass, temp_itbl_klass);
 4882     jccb(Assembler::notZero, L_loop_scan_resolved);
 4883 
 4884   jmpb(L_no_such_interface);
 4885 
 4886   Label L_ready;
 4887 
 4888   // See if we already have a holder klass. If not, go and scan for it.
 4889   bind(L_resolved_found);
 4890   testptr(temp_reg, temp_reg);
 4891   jccb(Assembler::zero, L_scan_holder);
 4892   jmpb(L_ready);
 4893 
 4894   bind(L_holder_found);
 4895   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4896 
 4897   // Finally, temp_reg contains holder_klass vtable offset
 4898   bind(L_ready);
 4899   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4900   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4901     load_klass(scan_temp, receiver, noreg);
 4902     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4903   } else {
 4904     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4905   }
 4906 }
 4907 
 4908 
 4909 // virtual method calling
 4910 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4911                                            RegisterOrConstant vtable_index,
 4912                                            Register method_result) {
 4913   const ByteSize base = Klass::vtable_start_offset();
 4914   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4915   Address vtable_entry_addr(recv_klass,
 4916                             vtable_index, Address::times_ptr,
 4917                             base + vtableEntry::method_offset());
 4918   movptr(method_result, vtable_entry_addr);
 4919 }
 4920 
 4921 
 4922 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4923                            Register super_klass,
 4924                            Register temp_reg,
 4925                            Label& L_success) {
 4926   Label L_failure;
 4927   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4928   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4929   bind(L_failure);
 4930 }
 4931 
 4932 
 4933 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4934                                                    Register super_klass,
 4935                                                    Register temp_reg,
 4936                                                    Label* L_success,
 4937                                                    Label* L_failure,
 4938                                                    Label* L_slow_path,
 4939                                         RegisterOrConstant super_check_offset) {
 4940   assert_different_registers(sub_klass, super_klass, temp_reg);
 4941   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4942   if (super_check_offset.is_register()) {
 4943     assert_different_registers(sub_klass, super_klass,
 4944                                super_check_offset.as_register());
 4945   } else if (must_load_sco) {
 4946     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4947   }
 4948 
 4949   Label L_fallthrough;
 4950   int label_nulls = 0;
 4951   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4952   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4953   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4954   assert(label_nulls <= 1, "at most one null in the batch");
 4955 
 4956   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4957   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4958   Address super_check_offset_addr(super_klass, sco_offset);
 4959 
 4960   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4961   // range of a jccb.  If this routine grows larger, reconsider at
 4962   // least some of these.
 4963 #define local_jcc(assembler_cond, label)                                \
 4964   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4965   else                             jcc( assembler_cond, label) /*omit semi*/
 4966 
 4967   // Hacked jmp, which may only be used just before L_fallthrough.
 4968 #define final_jmp(label)                                                \
 4969   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4970   else                            jmp(label)                /*omit semi*/
 4971 
 4972   // If the pointers are equal, we are done (e.g., String[] elements).
 4973   // This self-check enables sharing of secondary supertype arrays among
 4974   // non-primary types such as array-of-interface.  Otherwise, each such
 4975   // type would need its own customized SSA.
 4976   // We move this check to the front of the fast path because many
 4977   // type checks are in fact trivially successful in this manner,
 4978   // so we get a nicely predicted branch right at the start of the check.
 4979   cmpptr(sub_klass, super_klass);
 4980   local_jcc(Assembler::equal, *L_success);
 4981 
 4982   // Check the supertype display:
 4983   if (must_load_sco) {
 4984     // Positive movl does right thing on LP64.
 4985     movl(temp_reg, super_check_offset_addr);
 4986     super_check_offset = RegisterOrConstant(temp_reg);
 4987   }
 4988   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4989   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4990 
 4991   // This check has worked decisively for primary supers.
 4992   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4993   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4994   // This works in the same check above because of a tricky aliasing
 4995   // between the super_cache and the primary super display elements.
 4996   // (The 'super_check_addr' can address either, as the case requires.)
 4997   // Note that the cache is updated below if it does not help us find
 4998   // what we need immediately.
 4999   // So if it was a primary super, we can just fail immediately.
 5000   // Otherwise, it's the slow path for us (no success at this point).
 5001 
 5002   if (super_check_offset.is_register()) {
 5003     local_jcc(Assembler::equal, *L_success);
 5004     cmpl(super_check_offset.as_register(), sc_offset);
 5005     if (L_failure == &L_fallthrough) {
 5006       local_jcc(Assembler::equal, *L_slow_path);
 5007     } else {
 5008       local_jcc(Assembler::notEqual, *L_failure);
 5009       final_jmp(*L_slow_path);
 5010     }
 5011   } else if (super_check_offset.as_constant() == sc_offset) {
 5012     // Need a slow path; fast failure is impossible.
 5013     if (L_slow_path == &L_fallthrough) {
 5014       local_jcc(Assembler::equal, *L_success);
 5015     } else {
 5016       local_jcc(Assembler::notEqual, *L_slow_path);
 5017       final_jmp(*L_success);
 5018     }
 5019   } else {
 5020     // No slow path; it's a fast decision.
 5021     if (L_failure == &L_fallthrough) {
 5022       local_jcc(Assembler::equal, *L_success);
 5023     } else {
 5024       local_jcc(Assembler::notEqual, *L_failure);
 5025       final_jmp(*L_success);
 5026     }
 5027   }
 5028 
 5029   bind(L_fallthrough);
 5030 
 5031 #undef local_jcc
 5032 #undef final_jmp
 5033 }
 5034 
 5035 
 5036 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
 5037                                                           Register super_klass,
 5038                                                           Register temp_reg,
 5039                                                           Register temp2_reg,
 5040                                                           Label* L_success,
 5041                                                           Label* L_failure,
 5042                                                           bool set_cond_codes) {
 5043   assert_different_registers(sub_klass, super_klass, temp_reg);
 5044   if (temp2_reg != noreg)
 5045     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 5046 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 5047 
 5048   Label L_fallthrough;
 5049   int label_nulls = 0;
 5050   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 5051   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 5052   assert(label_nulls <= 1, "at most one null in the batch");
 5053 
 5054   // a couple of useful fields in sub_klass:
 5055   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 5056   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 5057   Address secondary_supers_addr(sub_klass, ss_offset);
 5058   Address super_cache_addr(     sub_klass, sc_offset);
 5059 
 5060   // Do a linear scan of the secondary super-klass chain.
 5061   // This code is rarely used, so simplicity is a virtue here.
 5062   // The repne_scan instruction uses fixed registers, which we must spill.
 5063   // Don't worry too much about pre-existing connections with the input regs.
 5064 
 5065   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 5066   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 5067 
 5068   // Get super_klass value into rax (even if it was in rdi or rcx).
 5069   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 5070   if (super_klass != rax) {
 5071     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 5072     mov(rax, super_klass);
 5073   }
 5074   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 5075   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 5076 
 5077 #ifndef PRODUCT
 5078   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 5079   ExternalAddress pst_counter_addr((address) pst_counter);
 5080   NOT_LP64(  incrementl(pst_counter_addr) );
 5081   LP64_ONLY( lea(rcx, pst_counter_addr) );
 5082   LP64_ONLY( incrementl(Address(rcx, 0)) );
 5083 #endif //PRODUCT
 5084 
 5085   // We will consult the secondary-super array.
 5086   movptr(rdi, secondary_supers_addr);
 5087   // Load the array length.  (Positive movl does right thing on LP64.)
 5088   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 5089   // Skip to start of data.
 5090   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 5091 
 5092   // Scan RCX words at [RDI] for an occurrence of RAX.
 5093   // Set NZ/Z based on last compare.
 5094   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 5095   // not change flags (only scas instruction which is repeated sets flags).
 5096   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 5097 
 5098     testptr(rax,rax); // Set Z = 0
 5099     repne_scan();
 5100 
 5101   // Unspill the temp. registers:
 5102   if (pushed_rdi)  pop(rdi);
 5103   if (pushed_rcx)  pop(rcx);
 5104   if (pushed_rax)  pop(rax);
 5105 
 5106   if (set_cond_codes) {
 5107     // Special hack for the AD files:  rdi is guaranteed non-zero.
 5108     assert(!pushed_rdi, "rdi must be left non-null");
 5109     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 5110   }
 5111 
 5112   if (L_failure == &L_fallthrough)
 5113         jccb(Assembler::notEqual, *L_failure);
 5114   else  jcc(Assembler::notEqual, *L_failure);
 5115 
 5116   // Success.  Cache the super we found and proceed in triumph.
 5117   movptr(super_cache_addr, super_klass);
 5118 
 5119   if (L_success != &L_fallthrough) {
 5120     jmp(*L_success);
 5121   }
 5122 
 5123 #undef IS_A_TEMP
 5124 
 5125   bind(L_fallthrough);
 5126 }
 5127 
 5128 #ifndef _LP64
 5129 
 5130 // 32-bit x86 only: always use the linear search.
 5131 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 5132                                                    Register super_klass,
 5133                                                    Register temp_reg,
 5134                                                    Register temp2_reg,
 5135                                                    Label* L_success,
 5136                                                    Label* L_failure,
 5137                                                    bool set_cond_codes) {
 5138   check_klass_subtype_slow_path_linear
 5139     (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes);
 5140 }
 5141 
 5142 #else // _LP64
 5143 
 5144 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 5145                                                    Register super_klass,
 5146                                                    Register temp_reg,
 5147                                                    Register temp2_reg,
 5148                                                    Label* L_success,
 5149                                                    Label* L_failure,
 5150                                                    bool set_cond_codes) {
 5151   assert(set_cond_codes == false, "must be false on 64-bit x86");
 5152   check_klass_subtype_slow_path
 5153     (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg,
 5154      L_success, L_failure);
 5155 }
 5156 
 5157 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 5158                                                    Register super_klass,
 5159                                                    Register temp_reg,
 5160                                                    Register temp2_reg,
 5161                                                    Register temp3_reg,
 5162                                                    Register temp4_reg,
 5163                                                    Label* L_success,
 5164                                                    Label* L_failure) {
 5165   if (UseSecondarySupersTable) {
 5166     check_klass_subtype_slow_path_table
 5167       (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg,
 5168        L_success, L_failure);
 5169   } else {
 5170     check_klass_subtype_slow_path_linear
 5171       (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false);
 5172   }
 5173 }
 5174 
 5175 Register MacroAssembler::allocate_if_noreg(Register r,
 5176                                   RegSetIterator<Register> &available_regs,
 5177                                   RegSet &regs_to_push) {
 5178   if (!r->is_valid()) {
 5179     r = *available_regs++;
 5180     regs_to_push += r;
 5181   }
 5182   return r;
 5183 }
 5184 
 5185 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
 5186                                                          Register super_klass,
 5187                                                          Register temp_reg,
 5188                                                          Register temp2_reg,
 5189                                                          Register temp3_reg,
 5190                                                          Register result_reg,
 5191                                                          Label* L_success,
 5192                                                          Label* L_failure) {
 5193   // NB! Callers may assume that, when temp2_reg is a valid register,
 5194   // this code sets it to a nonzero value.
 5195   bool temp2_reg_was_valid = temp2_reg->is_valid();
 5196 
 5197   RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg);
 5198 
 5199   Label L_fallthrough;
 5200   int label_nulls = 0;
 5201   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 5202   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 5203   assert(label_nulls <= 1, "at most one null in the batch");
 5204 
 5205   BLOCK_COMMENT("check_klass_subtype_slow_path_table");
 5206 
 5207   RegSetIterator<Register> available_regs
 5208     = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin();
 5209 
 5210   RegSet pushed_regs;
 5211 
 5212   temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
 5213   temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
 5214   temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);
 5215   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
 5216   Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs);
 5217 
 5218   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg);
 5219 
 5220   {
 5221 
 5222     int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 5223     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 5224     subptr(rsp, aligned_size);
 5225     push_set(pushed_regs, 0);
 5226 
 5227     lookup_secondary_supers_table_var(sub_klass,
 5228                                       super_klass,
 5229                                       temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
 5230     cmpq(result_reg, 0);
 5231 
 5232     // Unspill the temp. registers:
 5233     pop_set(pushed_regs, 0);
 5234     // Increment SP but do not clobber flags.
 5235     lea(rsp, Address(rsp, aligned_size));
 5236   }
 5237 
 5238   if (temp2_reg_was_valid) {
 5239     movq(temp2_reg, 1);
 5240   }
 5241 
 5242   jcc(Assembler::notEqual, *L_failure);
 5243 
 5244   if (L_success != &L_fallthrough) {
 5245     jmp(*L_success);
 5246   }
 5247 
 5248   bind(L_fallthrough);
 5249 }
 5250 
 5251 // population_count variant for running without the POPCNT
 5252 // instruction, which was introduced with SSE4.2 in 2008.
 5253 void MacroAssembler::population_count(Register dst, Register src,
 5254                                       Register scratch1, Register scratch2) {
 5255   assert_different_registers(src, scratch1, scratch2);
 5256   if (UsePopCountInstruction) {
 5257     Assembler::popcntq(dst, src);
 5258   } else {
 5259     assert_different_registers(src, scratch1, scratch2);
 5260     assert_different_registers(dst, scratch1, scratch2);
 5261     Label loop, done;
 5262 
 5263     mov(scratch1, src);
 5264     // dst = 0;
 5265     // while(scratch1 != 0) {
 5266     //   dst++;
 5267     //   scratch1 &= (scratch1 - 1);
 5268     // }
 5269     xorl(dst, dst);
 5270     testq(scratch1, scratch1);
 5271     jccb(Assembler::equal, done);
 5272     {
 5273       bind(loop);
 5274       incq(dst);
 5275       movq(scratch2, scratch1);
 5276       decq(scratch2);
 5277       andq(scratch1, scratch2);
 5278       jccb(Assembler::notEqual, loop);
 5279     }
 5280     bind(done);
 5281   }
 5282 #ifdef ASSERT
 5283   mov64(scratch1, 0xCafeBabeDeadBeef);
 5284   movq(scratch2, scratch1);
 5285 #endif
 5286 }
 5287 
 5288 // Ensure that the inline code and the stub are using the same registers.
 5289 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
 5290 do {                                                                 \
 5291   assert(r_super_klass  == rax, "mismatch");                         \
 5292   assert(r_array_base   == rbx, "mismatch");                         \
 5293   assert(r_array_length == rcx, "mismatch");                         \
 5294   assert(r_array_index  == rdx, "mismatch");                         \
 5295   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
 5296   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
 5297   assert(result         == rdi || result      == noreg, "mismatch"); \
 5298 } while(0)
 5299 
 5300 // Versions of salq and rorq that don't need count to be in rcx
 5301 
 5302 void MacroAssembler::salq(Register dest, Register count) {
 5303   if (count == rcx) {
 5304     Assembler::salq(dest);
 5305   } else {
 5306     assert_different_registers(rcx, dest);
 5307     xchgq(rcx, count);
 5308     Assembler::salq(dest);
 5309     xchgq(rcx, count);
 5310   }
 5311 }
 5312 
 5313 void MacroAssembler::rorq(Register dest, Register count) {
 5314   if (count == rcx) {
 5315     Assembler::rorq(dest);
 5316   } else {
 5317     assert_different_registers(rcx, dest);
 5318     xchgq(rcx, count);
 5319     Assembler::rorq(dest);
 5320     xchgq(rcx, count);
 5321   }
 5322 }
 5323 
 5324 // Return true: we succeeded in generating this code
 5325 //
 5326 // At runtime, return 0 in result if r_super_klass is a superclass of
 5327 // r_sub_klass, otherwise return nonzero. Use this if you know the
 5328 // super_klass_slot of the class you're looking for. This is always
 5329 // the case for instanceof and checkcast.
 5330 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
 5331                                                          Register r_super_klass,
 5332                                                          Register temp1,
 5333                                                          Register temp2,
 5334                                                          Register temp3,
 5335                                                          Register temp4,
 5336                                                          Register result,
 5337                                                          u1 super_klass_slot) {
 5338   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 5339 
 5340   Label L_fallthrough, L_success, L_failure;
 5341 
 5342   BLOCK_COMMENT("lookup_secondary_supers_table {");
 5343 
 5344   const Register
 5345     r_array_index  = temp1,
 5346     r_array_length = temp2,
 5347     r_array_base   = temp3,
 5348     r_bitmap       = temp4;
 5349 
 5350   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 5351 
 5352   xorq(result, result); // = 0
 5353 
 5354   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5355   movq(r_array_index, r_bitmap);
 5356 
 5357   // First check the bitmap to see if super_klass might be present. If
 5358   // the bit is zero, we are certain that super_klass is not one of
 5359   // the secondary supers.
 5360   u1 bit = super_klass_slot;
 5361   {
 5362     // NB: If the count in a x86 shift instruction is 0, the flags are
 5363     // not affected, so we do a testq instead.
 5364     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
 5365     if (shift_count != 0) {
 5366       salq(r_array_index, shift_count);
 5367     } else {
 5368       testq(r_array_index, r_array_index);
 5369     }
 5370   }
 5371   // We test the MSB of r_array_index, i.e. its sign bit
 5372   jcc(Assembler::positive, L_failure);
 5373 
 5374   // Get the first array index that can contain super_klass into r_array_index.
 5375   if (bit != 0) {
 5376     population_count(r_array_index, r_array_index, temp2, temp3);
 5377   } else {
 5378     movl(r_array_index, 1);
 5379   }
 5380   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5381 
 5382   // We will consult the secondary-super array.
 5383   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5384 
 5385   // We're asserting that the first word in an Array<Klass*> is the
 5386   // length, and the second word is the first word of the data. If
 5387   // that ever changes, r_array_base will have to be adjusted here.
 5388   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5389   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5390 
 5391   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5392   jccb(Assembler::equal, L_success);
 5393 
 5394   // Is there another entry to check? Consult the bitmap.
 5395   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
 5396   jccb(Assembler::carryClear, L_failure);
 5397 
 5398   // Linear probe. Rotate the bitmap so that the next bit to test is
 5399   // in Bit 1.
 5400   if (bit != 0) {
 5401     rorq(r_bitmap, bit);
 5402   }
 5403 
 5404   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5405   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5406   // Kills: r_array_length.
 5407   // Returns: result.
 5408   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
 5409   // Result (0/1) is in rdi
 5410   jmpb(L_fallthrough);
 5411 
 5412   bind(L_failure);
 5413   incq(result); // 0 => 1
 5414 
 5415   bind(L_success);
 5416   // result = 0;
 5417 
 5418   bind(L_fallthrough);
 5419   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5420 
 5421   if (VerifySecondarySupers) {
 5422     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5423                                   temp1, temp2, temp3);
 5424   }
 5425 }
 5426 
 5427 // At runtime, return 0 in result if r_super_klass is a superclass of
 5428 // r_sub_klass, otherwise return nonzero. Use this version of
 5429 // lookup_secondary_supers_table() if you don't know ahead of time
 5430 // which superclass will be searched for. Used by interpreter and
 5431 // runtime stubs. It is larger and has somewhat greater latency than
 5432 // the version above, which takes a constant super_klass_slot.
 5433 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
 5434                                                        Register r_super_klass,
 5435                                                        Register temp1,
 5436                                                        Register temp2,
 5437                                                        Register temp3,
 5438                                                        Register temp4,
 5439                                                        Register result) {
 5440   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 5441   assert_different_registers(r_sub_klass, r_super_klass, rcx);
 5442   RegSet temps = RegSet::of(temp1, temp2, temp3, temp4);
 5443 
 5444   Label L_fallthrough, L_success, L_failure;
 5445 
 5446   BLOCK_COMMENT("lookup_secondary_supers_table {");
 5447 
 5448   RegSetIterator<Register> available_regs = (temps - rcx).begin();
 5449 
 5450   // FIXME. Once we are sure that all paths reaching this point really
 5451   // do pass rcx as one of our temps we can get rid of the following
 5452   // workaround.
 5453   assert(temps.contains(rcx), "fix this code");
 5454 
 5455   // We prefer to have our shift count in rcx. If rcx is one of our
 5456   // temps, use it for slot. If not, pick any of our temps.
 5457   Register slot;
 5458   if (!temps.contains(rcx)) {
 5459     slot = *available_regs++;
 5460   } else {
 5461     slot = rcx;
 5462   }
 5463 
 5464   const Register r_array_index = *available_regs++;
 5465   const Register r_bitmap      = *available_regs++;
 5466 
 5467   // The logic above guarantees this property, but we state it here.
 5468   assert_different_registers(r_array_index, r_bitmap, rcx);
 5469 
 5470   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5471   movq(r_array_index, r_bitmap);
 5472 
 5473   // First check the bitmap to see if super_klass might be present. If
 5474   // the bit is zero, we are certain that super_klass is not one of
 5475   // the secondary supers.
 5476   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5477   xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
 5478   salq(r_array_index, slot);
 5479 
 5480   testq(r_array_index, r_array_index);
 5481   // We test the MSB of r_array_index, i.e. its sign bit
 5482   jcc(Assembler::positive, L_failure);
 5483 
 5484   const Register r_array_base = *available_regs++;
 5485 
 5486   // Get the first array index that can contain super_klass into r_array_index.
 5487   // Note: Clobbers r_array_base and slot.
 5488   population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot);
 5489 
 5490   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5491 
 5492   // We will consult the secondary-super array.
 5493   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5494 
 5495   // We're asserting that the first word in an Array<Klass*> is the
 5496   // length, and the second word is the first word of the data. If
 5497   // that ever changes, r_array_base will have to be adjusted here.
 5498   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5499   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5500 
 5501   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5502   jccb(Assembler::equal, L_success);
 5503 
 5504   // Restore slot to its true value
 5505   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5506 
 5507   // Linear probe. Rotate the bitmap so that the next bit to test is
 5508   // in Bit 1.
 5509   rorq(r_bitmap, slot);
 5510 
 5511   // Is there another entry to check? Consult the bitmap.
 5512   btq(r_bitmap, 1);
 5513   jccb(Assembler::carryClear, L_failure);
 5514 
 5515   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5516   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5517   // Kills: r_array_length.
 5518   // Returns: result.
 5519   lookup_secondary_supers_table_slow_path(r_super_klass,
 5520                                           r_array_base,
 5521                                           r_array_index,
 5522                                           r_bitmap,
 5523                                           /*temp1*/result,
 5524                                           /*temp2*/slot,
 5525                                           &L_success,
 5526                                           nullptr);
 5527 
 5528   bind(L_failure);
 5529   movq(result, 1);
 5530   jmpb(L_fallthrough);
 5531 
 5532   bind(L_success);
 5533   xorq(result, result); // = 0
 5534 
 5535   bind(L_fallthrough);
 5536   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5537 
 5538   if (VerifySecondarySupers) {
 5539     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5540                                   temp1, temp2, temp3);
 5541   }
 5542 }
 5543 
 5544 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
 5545                                  Label* L_success, Label* L_failure) {
 5546   Label L_loop, L_fallthrough;
 5547   {
 5548     int label_nulls = 0;
 5549     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
 5550     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
 5551     assert(label_nulls <= 1, "at most one null in the batch");
 5552   }
 5553   bind(L_loop);
 5554   cmpq(value, Address(addr, count, Address::times_8));
 5555   jcc(Assembler::equal, *L_success);
 5556   addl(count, 1);
 5557   cmpl(count, limit);
 5558   jcc(Assembler::less, L_loop);
 5559 
 5560   if (&L_fallthrough != L_failure) {
 5561     jmp(*L_failure);
 5562   }
 5563   bind(L_fallthrough);
 5564 }
 5565 
 5566 // Called by code generated by check_klass_subtype_slow_path
 5567 // above. This is called when there is a collision in the hashed
 5568 // lookup in the secondary supers array.
 5569 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
 5570                                                              Register r_array_base,
 5571                                                              Register r_array_index,
 5572                                                              Register r_bitmap,
 5573                                                              Register temp1,
 5574                                                              Register temp2,
 5575                                                              Label* L_success,
 5576                                                              Label* L_failure) {
 5577   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
 5578 
 5579   const Register
 5580     r_array_length = temp1,
 5581     r_sub_klass    = noreg,
 5582     result         = noreg;
 5583 
 5584   Label L_fallthrough;
 5585   int label_nulls = 0;
 5586   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 5587   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 5588   assert(label_nulls <= 1, "at most one null in the batch");
 5589 
 5590   // Load the array length.
 5591   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5592   // And adjust the array base to point to the data.
 5593   // NB! Effectively increments current slot index by 1.
 5594   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
 5595   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5596 
 5597   // Linear probe
 5598   Label L_huge;
 5599 
 5600   // The bitmap is full to bursting.
 5601   // Implicit invariant: BITMAP_FULL implies (length > 0)
 5602   cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
 5603   jcc(Assembler::greater, L_huge);
 5604 
 5605   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
 5606   // current slot (at secondary_supers[r_array_index]) has not yet
 5607   // been inspected, and r_array_index may be out of bounds if we
 5608   // wrapped around the end of the array.
 5609 
 5610   { // This is conventional linear probing, but instead of terminating
 5611     // when a null entry is found in the table, we maintain a bitmap
 5612     // in which a 0 indicates missing entries.
 5613     // The check above guarantees there are 0s in the bitmap, so the loop
 5614     // eventually terminates.
 5615 
 5616     xorl(temp2, temp2); // = 0;
 5617 
 5618     Label L_again;
 5619     bind(L_again);
 5620 
 5621     // Check for array wraparound.
 5622     cmpl(r_array_index, r_array_length);
 5623     cmovl(Assembler::greaterEqual, r_array_index, temp2);
 5624 
 5625     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5626     jcc(Assembler::equal, *L_success);
 5627 
 5628     // If the next bit in bitmap is zero, we're done.
 5629     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
 5630     jcc(Assembler::carryClear, *L_failure);
 5631 
 5632     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
 5633     addl(r_array_index, 1);
 5634 
 5635     jmp(L_again);
 5636   }
 5637 
 5638   { // Degenerate case: more than 64 secondary supers.
 5639     // FIXME: We could do something smarter here, maybe a vectorized
 5640     // comparison or a binary search, but is that worth any added
 5641     // complexity?
 5642     bind(L_huge);
 5643     xorl(r_array_index, r_array_index); // = 0
 5644     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
 5645                 L_success,
 5646                 (&L_fallthrough != L_failure ? L_failure : nullptr));
 5647 
 5648     bind(L_fallthrough);
 5649   }
 5650 }
 5651 
 5652 struct VerifyHelperArguments {
 5653   Klass* _super;
 5654   Klass* _sub;
 5655   intptr_t _linear_result;
 5656   intptr_t _table_result;
 5657 };
 5658 
 5659 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
 5660   Klass::on_secondary_supers_verification_failure(args->_super,
 5661                                                   args->_sub,
 5662                                                   args->_linear_result,
 5663                                                   args->_table_result,
 5664                                                   msg);
 5665 }
 5666 
 5667 // Make sure that the hashed lookup and a linear scan agree.
 5668 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
 5669                                                    Register r_super_klass,
 5670                                                    Register result,
 5671                                                    Register temp1,
 5672                                                    Register temp2,
 5673                                                    Register temp3) {
 5674   const Register
 5675       r_array_index  = temp1,
 5676       r_array_length = temp2,
 5677       r_array_base   = temp3,
 5678       r_bitmap       = noreg;
 5679 
 5680   BLOCK_COMMENT("verify_secondary_supers_table {");
 5681 
 5682   Label L_success, L_failure, L_check, L_done;
 5683 
 5684   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5685   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5686   // And adjust the array base to point to the data.
 5687   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5688 
 5689   testl(r_array_length, r_array_length); // array_length == 0?
 5690   jcc(Assembler::zero, L_failure);
 5691 
 5692   movl(r_array_index, 0);
 5693   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
 5694   // fall through to L_failure
 5695 
 5696   const Register linear_result = r_array_index; // reuse temp1
 5697 
 5698   bind(L_failure); // not present
 5699   movl(linear_result, 1);
 5700   jmp(L_check);
 5701 
 5702   bind(L_success); // present
 5703   movl(linear_result, 0);
 5704 
 5705   bind(L_check);
 5706   cmpl(linear_result, result);
 5707   jcc(Assembler::equal, L_done);
 5708 
 5709   { // To avoid calling convention issues, build a record on the stack
 5710     // and pass the pointer to that instead.
 5711     push(result);
 5712     push(linear_result);
 5713     push(r_sub_klass);
 5714     push(r_super_klass);
 5715     movptr(c_rarg1, rsp);
 5716     movptr(c_rarg0, (uintptr_t) "mismatch");
 5717     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
 5718     should_not_reach_here();
 5719   }
 5720   bind(L_done);
 5721 
 5722   BLOCK_COMMENT("} verify_secondary_supers_table");
 5723 }
 5724 
 5725 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
 5726 
 5727 #endif // LP64
 5728 
 5729 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 5730   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5731 
 5732   Label L_fallthrough;
 5733   if (L_fast_path == nullptr) {
 5734     L_fast_path = &L_fallthrough;
 5735   } else if (L_slow_path == nullptr) {
 5736     L_slow_path = &L_fallthrough;
 5737   }
 5738 
 5739   // Fast path check: class is fully initialized.
 5740   // init_state needs acquire, but x86 is TSO, and so we are already good.
 5741   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5742   jcc(Assembler::equal, *L_fast_path);
 5743 
 5744   // Fast path check: current thread is initializer thread
 5745   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 5746   if (L_slow_path == &L_fallthrough) {
 5747     jcc(Assembler::equal, *L_fast_path);
 5748     bind(*L_slow_path);
 5749   } else if (L_fast_path == &L_fallthrough) {
 5750     jcc(Assembler::notEqual, *L_slow_path);
 5751     bind(*L_fast_path);
 5752   } else {
 5753     Unimplemented();
 5754   }
 5755 }
 5756 
 5757 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5758   if (VM_Version::supports_cmov()) {
 5759     cmovl(cc, dst, src);
 5760   } else {
 5761     Label L;
 5762     jccb(negate_condition(cc), L);
 5763     movl(dst, src);
 5764     bind(L);
 5765   }
 5766 }
 5767 
 5768 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5769   if (VM_Version::supports_cmov()) {
 5770     cmovl(cc, dst, src);
 5771   } else {
 5772     Label L;
 5773     jccb(negate_condition(cc), L);
 5774     movl(dst, src);
 5775     bind(L);
 5776   }
 5777 }
 5778 
 5779 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5780   if (!VerifyOops || VerifyAdapterSharing) {
 5781     // Below address of the code string confuses VerifyAdapterSharing
 5782     // because it may differ between otherwise equivalent adapters.
 5783     return;
 5784   }
 5785 
 5786   BLOCK_COMMENT("verify_oop {");
 5787 #ifdef _LP64
 5788   push(rscratch1);
 5789 #endif
 5790   push(rax);                          // save rax
 5791   push(reg);                          // pass register argument
 5792 
 5793   // Pass register number to verify_oop_subroutine
 5794   const char* b = nullptr;
 5795   {
 5796     ResourceMark rm;
 5797     stringStream ss;
 5798     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5799     b = code_string(ss.as_string());
 5800   }
 5801   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5802   pushptr(buffer.addr(), rscratch1);
 5803 
 5804   // call indirectly to solve generation ordering problem
 5805   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5806   call(rax);
 5807   // Caller pops the arguments (oop, message) and restores rax, r10
 5808   BLOCK_COMMENT("} verify_oop");
 5809 }
 5810 
 5811 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5812   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5813     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5814     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5815     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5816   } else if (VM_Version::supports_avx()) {
 5817     vpcmpeqd(dst, dst, dst, vector_len);
 5818   } else {
 5819     pcmpeqd(dst, dst);
 5820   }
 5821 }
 5822 
 5823 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5824                                          int extra_slot_offset) {
 5825   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5826   int stackElementSize = Interpreter::stackElementSize;
 5827   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5828 #ifdef ASSERT
 5829   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5830   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5831 #endif
 5832   Register             scale_reg    = noreg;
 5833   Address::ScaleFactor scale_factor = Address::no_scale;
 5834   if (arg_slot.is_constant()) {
 5835     offset += arg_slot.as_constant() * stackElementSize;
 5836   } else {
 5837     scale_reg    = arg_slot.as_register();
 5838     scale_factor = Address::times(stackElementSize);
 5839   }
 5840   offset += wordSize;           // return PC is on stack
 5841   return Address(rsp, scale_reg, scale_factor, offset);
 5842 }
 5843 
 5844 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5845   if (!VerifyOops || VerifyAdapterSharing) {
 5846     // Below address of the code string confuses VerifyAdapterSharing
 5847     // because it may differ between otherwise equivalent adapters.
 5848     return;
 5849   }
 5850 
 5851 #ifdef _LP64
 5852   push(rscratch1);
 5853 #endif
 5854   push(rax); // save rax,
 5855   // addr may contain rsp so we will have to adjust it based on the push
 5856   // we just did (and on 64 bit we do two pushes)
 5857   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5858   // stores rax into addr which is backwards of what was intended.
 5859   if (addr.uses(rsp)) {
 5860     lea(rax, addr);
 5861     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5862   } else {
 5863     pushptr(addr);
 5864   }
 5865 
 5866   // Pass register number to verify_oop_subroutine
 5867   const char* b = nullptr;
 5868   {
 5869     ResourceMark rm;
 5870     stringStream ss;
 5871     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5872     b = code_string(ss.as_string());
 5873   }
 5874   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5875   pushptr(buffer.addr(), rscratch1);
 5876 
 5877   // call indirectly to solve generation ordering problem
 5878   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5879   call(rax);
 5880   // Caller pops the arguments (addr, message) and restores rax, r10.
 5881 }
 5882 
 5883 void MacroAssembler::verify_tlab() {
 5884 #ifdef ASSERT
 5885   if (UseTLAB && VerifyOops) {
 5886     Label next, ok;
 5887     Register t1 = rsi;
 5888     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5889 
 5890     push(t1);
 5891     NOT_LP64(push(thread_reg));
 5892     NOT_LP64(get_thread(thread_reg));
 5893 
 5894     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5895     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5896     jcc(Assembler::aboveEqual, next);
 5897     STOP("assert(top >= start)");
 5898     should_not_reach_here();
 5899 
 5900     bind(next);
 5901     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5902     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5903     jcc(Assembler::aboveEqual, ok);
 5904     STOP("assert(top <= end)");
 5905     should_not_reach_here();
 5906 
 5907     bind(ok);
 5908     NOT_LP64(pop(thread_reg));
 5909     pop(t1);
 5910   }
 5911 #endif
 5912 }
 5913 
 5914 class ControlWord {
 5915  public:
 5916   int32_t _value;
 5917 
 5918   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5919   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5920   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5921   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5922   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5923   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5924   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5925   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5926 
 5927   void print() const {
 5928     // rounding control
 5929     const char* rc;
 5930     switch (rounding_control()) {
 5931       case 0: rc = "round near"; break;
 5932       case 1: rc = "round down"; break;
 5933       case 2: rc = "round up  "; break;
 5934       case 3: rc = "chop      "; break;
 5935       default:
 5936         rc = nullptr; // silence compiler warnings
 5937         fatal("Unknown rounding control: %d", rounding_control());
 5938     };
 5939     // precision control
 5940     const char* pc;
 5941     switch (precision_control()) {
 5942       case 0: pc = "24 bits "; break;
 5943       case 1: pc = "reserved"; break;
 5944       case 2: pc = "53 bits "; break;
 5945       case 3: pc = "64 bits "; break;
 5946       default:
 5947         pc = nullptr; // silence compiler warnings
 5948         fatal("Unknown precision control: %d", precision_control());
 5949     };
 5950     // flags
 5951     char f[9];
 5952     f[0] = ' ';
 5953     f[1] = ' ';
 5954     f[2] = (precision   ()) ? 'P' : 'p';
 5955     f[3] = (underflow   ()) ? 'U' : 'u';
 5956     f[4] = (overflow    ()) ? 'O' : 'o';
 5957     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5958     f[6] = (denormalized()) ? 'D' : 'd';
 5959     f[7] = (invalid     ()) ? 'I' : 'i';
 5960     f[8] = '\x0';
 5961     // output
 5962     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5963   }
 5964 
 5965 };
 5966 
 5967 class StatusWord {
 5968  public:
 5969   int32_t _value;
 5970 
 5971   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5972   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5973   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5974   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5975   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5976   int  top() const                     { return  (_value >> 11) & 7      ; }
 5977   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5978   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5979   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5980   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5981   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5982   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5983   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5984   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5985 
 5986   void print() const {
 5987     // condition codes
 5988     char c[5];
 5989     c[0] = (C3()) ? '3' : '-';
 5990     c[1] = (C2()) ? '2' : '-';
 5991     c[2] = (C1()) ? '1' : '-';
 5992     c[3] = (C0()) ? '0' : '-';
 5993     c[4] = '\x0';
 5994     // flags
 5995     char f[9];
 5996     f[0] = (error_status()) ? 'E' : '-';
 5997     f[1] = (stack_fault ()) ? 'S' : '-';
 5998     f[2] = (precision   ()) ? 'P' : '-';
 5999     f[3] = (underflow   ()) ? 'U' : '-';
 6000     f[4] = (overflow    ()) ? 'O' : '-';
 6001     f[5] = (zero_divide ()) ? 'Z' : '-';
 6002     f[6] = (denormalized()) ? 'D' : '-';
 6003     f[7] = (invalid     ()) ? 'I' : '-';
 6004     f[8] = '\x0';
 6005     // output
 6006     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 6007   }
 6008 
 6009 };
 6010 
 6011 class TagWord {
 6012  public:
 6013   int32_t _value;
 6014 
 6015   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 6016 
 6017   void print() const {
 6018     printf("%04x", _value & 0xFFFF);
 6019   }
 6020 
 6021 };
 6022 
 6023 class FPU_Register {
 6024  public:
 6025   int32_t _m0;
 6026   int32_t _m1;
 6027   int16_t _ex;
 6028 
 6029   bool is_indefinite() const           {
 6030     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 6031   }
 6032 
 6033   void print() const {
 6034     char  sign = (_ex < 0) ? '-' : '+';
 6035     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 6036     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 6037   };
 6038 
 6039 };
 6040 
 6041 class FPU_State {
 6042  public:
 6043   enum {
 6044     register_size       = 10,
 6045     number_of_registers =  8,
 6046     register_mask       =  7
 6047   };
 6048 
 6049   ControlWord  _control_word;
 6050   StatusWord   _status_word;
 6051   TagWord      _tag_word;
 6052   int32_t      _error_offset;
 6053   int32_t      _error_selector;
 6054   int32_t      _data_offset;
 6055   int32_t      _data_selector;
 6056   int8_t       _register[register_size * number_of_registers];
 6057 
 6058   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 6059   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 6060 
 6061   const char* tag_as_string(int tag) const {
 6062     switch (tag) {
 6063       case 0: return "valid";
 6064       case 1: return "zero";
 6065       case 2: return "special";
 6066       case 3: return "empty";
 6067     }
 6068     ShouldNotReachHere();
 6069     return nullptr;
 6070   }
 6071 
 6072   void print() const {
 6073     // print computation registers
 6074     { int t = _status_word.top();
 6075       for (int i = 0; i < number_of_registers; i++) {
 6076         int j = (i - t) & register_mask;
 6077         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 6078         st(j)->print();
 6079         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 6080       }
 6081     }
 6082     printf("\n");
 6083     // print control registers
 6084     printf("ctrl = "); _control_word.print(); printf("\n");
 6085     printf("stat = "); _status_word .print(); printf("\n");
 6086     printf("tags = "); _tag_word    .print(); printf("\n");
 6087   }
 6088 
 6089 };
 6090 
 6091 class Flag_Register {
 6092  public:
 6093   int32_t _value;
 6094 
 6095   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 6096   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 6097   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 6098   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 6099   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 6100   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 6101   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 6102 
 6103   void print() const {
 6104     // flags
 6105     char f[8];
 6106     f[0] = (overflow       ()) ? 'O' : '-';
 6107     f[1] = (direction      ()) ? 'D' : '-';
 6108     f[2] = (sign           ()) ? 'S' : '-';
 6109     f[3] = (zero           ()) ? 'Z' : '-';
 6110     f[4] = (auxiliary_carry()) ? 'A' : '-';
 6111     f[5] = (parity         ()) ? 'P' : '-';
 6112     f[6] = (carry          ()) ? 'C' : '-';
 6113     f[7] = '\x0';
 6114     // output
 6115     printf("%08x  flags = %s", _value, f);
 6116   }
 6117 
 6118 };
 6119 
 6120 class IU_Register {
 6121  public:
 6122   int32_t _value;
 6123 
 6124   void print() const {
 6125     printf("%08x  %11d", _value, _value);
 6126   }
 6127 
 6128 };
 6129 
 6130 class IU_State {
 6131  public:
 6132   Flag_Register _eflags;
 6133   IU_Register   _rdi;
 6134   IU_Register   _rsi;
 6135   IU_Register   _rbp;
 6136   IU_Register   _rsp;
 6137   IU_Register   _rbx;
 6138   IU_Register   _rdx;
 6139   IU_Register   _rcx;
 6140   IU_Register   _rax;
 6141 
 6142   void print() const {
 6143     // computation registers
 6144     printf("rax,  = "); _rax.print(); printf("\n");
 6145     printf("rbx,  = "); _rbx.print(); printf("\n");
 6146     printf("rcx  = "); _rcx.print(); printf("\n");
 6147     printf("rdx  = "); _rdx.print(); printf("\n");
 6148     printf("rdi  = "); _rdi.print(); printf("\n");
 6149     printf("rsi  = "); _rsi.print(); printf("\n");
 6150     printf("rbp,  = "); _rbp.print(); printf("\n");
 6151     printf("rsp  = "); _rsp.print(); printf("\n");
 6152     printf("\n");
 6153     // control registers
 6154     printf("flgs = "); _eflags.print(); printf("\n");
 6155   }
 6156 };
 6157 
 6158 
 6159 class CPU_State {
 6160  public:
 6161   FPU_State _fpu_state;
 6162   IU_State  _iu_state;
 6163 
 6164   void print() const {
 6165     printf("--------------------------------------------------\n");
 6166     _iu_state .print();
 6167     printf("\n");
 6168     _fpu_state.print();
 6169     printf("--------------------------------------------------\n");
 6170   }
 6171 
 6172 };
 6173 
 6174 
 6175 static void _print_CPU_state(CPU_State* state) {
 6176   state->print();
 6177 };
 6178 
 6179 
 6180 void MacroAssembler::print_CPU_state() {
 6181   push_CPU_state();
 6182   push(rsp);                // pass CPU state
 6183   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 6184   addptr(rsp, wordSize);       // discard argument
 6185   pop_CPU_state();
 6186 }
 6187 
 6188 
 6189 #ifndef _LP64
 6190 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 6191   static int counter = 0;
 6192   FPU_State* fs = &state->_fpu_state;
 6193   counter++;
 6194   // For leaf calls, only verify that the top few elements remain empty.
 6195   // We only need 1 empty at the top for C2 code.
 6196   if( stack_depth < 0 ) {
 6197     if( fs->tag_for_st(7) != 3 ) {
 6198       printf("FPR7 not empty\n");
 6199       state->print();
 6200       assert(false, "error");
 6201       return false;
 6202     }
 6203     return true;                // All other stack states do not matter
 6204   }
 6205 
 6206   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 6207          "bad FPU control word");
 6208 
 6209   // compute stack depth
 6210   int i = 0;
 6211   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 6212   int d = i;
 6213   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 6214   // verify findings
 6215   if (i != FPU_State::number_of_registers) {
 6216     // stack not contiguous
 6217     printf("%s: stack not contiguous at ST%d\n", s, i);
 6218     state->print();
 6219     assert(false, "error");
 6220     return false;
 6221   }
 6222   // check if computed stack depth corresponds to expected stack depth
 6223   if (stack_depth < 0) {
 6224     // expected stack depth is -stack_depth or less
 6225     if (d > -stack_depth) {
 6226       // too many elements on the stack
 6227       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 6228       state->print();
 6229       assert(false, "error");
 6230       return false;
 6231     }
 6232   } else {
 6233     // expected stack depth is stack_depth
 6234     if (d != stack_depth) {
 6235       // wrong stack depth
 6236       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 6237       state->print();
 6238       assert(false, "error");
 6239       return false;
 6240     }
 6241   }
 6242   // everything is cool
 6243   return true;
 6244 }
 6245 
 6246 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 6247   if (!VerifyFPU) return;
 6248   push_CPU_state();
 6249   push(rsp);                // pass CPU state
 6250   ExternalAddress msg((address) s);
 6251   // pass message string s
 6252   pushptr(msg.addr(), noreg);
 6253   push(stack_depth);        // pass stack depth
 6254   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 6255   addptr(rsp, 3 * wordSize);   // discard arguments
 6256   // check for error
 6257   { Label L;
 6258     testl(rax, rax);
 6259     jcc(Assembler::notZero, L);
 6260     int3();                  // break if error condition
 6261     bind(L);
 6262   }
 6263   pop_CPU_state();
 6264 }
 6265 #endif // _LP64
 6266 
 6267 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 6268   // Either restore the MXCSR register after returning from the JNI Call
 6269   // or verify that it wasn't changed (with -Xcheck:jni flag).
 6270   if (VM_Version::supports_sse()) {
 6271     if (RestoreMXCSROnJNICalls) {
 6272       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 6273     } else if (CheckJNICalls) {
 6274       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 6275     }
 6276   }
 6277   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 6278   vzeroupper();
 6279 
 6280 #ifndef _LP64
 6281   // Either restore the x87 floating pointer control word after returning
 6282   // from the JNI call or verify that it wasn't changed.
 6283   if (CheckJNICalls) {
 6284     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 6285   }
 6286 #endif // _LP64
 6287 }
 6288 
 6289 // ((OopHandle)result).resolve();
 6290 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 6291   assert_different_registers(result, tmp);
 6292 
 6293   // Only 64 bit platforms support GCs that require a tmp register
 6294   // Only IN_HEAP loads require a thread_tmp register
 6295   // OopHandle::resolve is an indirection like jobject.
 6296   access_load_at(T_OBJECT, IN_NATIVE,
 6297                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 6298 }
 6299 
 6300 // ((WeakHandle)result).resolve();
 6301 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 6302   assert_different_registers(rresult, rtmp);
 6303   Label resolved;
 6304 
 6305   // A null weak handle resolves to null.
 6306   cmpptr(rresult, 0);
 6307   jcc(Assembler::equal, resolved);
 6308 
 6309   // Only 64 bit platforms support GCs that require a tmp register
 6310   // Only IN_HEAP loads require a thread_tmp register
 6311   // WeakHandle::resolve is an indirection like jweak.
 6312   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 6313                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 6314   bind(resolved);
 6315 }
 6316 
 6317 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 6318   // get mirror
 6319   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 6320   load_method_holder(mirror, method);
 6321   movptr(mirror, Address(mirror, mirror_offset));
 6322   resolve_oop_handle(mirror, tmp);
 6323 }
 6324 
 6325 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 6326   load_method_holder(rresult, rmethod);
 6327   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 6328 }
 6329 
 6330 void MacroAssembler::load_method_holder(Register holder, Register method) {
 6331   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 6332   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 6333   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 6334 }
 6335 
 6336 void MacroAssembler::load_metadata(Register dst, Register src) {
 6337 #ifdef _LP64
 6338   if (UseCompactObjectHeaders) {
 6339     load_narrow_klass_compact(dst, src);
 6340   } else if (UseCompressedClassPointers) {
 6341     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6342   } else
 6343 #endif
 6344   {
 6345     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6346   }
 6347 }
 6348 
 6349 #ifdef _LP64
 6350 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
 6351   assert(UseCompactObjectHeaders, "expect compact object headers");
 6352   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
 6353   shrq(dst, markWord::klass_shift);
 6354 }
 6355 #endif
 6356 
 6357 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 6358   assert_different_registers(src, tmp);
 6359   assert_different_registers(dst, tmp);
 6360 #ifdef _LP64
 6361   if (UseCompactObjectHeaders) {
 6362     load_narrow_klass_compact(dst, src);
 6363     decode_klass_not_null(dst, tmp);
 6364   } else if (UseCompressedClassPointers) {
 6365     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6366     decode_klass_not_null(dst, tmp);
 6367   } else
 6368 #endif
 6369   {
 6370     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6371   }
 6372 }
 6373 
 6374 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
 6375   load_klass(dst, src, tmp);
 6376   movptr(dst, Address(dst, Klass::prototype_header_offset()));
 6377 }
 6378 
 6379 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 6380   assert(!UseCompactObjectHeaders, "not with compact headers");
 6381   assert_different_registers(src, tmp);
 6382   assert_different_registers(dst, tmp);
 6383 #ifdef _LP64
 6384   if (UseCompressedClassPointers) {
 6385     encode_klass_not_null(src, tmp);
 6386     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6387   } else
 6388 #endif
 6389     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6390 }
 6391 
 6392 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
 6393 #ifdef _LP64
 6394   if (UseCompactObjectHeaders) {
 6395     assert(tmp != noreg, "need tmp");
 6396     assert_different_registers(klass, obj, tmp);
 6397     load_narrow_klass_compact(tmp, obj);
 6398     cmpl(klass, tmp);
 6399   } else if (UseCompressedClassPointers) {
 6400     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6401   } else
 6402 #endif
 6403   {
 6404     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6405   }
 6406 }
 6407 
 6408 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
 6409 #ifdef _LP64
 6410   if (UseCompactObjectHeaders) {
 6411     assert(tmp2 != noreg, "need tmp2");
 6412     assert_different_registers(obj1, obj2, tmp1, tmp2);
 6413     load_narrow_klass_compact(tmp1, obj1);
 6414     load_narrow_klass_compact(tmp2, obj2);
 6415     cmpl(tmp1, tmp2);
 6416   } else if (UseCompressedClassPointers) {
 6417     movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6418     cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6419   } else
 6420 #endif
 6421   {
 6422     movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6423     cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6424   }
 6425 }
 6426 
 6427 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 6428                                     Register tmp1, Register thread_tmp) {
 6429   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6430   decorators = AccessInternal::decorator_fixup(decorators, type);
 6431   bool as_raw = (decorators & AS_RAW) != 0;
 6432   if (as_raw) {
 6433     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6434   } else {
 6435     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6436   }
 6437 }
 6438 
 6439 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 6440                                      Register tmp1, Register tmp2, Register tmp3) {
 6441   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6442   decorators = AccessInternal::decorator_fixup(decorators, type);
 6443   bool as_raw = (decorators & AS_RAW) != 0;
 6444   if (as_raw) {
 6445     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6446   } else {
 6447     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6448   }
 6449 }
 6450 
 6451 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
 6452                                      Register inline_layout_info) {
 6453   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6454   bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
 6455 }
 6456 
 6457 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
 6458   movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6459   movl(offset, Address(offset, InlineKlass::payload_offset_offset()));
 6460 }
 6461 
 6462 void MacroAssembler::payload_addr(Register oop, Register data, Register inline_klass) {
 6463   // ((address) (void*) o) + vk->payload_offset();
 6464   Register offset = (data == oop) ? rscratch1 : data;
 6465   payload_offset(inline_klass, offset);
 6466   if (data == oop) {
 6467     addptr(data, offset);
 6468   } else {
 6469     lea(data, Address(oop, offset));
 6470   }
 6471 }
 6472 
 6473 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
 6474                                                 Register index, Register data) {
 6475   assert(index != rcx, "index needs to shift by rcx");
 6476   assert_different_registers(array, array_klass, index);
 6477   assert_different_registers(rcx, array, index);
 6478 
 6479   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
 6480   movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
 6481 
 6482   // Klass::layout_helper_log2_element_size(lh)
 6483   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
 6484   shrl(rcx, Klass::_lh_log2_element_size_shift);
 6485   andl(rcx, Klass::_lh_log2_element_size_mask);
 6486   shlptr(index); // index << rcx
 6487 
 6488   lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT)));
 6489 }
 6490 
 6491 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 6492                                    Register thread_tmp, DecoratorSet decorators) {
 6493   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 6494 }
 6495 
 6496 // Doesn't do verification, generates fixed size code
 6497 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 6498                                             Register thread_tmp, DecoratorSet decorators) {
 6499   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 6500 }
 6501 
 6502 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 6503                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 6504   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 6505 }
 6506 
 6507 // Used for storing nulls.
 6508 void MacroAssembler::store_heap_oop_null(Address dst) {
 6509   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 6510 }
 6511 
 6512 #ifdef _LP64
 6513 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 6514   assert(!UseCompactObjectHeaders, "Don't use with compact headers");
 6515   if (UseCompressedClassPointers) {
 6516     // Store to klass gap in destination
 6517     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 6518   }
 6519 }
 6520 
 6521 #ifdef ASSERT
 6522 void MacroAssembler::verify_heapbase(const char* msg) {
 6523   assert (UseCompressedOops, "should be compressed");
 6524   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6525   if (CheckCompressedOops) {
 6526     Label ok;
 6527     ExternalAddress src2(CompressedOops::base_addr());
 6528     const bool is_src2_reachable = reachable(src2);
 6529     if (!is_src2_reachable) {
 6530       push(rscratch1);  // cmpptr trashes rscratch1
 6531     }
 6532     cmpptr(r12_heapbase, src2, rscratch1);
 6533     jcc(Assembler::equal, ok);
 6534     STOP(msg);
 6535     bind(ok);
 6536     if (!is_src2_reachable) {
 6537       pop(rscratch1);
 6538     }
 6539   }
 6540 }
 6541 #endif
 6542 
 6543 // Algorithm must match oop.inline.hpp encode_heap_oop.
 6544 void MacroAssembler::encode_heap_oop(Register r) {
 6545 #ifdef ASSERT
 6546   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 6547 #endif
 6548   verify_oop_msg(r, "broken oop in encode_heap_oop");
 6549   if (CompressedOops::base() == nullptr) {
 6550     if (CompressedOops::shift() != 0) {
 6551       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6552       shrq(r, LogMinObjAlignmentInBytes);
 6553     }
 6554     return;
 6555   }
 6556   testq(r, r);
 6557   cmovq(Assembler::equal, r, r12_heapbase);
 6558   subq(r, r12_heapbase);
 6559   shrq(r, LogMinObjAlignmentInBytes);
 6560 }
 6561 
 6562 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 6563 #ifdef ASSERT
 6564   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 6565   if (CheckCompressedOops) {
 6566     Label ok;
 6567     testq(r, r);
 6568     jcc(Assembler::notEqual, ok);
 6569     STOP("null oop passed to encode_heap_oop_not_null");
 6570     bind(ok);
 6571   }
 6572 #endif
 6573   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 6574   if (CompressedOops::base() != nullptr) {
 6575     subq(r, r12_heapbase);
 6576   }
 6577   if (CompressedOops::shift() != 0) {
 6578     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6579     shrq(r, LogMinObjAlignmentInBytes);
 6580   }
 6581 }
 6582 
 6583 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 6584 #ifdef ASSERT
 6585   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 6586   if (CheckCompressedOops) {
 6587     Label ok;
 6588     testq(src, src);
 6589     jcc(Assembler::notEqual, ok);
 6590     STOP("null oop passed to encode_heap_oop_not_null2");
 6591     bind(ok);
 6592   }
 6593 #endif
 6594   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 6595   if (dst != src) {
 6596     movq(dst, src);
 6597   }
 6598   if (CompressedOops::base() != nullptr) {
 6599     subq(dst, r12_heapbase);
 6600   }
 6601   if (CompressedOops::shift() != 0) {
 6602     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6603     shrq(dst, LogMinObjAlignmentInBytes);
 6604   }
 6605 }
 6606 
 6607 void  MacroAssembler::decode_heap_oop(Register r) {
 6608 #ifdef ASSERT
 6609   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 6610 #endif
 6611   if (CompressedOops::base() == nullptr) {
 6612     if (CompressedOops::shift() != 0) {
 6613       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6614       shlq(r, LogMinObjAlignmentInBytes);
 6615     }
 6616   } else {
 6617     Label done;
 6618     shlq(r, LogMinObjAlignmentInBytes);
 6619     jccb(Assembler::equal, done);
 6620     addq(r, r12_heapbase);
 6621     bind(done);
 6622   }
 6623   verify_oop_msg(r, "broken oop in decode_heap_oop");
 6624 }
 6625 
 6626 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 6627   // Note: it will change flags
 6628   assert (UseCompressedOops, "should only be used for compressed headers");
 6629   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6630   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6631   // vtableStubs also counts instructions in pd_code_size_limit.
 6632   // Also do not verify_oop as this is called by verify_oop.
 6633   if (CompressedOops::shift() != 0) {
 6634     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6635     shlq(r, LogMinObjAlignmentInBytes);
 6636     if (CompressedOops::base() != nullptr) {
 6637       addq(r, r12_heapbase);
 6638     }
 6639   } else {
 6640     assert (CompressedOops::base() == nullptr, "sanity");
 6641   }
 6642 }
 6643 
 6644 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 6645   // Note: it will change flags
 6646   assert (UseCompressedOops, "should only be used for compressed headers");
 6647   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6648   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6649   // vtableStubs also counts instructions in pd_code_size_limit.
 6650   // Also do not verify_oop as this is called by verify_oop.
 6651   if (CompressedOops::shift() != 0) {
 6652     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6653     if (LogMinObjAlignmentInBytes == Address::times_8) {
 6654       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 6655     } else {
 6656       if (dst != src) {
 6657         movq(dst, src);
 6658       }
 6659       shlq(dst, LogMinObjAlignmentInBytes);
 6660       if (CompressedOops::base() != nullptr) {
 6661         addq(dst, r12_heapbase);
 6662       }
 6663     }
 6664   } else {
 6665     assert (CompressedOops::base() == nullptr, "sanity");
 6666     if (dst != src) {
 6667       movq(dst, src);
 6668     }
 6669   }
 6670 }
 6671 
 6672 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 6673   assert_different_registers(r, tmp);
 6674   if (CompressedKlassPointers::base() != nullptr) {
 6675     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6676     subq(r, tmp);
 6677   }
 6678   if (CompressedKlassPointers::shift() != 0) {
 6679     shrq(r, CompressedKlassPointers::shift());
 6680   }
 6681 }
 6682 
 6683 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 6684   assert_different_registers(src, dst);
 6685   if (CompressedKlassPointers::base() != nullptr) {
 6686     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 6687     addq(dst, src);
 6688   } else {
 6689     movptr(dst, src);
 6690   }
 6691   if (CompressedKlassPointers::shift() != 0) {
 6692     shrq(dst, CompressedKlassPointers::shift());
 6693   }
 6694 }
 6695 
 6696 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 6697   assert_different_registers(r, tmp);
 6698   // Note: it will change flags
 6699   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 6700   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6701   // vtableStubs also counts instructions in pd_code_size_limit.
 6702   // Also do not verify_oop as this is called by verify_oop.
 6703   if (CompressedKlassPointers::shift() != 0) {
 6704     shlq(r, CompressedKlassPointers::shift());
 6705   }
 6706   if (CompressedKlassPointers::base() != nullptr) {
 6707     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6708     addq(r, tmp);
 6709   }
 6710 }
 6711 
 6712 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 6713   assert_different_registers(src, dst);
 6714   // Note: it will change flags
 6715   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6716   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6717   // vtableStubs also counts instructions in pd_code_size_limit.
 6718   // Also do not verify_oop as this is called by verify_oop.
 6719 
 6720   if (CompressedKlassPointers::base() == nullptr &&
 6721       CompressedKlassPointers::shift() == 0) {
 6722     // The best case scenario is that there is no base or shift. Then it is already
 6723     // a pointer that needs nothing but a register rename.
 6724     movl(dst, src);
 6725   } else {
 6726     if (CompressedKlassPointers::shift() <= Address::times_8) {
 6727       if (CompressedKlassPointers::base() != nullptr) {
 6728         mov64(dst, (int64_t)CompressedKlassPointers::base());
 6729       } else {
 6730         xorq(dst, dst);
 6731       }
 6732       if (CompressedKlassPointers::shift() != 0) {
 6733         assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
 6734         leaq(dst, Address(dst, src, Address::times_8, 0));
 6735       } else {
 6736         addq(dst, src);
 6737       }
 6738     } else {
 6739       if (CompressedKlassPointers::base() != nullptr) {
 6740         const uint64_t base_right_shifted =
 6741             (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
 6742         mov64(dst, base_right_shifted);
 6743       } else {
 6744         xorq(dst, dst);
 6745       }
 6746       addq(dst, src);
 6747       shlq(dst, CompressedKlassPointers::shift());
 6748     }
 6749   }
 6750 }
 6751 
 6752 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 6753   assert (UseCompressedOops, "should only be used for compressed headers");
 6754   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6755   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6756   int oop_index = oop_recorder()->find_index(obj);
 6757   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6758   mov_narrow_oop(dst, oop_index, rspec);
 6759 }
 6760 
 6761 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 6762   assert (UseCompressedOops, "should only be used for compressed headers");
 6763   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6764   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6765   int oop_index = oop_recorder()->find_index(obj);
 6766   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6767   mov_narrow_oop(dst, oop_index, rspec);
 6768 }
 6769 
 6770 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 6771   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6772   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6773   int klass_index = oop_recorder()->find_index(k);
 6774   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6775   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6776 }
 6777 
 6778 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 6779   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6780   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6781   int klass_index = oop_recorder()->find_index(k);
 6782   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6783   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6784 }
 6785 
 6786 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6787   assert (UseCompressedOops, "should only be used for compressed headers");
 6788   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6789   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6790   int oop_index = oop_recorder()->find_index(obj);
 6791   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6792   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6793 }
 6794 
 6795 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6796   assert (UseCompressedOops, "should only be used for compressed headers");
 6797   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6798   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6799   int oop_index = oop_recorder()->find_index(obj);
 6800   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6801   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6802 }
 6803 
 6804 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6805   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6806   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6807   int klass_index = oop_recorder()->find_index(k);
 6808   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6809   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6810 }
 6811 
 6812 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6813   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6814   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6815   int klass_index = oop_recorder()->find_index(k);
 6816   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6817   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6818 }
 6819 
 6820 void MacroAssembler::reinit_heapbase() {
 6821   if (UseCompressedOops) {
 6822     if (Universe::heap() != nullptr) {
 6823       if (CompressedOops::base() == nullptr) {
 6824         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6825       } else {
 6826         mov64(r12_heapbase, (int64_t)CompressedOops::base());
 6827       }
 6828     } else {
 6829       movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr()));
 6830     }
 6831   }
 6832 }
 6833 
 6834 #endif // _LP64
 6835 
 6836 #if COMPILER2_OR_JVMCI
 6837 
 6838 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6839 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
 6840   // cnt - number of qwords (8-byte words).
 6841   // base - start address, qword aligned.
 6842   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6843   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6844   if (use64byteVector) {
 6845     evpbroadcastq(xtmp, val, AVX_512bit);
 6846   } else if (MaxVectorSize >= 32) {
 6847     movdq(xtmp, val);
 6848     punpcklqdq(xtmp, xtmp);
 6849     vinserti128_high(xtmp, xtmp);
 6850   } else {
 6851     movdq(xtmp, val);
 6852     punpcklqdq(xtmp, xtmp);
 6853   }
 6854   jmp(L_zero_64_bytes);
 6855 
 6856   BIND(L_loop);
 6857   if (MaxVectorSize >= 32) {
 6858     fill64(base, 0, xtmp, use64byteVector);
 6859   } else {
 6860     movdqu(Address(base,  0), xtmp);
 6861     movdqu(Address(base, 16), xtmp);
 6862     movdqu(Address(base, 32), xtmp);
 6863     movdqu(Address(base, 48), xtmp);
 6864   }
 6865   addptr(base, 64);
 6866 
 6867   BIND(L_zero_64_bytes);
 6868   subptr(cnt, 8);
 6869   jccb(Assembler::greaterEqual, L_loop);
 6870 
 6871   // Copy trailing 64 bytes
 6872   if (use64byteVector) {
 6873     addptr(cnt, 8);
 6874     jccb(Assembler::equal, L_end);
 6875     fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
 6876     jmp(L_end);
 6877   } else {
 6878     addptr(cnt, 4);
 6879     jccb(Assembler::less, L_tail);
 6880     if (MaxVectorSize >= 32) {
 6881       vmovdqu(Address(base, 0), xtmp);
 6882     } else {
 6883       movdqu(Address(base,  0), xtmp);
 6884       movdqu(Address(base, 16), xtmp);
 6885     }
 6886   }
 6887   addptr(base, 32);
 6888   subptr(cnt, 4);
 6889 
 6890   BIND(L_tail);
 6891   addptr(cnt, 4);
 6892   jccb(Assembler::lessEqual, L_end);
 6893   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6894     fill32_masked(3, base, 0, xtmp, mask, cnt, val);
 6895   } else {
 6896     decrement(cnt);
 6897 
 6898     BIND(L_sloop);
 6899     movq(Address(base, 0), xtmp);
 6900     addptr(base, 8);
 6901     decrement(cnt);
 6902     jccb(Assembler::greaterEqual, L_sloop);
 6903   }
 6904   BIND(L_end);
 6905 }
 6906 
 6907 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
 6908   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
 6909   // An inline type might be returned. If fields are in registers we
 6910   // need to allocate an inline type instance and initialize it with
 6911   // the value of the fields.
 6912   Label skip;
 6913   // We only need a new buffered inline type if a new one is not returned
 6914   testptr(rax, 1);
 6915   jcc(Assembler::zero, skip);
 6916   int call_offset = -1;
 6917 
 6918 #ifdef _LP64
 6919   // The following code is similar to allocate_instance but has some slight differences,
 6920   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
 6921   // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these.
 6922   Label slow_case;
 6923   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
 6924   mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
 6925   if (vk != nullptr) {
 6926     // Called from C1, where the return type is statically known.
 6927     movptr(rbx, (intptr_t)vk->get_InlineKlass());
 6928     jint lh = vk->layout_helper();
 6929     assert(lh != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
 6930     if (UseTLAB && !Klass::layout_helper_needs_slow_path(lh)) {
 6931       tlab_allocate(r15_thread, rax, noreg, lh, r13, r14, slow_case);
 6932     } else {
 6933       jmp(slow_case);
 6934     }
 6935   } else {
 6936     // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
 6937     mov(rbx, rax);
 6938     andptr(rbx, -2);
 6939     if (UseTLAB) {
 6940       movl(r14, Address(rbx, Klass::layout_helper_offset()));
 6941       testl(r14, Klass::_lh_instance_slow_path_bit);
 6942       jcc(Assembler::notZero, slow_case);
 6943       tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
 6944     } else {
 6945       jmp(slow_case);
 6946     }
 6947   }
 6948   if (UseTLAB) {
 6949     // 2. Initialize buffered inline instance header
 6950     Register buffer_obj = rax;
 6951     if (UseCompactObjectHeaders) {
 6952       Register mark_word = r13;
 6953       movptr(mark_word, Address(rbx, Klass::prototype_header_offset()));
 6954       movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 6955     } else {
 6956       movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
 6957       xorl(r13, r13);
 6958       store_klass_gap(buffer_obj, r13);
 6959       if (vk == nullptr) {
 6960         // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
 6961         mov(r13, rbx);
 6962       }
 6963       store_klass(buffer_obj, rbx, rscratch1);
 6964     }
 6965     // 3. Initialize its fields with an inline class specific handler
 6966     if (vk != nullptr) {
 6967       call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
 6968     } else {
 6969       movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6970       movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
 6971       call(rbx);
 6972     }
 6973     jmp(skip);
 6974   }
 6975   bind(slow_case);
 6976   // We failed to allocate a new inline type, fall back to a runtime
 6977   // call. Some oop field may be live in some registers but we can't
 6978   // tell. That runtime call will take care of preserving them
 6979   // across a GC if there's one.
 6980   mov(rax, rscratch1);
 6981 #endif
 6982 
 6983   if (from_interpreter) {
 6984     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
 6985   } else {
 6986     call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
 6987     call_offset = offset();
 6988   }
 6989 
 6990   bind(skip);
 6991   return call_offset;
 6992 }
 6993 
 6994 // Move a value between registers/stack slots and update the reg_state
 6995 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
 6996   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
 6997   if (reg_state[to->value()] == reg_written) {
 6998     return true; // Already written
 6999   }
 7000   if (from != to && bt != T_VOID) {
 7001     if (reg_state[to->value()] == reg_readonly) {
 7002       return false; // Not yet writable
 7003     }
 7004     if (from->is_reg()) {
 7005       if (to->is_reg()) {
 7006         if (from->is_XMMRegister()) {
 7007           if (bt == T_DOUBLE) {
 7008             movdbl(to->as_XMMRegister(), from->as_XMMRegister());
 7009           } else {
 7010             assert(bt == T_FLOAT, "must be float");
 7011             movflt(to->as_XMMRegister(), from->as_XMMRegister());
 7012           }
 7013         } else {
 7014           movq(to->as_Register(), from->as_Register());
 7015         }
 7016       } else {
 7017         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7018         Address to_addr = Address(rsp, st_off);
 7019         if (from->is_XMMRegister()) {
 7020           if (bt == T_DOUBLE) {
 7021             movdbl(to_addr, from->as_XMMRegister());
 7022           } else {
 7023             assert(bt == T_FLOAT, "must be float");
 7024             movflt(to_addr, from->as_XMMRegister());
 7025           }
 7026         } else {
 7027           movq(to_addr, from->as_Register());
 7028         }
 7029       }
 7030     } else {
 7031       Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
 7032       if (to->is_reg()) {
 7033         if (to->is_XMMRegister()) {
 7034           if (bt == T_DOUBLE) {
 7035             movdbl(to->as_XMMRegister(), from_addr);
 7036           } else {
 7037             assert(bt == T_FLOAT, "must be float");
 7038             movflt(to->as_XMMRegister(), from_addr);
 7039           }
 7040         } else {
 7041           movq(to->as_Register(), from_addr);
 7042         }
 7043       } else {
 7044         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7045         movq(r13, from_addr);
 7046         movq(Address(rsp, st_off), r13);
 7047       }
 7048     }
 7049   }
 7050   // Update register states
 7051   reg_state[from->value()] = reg_writable;
 7052   reg_state[to->value()] = reg_written;
 7053   return true;
 7054 }
 7055 
 7056 // Calculate the extra stack space required for packing or unpacking inline
 7057 // args and adjust the stack pointer
 7058 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
 7059   // Two additional slots to account for return address
 7060   int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
 7061   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
 7062   // Save the return address, adjust the stack (make sure it is properly
 7063   // 16-byte aligned) and copy the return address to the new top of the stack.
 7064   // The stack will be repaired on return (see MacroAssembler::remove_frame).
 7065   assert(sp_inc > 0, "sanity");
 7066   pop(r13);
 7067   subptr(rsp, sp_inc);
 7068   push(r13);
 7069   return sp_inc;
 7070 }
 7071 
 7072 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
 7073 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
 7074                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
 7075                                           RegState reg_state[]) {
 7076   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
 7077   assert(from->is_valid(), "source must be valid");
 7078   bool progress = false;
 7079 #ifdef ASSERT
 7080   const int start_offset = offset();
 7081 #endif
 7082 
 7083   Label L_null, L_notNull;
 7084   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
 7085   Register tmp1 = r10;
 7086   Register tmp2 = r13;
 7087   Register fromReg = noreg;
 7088   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
 7089   bool done = true;
 7090   bool mark_done = true;
 7091   VMReg toReg;
 7092   BasicType bt;
 7093   // Check if argument requires a null check
 7094   bool null_check = false;
 7095   VMReg nullCheckReg;
 7096   while (stream.next(nullCheckReg, bt)) {
 7097     if (sig->at(stream.sig_index())._offset == -1) {
 7098       null_check = true;
 7099       break;
 7100     }
 7101   }
 7102   stream.reset(sig_index, to_index);
 7103   while (stream.next(toReg, bt)) {
 7104     assert(toReg->is_valid(), "destination must be valid");
 7105     int idx = (int)toReg->value();
 7106     if (reg_state[idx] == reg_readonly) {
 7107       if (idx != from->value()) {
 7108         mark_done = false;
 7109       }
 7110       done = false;
 7111       continue;
 7112     } else if (reg_state[idx] == reg_written) {
 7113       continue;
 7114     }
 7115     assert(reg_state[idx] == reg_writable, "must be writable");
 7116     reg_state[idx] = reg_written;
 7117     progress = true;
 7118 
 7119     if (fromReg == noreg) {
 7120       if (from->is_reg()) {
 7121         fromReg = from->as_Register();
 7122       } else {
 7123         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7124         movq(tmp1, Address(rsp, st_off));
 7125         fromReg = tmp1;
 7126       }
 7127       if (null_check) {
 7128         // Nullable inline type argument, emit null check
 7129         testptr(fromReg, fromReg);
 7130         jcc(Assembler::zero, L_null);
 7131       }
 7132     }
 7133     int off = sig->at(stream.sig_index())._offset;
 7134     if (off == -1) {
 7135       assert(null_check, "Missing null check at");
 7136       if (toReg->is_stack()) {
 7137         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7138         movq(Address(rsp, st_off), 1);
 7139       } else {
 7140         movq(toReg->as_Register(), 1);
 7141       }
 7142       continue;
 7143     }
 7144     assert(off > 0, "offset in object should be positive");
 7145     Address fromAddr = Address(fromReg, off);
 7146     if (!toReg->is_XMMRegister()) {
 7147       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
 7148       if (is_reference_type(bt)) {
 7149         load_heap_oop(dst, fromAddr);
 7150       } else {
 7151         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
 7152         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
 7153       }
 7154       if (toReg->is_stack()) {
 7155         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7156         movq(Address(rsp, st_off), dst);
 7157       }
 7158     } else if (bt == T_DOUBLE) {
 7159       movdbl(toReg->as_XMMRegister(), fromAddr);
 7160     } else {
 7161       assert(bt == T_FLOAT, "must be float");
 7162       movflt(toReg->as_XMMRegister(), fromAddr);
 7163     }
 7164   }
 7165   if (progress && null_check) {
 7166     if (done) {
 7167       jmp(L_notNull);
 7168       bind(L_null);
 7169       // Set IsInit field to zero to signal that the argument is null.
 7170       // Also set all oop fields to zero to make the GC happy.
 7171       stream.reset(sig_index, to_index);
 7172       while (stream.next(toReg, bt)) {
 7173         if (sig->at(stream.sig_index())._offset == -1 ||
 7174             bt == T_OBJECT || bt == T_ARRAY) {
 7175           if (toReg->is_stack()) {
 7176             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7177             movq(Address(rsp, st_off), 0);
 7178           } else {
 7179             xorq(toReg->as_Register(), toReg->as_Register());
 7180           }
 7181         }
 7182       }
 7183       bind(L_notNull);
 7184     } else {
 7185       bind(L_null);
 7186     }
 7187   }
 7188 
 7189   sig_index = stream.sig_index();
 7190   to_index = stream.regs_index();
 7191 
 7192   if (mark_done && reg_state[from->value()] != reg_written) {
 7193     // This is okay because no one else will write to that slot
 7194     reg_state[from->value()] = reg_writable;
 7195   }
 7196   from_index--;
 7197   assert(progress || (start_offset == offset()), "should not emit code");
 7198   return done;
 7199 }
 7200 
 7201 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
 7202                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
 7203                                         RegState reg_state[], Register val_array) {
 7204   assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter");
 7205   assert(to->is_valid(), "destination must be valid");
 7206 
 7207   if (reg_state[to->value()] == reg_written) {
 7208     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 7209     return true; // Already written
 7210   }
 7211 
 7212   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
 7213   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
 7214   Register val_obj_tmp = r11;
 7215   Register from_reg_tmp = r14;
 7216   Register tmp1 = r10;
 7217   Register tmp2 = r13;
 7218   Register tmp3 = rbx;
 7219   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
 7220 
 7221   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
 7222 
 7223   if (reg_state[to->value()] == reg_readonly) {
 7224     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
 7225       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 7226       return false; // Not yet writable
 7227     }
 7228     val_obj = val_obj_tmp;
 7229   }
 7230 
 7231   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT);
 7232   load_heap_oop(val_obj, Address(val_array, index));
 7233 
 7234   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
 7235   VMReg fromReg;
 7236   BasicType bt;
 7237   Label L_null;
 7238   while (stream.next(fromReg, bt)) {
 7239     assert(fromReg->is_valid(), "source must be valid");
 7240     reg_state[fromReg->value()] = reg_writable;
 7241 
 7242     int off = sig->at(stream.sig_index())._offset;
 7243     if (off == -1) {
 7244       // Nullable inline type argument, emit null check
 7245       Label L_notNull;
 7246       if (fromReg->is_stack()) {
 7247         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7248         testb(Address(rsp, ld_off), 1);
 7249       } else {
 7250         testb(fromReg->as_Register(), 1);
 7251       }
 7252       jcc(Assembler::notZero, L_notNull);
 7253       movptr(val_obj, 0);
 7254       jmp(L_null);
 7255       bind(L_notNull);
 7256       continue;
 7257     }
 7258 
 7259     assert(off > 0, "offset in object should be positive");
 7260     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 7261 
 7262     Address dst(val_obj, off);
 7263     if (!fromReg->is_XMMRegister()) {
 7264       Register src;
 7265       if (fromReg->is_stack()) {
 7266         src = from_reg_tmp;
 7267         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 7268         load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 7269       } else {
 7270         src = fromReg->as_Register();
 7271       }
 7272       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
 7273       if (is_reference_type(bt)) {
 7274         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 7275       } else {
 7276         store_sized_value(dst, src, size_in_bytes);
 7277       }
 7278     } else if (bt == T_DOUBLE) {
 7279       movdbl(dst, fromReg->as_XMMRegister());
 7280     } else {
 7281       assert(bt == T_FLOAT, "must be float");
 7282       movflt(dst, fromReg->as_XMMRegister());
 7283     }
 7284   }
 7285   bind(L_null);
 7286   sig_index = stream.sig_index();
 7287   from_index = stream.regs_index();
 7288 
 7289   assert(reg_state[to->value()] == reg_writable, "must have already been read");
 7290   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
 7291   assert(success, "to register must be writeable");
 7292   return true;
 7293 }
 7294 
 7295 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
 7296   return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
 7297 }
 7298 
 7299 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
 7300   assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 7301   if (needs_stack_repair) {
 7302     movq(rbp, Address(rsp, initial_framesize));
 7303     // The stack increment resides just below the saved rbp
 7304     addq(rsp, Address(rsp, initial_framesize - wordSize));
 7305   } else {
 7306     if (initial_framesize > 0) {
 7307       addq(rsp, initial_framesize);
 7308     }
 7309     pop(rbp);
 7310   }
 7311 }
 7312 
 7313 // Clearing constant sized memory using YMM/ZMM registers.
 7314 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 7315   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 7316   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 7317 
 7318   int vector64_count = (cnt & (~0x7)) >> 3;
 7319   cnt = cnt & 0x7;
 7320   const int fill64_per_loop = 4;
 7321   const int max_unrolled_fill64 = 8;
 7322 
 7323   // 64 byte initialization loop.
 7324   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 7325   int start64 = 0;
 7326   if (vector64_count > max_unrolled_fill64) {
 7327     Label LOOP;
 7328     Register index = rtmp;
 7329 
 7330     start64 = vector64_count - (vector64_count % fill64_per_loop);
 7331 
 7332     movl(index, 0);
 7333     BIND(LOOP);
 7334     for (int i = 0; i < fill64_per_loop; i++) {
 7335       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 7336     }
 7337     addl(index, fill64_per_loop * 64);
 7338     cmpl(index, start64 * 64);
 7339     jccb(Assembler::less, LOOP);
 7340   }
 7341   for (int i = start64; i < vector64_count; i++) {
 7342     fill64(base, i * 64, xtmp, use64byteVector);
 7343   }
 7344 
 7345   // Clear remaining 64 byte tail.
 7346   int disp = vector64_count * 64;
 7347   if (cnt) {
 7348     switch (cnt) {
 7349       case 1:
 7350         movq(Address(base, disp), xtmp);
 7351         break;
 7352       case 2:
 7353         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 7354         break;
 7355       case 3:
 7356         movl(rtmp, 0x7);
 7357         kmovwl(mask, rtmp);
 7358         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 7359         break;
 7360       case 4:
 7361         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 7362         break;
 7363       case 5:
 7364         if (use64byteVector) {
 7365           movl(rtmp, 0x1F);
 7366           kmovwl(mask, rtmp);
 7367           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 7368         } else {
 7369           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 7370           movq(Address(base, disp + 32), xtmp);
 7371         }
 7372         break;
 7373       case 6:
 7374         if (use64byteVector) {
 7375           movl(rtmp, 0x3F);
 7376           kmovwl(mask, rtmp);
 7377           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 7378         } else {
 7379           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 7380           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 7381         }
 7382         break;
 7383       case 7:
 7384         if (use64byteVector) {
 7385           movl(rtmp, 0x7F);
 7386           kmovwl(mask, rtmp);
 7387           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 7388         } else {
 7389           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 7390           movl(rtmp, 0x7);
 7391           kmovwl(mask, rtmp);
 7392           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 7393         }
 7394         break;
 7395       default:
 7396         fatal("Unexpected length : %d\n",cnt);
 7397         break;
 7398     }
 7399   }
 7400 }
 7401 
 7402 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
 7403                                bool is_large, bool word_copy_only, KRegister mask) {
 7404   // cnt      - number of qwords (8-byte words).
 7405   // base     - start address, qword aligned.
 7406   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 7407   assert(base==rdi, "base register must be edi for rep stos");
 7408   assert(val==rax,   "val register must be eax for rep stos");
 7409   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 7410   assert(InitArrayShortSize % BytesPerLong == 0,
 7411     "InitArrayShortSize should be the multiple of BytesPerLong");
 7412 
 7413   Label DONE;
 7414 
 7415   if (!is_large) {
 7416     Label LOOP, LONG;
 7417     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 7418     jccb(Assembler::greater, LONG);
 7419 
 7420     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 7421 
 7422     decrement(cnt);
 7423     jccb(Assembler::negative, DONE); // Zero length
 7424 
 7425     // Use individual pointer-sized stores for small counts:
 7426     BIND(LOOP);
 7427     movptr(Address(base, cnt, Address::times_ptr), val);
 7428     decrement(cnt);
 7429     jccb(Assembler::greaterEqual, LOOP);
 7430     jmpb(DONE);
 7431 
 7432     BIND(LONG);
 7433   }
 7434 
 7435   // Use longer rep-prefixed ops for non-small counts:
 7436   if (UseFastStosb && !word_copy_only) {
 7437     shlptr(cnt, 3); // convert to number of bytes
 7438     rep_stosb();
 7439   } else if (UseXMMForObjInit) {
 7440     xmm_clear_mem(base, cnt, val, xtmp, mask);
 7441   } else {
 7442     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 7443     rep_stos();
 7444   }
 7445 
 7446   BIND(DONE);
 7447 }
 7448 
 7449 #endif //COMPILER2_OR_JVMCI
 7450 
 7451 
 7452 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 7453                                    Register to, Register value, Register count,
 7454                                    Register rtmp, XMMRegister xtmp) {
 7455   ShortBranchVerifier sbv(this);
 7456   assert_different_registers(to, value, count, rtmp);
 7457   Label L_exit;
 7458   Label L_fill_2_bytes, L_fill_4_bytes;
 7459 
 7460 #if defined(COMPILER2) && defined(_LP64)
 7461   if(MaxVectorSize >=32 &&
 7462      VM_Version::supports_avx512vlbw() &&
 7463      VM_Version::supports_bmi2()) {
 7464     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 7465     return;
 7466   }
 7467 #endif
 7468 
 7469   int shift = -1;
 7470   switch (t) {
 7471     case T_BYTE:
 7472       shift = 2;
 7473       break;
 7474     case T_SHORT:
 7475       shift = 1;
 7476       break;
 7477     case T_INT:
 7478       shift = 0;
 7479       break;
 7480     default: ShouldNotReachHere();
 7481   }
 7482 
 7483   if (t == T_BYTE) {
 7484     andl(value, 0xff);
 7485     movl(rtmp, value);
 7486     shll(rtmp, 8);
 7487     orl(value, rtmp);
 7488   }
 7489   if (t == T_SHORT) {
 7490     andl(value, 0xffff);
 7491   }
 7492   if (t == T_BYTE || t == T_SHORT) {
 7493     movl(rtmp, value);
 7494     shll(rtmp, 16);
 7495     orl(value, rtmp);
 7496   }
 7497 
 7498   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 7499   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 7500   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 7501     Label L_skip_align2;
 7502     // align source address at 4 bytes address boundary
 7503     if (t == T_BYTE) {
 7504       Label L_skip_align1;
 7505       // One byte misalignment happens only for byte arrays
 7506       testptr(to, 1);
 7507       jccb(Assembler::zero, L_skip_align1);
 7508       movb(Address(to, 0), value);
 7509       increment(to);
 7510       decrement(count);
 7511       BIND(L_skip_align1);
 7512     }
 7513     // Two bytes misalignment happens only for byte and short (char) arrays
 7514     testptr(to, 2);
 7515     jccb(Assembler::zero, L_skip_align2);
 7516     movw(Address(to, 0), value);
 7517     addptr(to, 2);
 7518     subptr(count, 1<<(shift-1));
 7519     BIND(L_skip_align2);
 7520   }
 7521   if (UseSSE < 2) {
 7522     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 7523     // Fill 32-byte chunks
 7524     subptr(count, 8 << shift);
 7525     jcc(Assembler::less, L_check_fill_8_bytes);
 7526     align(16);
 7527 
 7528     BIND(L_fill_32_bytes_loop);
 7529 
 7530     for (int i = 0; i < 32; i += 4) {
 7531       movl(Address(to, i), value);
 7532     }
 7533 
 7534     addptr(to, 32);
 7535     subptr(count, 8 << shift);
 7536     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 7537     BIND(L_check_fill_8_bytes);
 7538     addptr(count, 8 << shift);
 7539     jccb(Assembler::zero, L_exit);
 7540     jmpb(L_fill_8_bytes);
 7541 
 7542     //
 7543     // length is too short, just fill qwords
 7544     //
 7545     BIND(L_fill_8_bytes_loop);
 7546     movl(Address(to, 0), value);
 7547     movl(Address(to, 4), value);
 7548     addptr(to, 8);
 7549     BIND(L_fill_8_bytes);
 7550     subptr(count, 1 << (shift + 1));
 7551     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 7552     // fall through to fill 4 bytes
 7553   } else {
 7554     Label L_fill_32_bytes;
 7555     if (!UseUnalignedLoadStores) {
 7556       // align to 8 bytes, we know we are 4 byte aligned to start
 7557       testptr(to, 4);
 7558       jccb(Assembler::zero, L_fill_32_bytes);
 7559       movl(Address(to, 0), value);
 7560       addptr(to, 4);
 7561       subptr(count, 1<<shift);
 7562     }
 7563     BIND(L_fill_32_bytes);
 7564     {
 7565       assert( UseSSE >= 2, "supported cpu only" );
 7566       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 7567       movdl(xtmp, value);
 7568       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 7569         Label L_check_fill_32_bytes;
 7570         if (UseAVX > 2) {
 7571           // Fill 64-byte chunks
 7572           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 7573 
 7574           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 7575           cmpptr(count, VM_Version::avx3_threshold());
 7576           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 7577 
 7578           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 7579 
 7580           subptr(count, 16 << shift);
 7581           jccb(Assembler::less, L_check_fill_32_bytes);
 7582           align(16);
 7583 
 7584           BIND(L_fill_64_bytes_loop_avx3);
 7585           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 7586           addptr(to, 64);
 7587           subptr(count, 16 << shift);
 7588           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 7589           jmpb(L_check_fill_32_bytes);
 7590 
 7591           BIND(L_check_fill_64_bytes_avx2);
 7592         }
 7593         // Fill 64-byte chunks
 7594         Label L_fill_64_bytes_loop;
 7595         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 7596 
 7597         subptr(count, 16 << shift);
 7598         jcc(Assembler::less, L_check_fill_32_bytes);
 7599         align(16);
 7600 
 7601         BIND(L_fill_64_bytes_loop);
 7602         vmovdqu(Address(to, 0), xtmp);
 7603         vmovdqu(Address(to, 32), xtmp);
 7604         addptr(to, 64);
 7605         subptr(count, 16 << shift);
 7606         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 7607 
 7608         BIND(L_check_fill_32_bytes);
 7609         addptr(count, 8 << shift);
 7610         jccb(Assembler::less, L_check_fill_8_bytes);
 7611         vmovdqu(Address(to, 0), xtmp);
 7612         addptr(to, 32);
 7613         subptr(count, 8 << shift);
 7614 
 7615         BIND(L_check_fill_8_bytes);
 7616         // clean upper bits of YMM registers
 7617         movdl(xtmp, value);
 7618         pshufd(xtmp, xtmp, 0);
 7619       } else {
 7620         // Fill 32-byte chunks
 7621         pshufd(xtmp, xtmp, 0);
 7622 
 7623         subptr(count, 8 << shift);
 7624         jcc(Assembler::less, L_check_fill_8_bytes);
 7625         align(16);
 7626 
 7627         BIND(L_fill_32_bytes_loop);
 7628 
 7629         if (UseUnalignedLoadStores) {
 7630           movdqu(Address(to, 0), xtmp);
 7631           movdqu(Address(to, 16), xtmp);
 7632         } else {
 7633           movq(Address(to, 0), xtmp);
 7634           movq(Address(to, 8), xtmp);
 7635           movq(Address(to, 16), xtmp);
 7636           movq(Address(to, 24), xtmp);
 7637         }
 7638 
 7639         addptr(to, 32);
 7640         subptr(count, 8 << shift);
 7641         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 7642 
 7643         BIND(L_check_fill_8_bytes);
 7644       }
 7645       addptr(count, 8 << shift);
 7646       jccb(Assembler::zero, L_exit);
 7647       jmpb(L_fill_8_bytes);
 7648 
 7649       //
 7650       // length is too short, just fill qwords
 7651       //
 7652       BIND(L_fill_8_bytes_loop);
 7653       movq(Address(to, 0), xtmp);
 7654       addptr(to, 8);
 7655       BIND(L_fill_8_bytes);
 7656       subptr(count, 1 << (shift + 1));
 7657       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 7658     }
 7659   }
 7660   // fill trailing 4 bytes
 7661   BIND(L_fill_4_bytes);
 7662   testl(count, 1<<shift);
 7663   jccb(Assembler::zero, L_fill_2_bytes);
 7664   movl(Address(to, 0), value);
 7665   if (t == T_BYTE || t == T_SHORT) {
 7666     Label L_fill_byte;
 7667     addptr(to, 4);
 7668     BIND(L_fill_2_bytes);
 7669     // fill trailing 2 bytes
 7670     testl(count, 1<<(shift-1));
 7671     jccb(Assembler::zero, L_fill_byte);
 7672     movw(Address(to, 0), value);
 7673     if (t == T_BYTE) {
 7674       addptr(to, 2);
 7675       BIND(L_fill_byte);
 7676       // fill trailing byte
 7677       testl(count, 1);
 7678       jccb(Assembler::zero, L_exit);
 7679       movb(Address(to, 0), value);
 7680     } else {
 7681       BIND(L_fill_byte);
 7682     }
 7683   } else {
 7684     BIND(L_fill_2_bytes);
 7685   }
 7686   BIND(L_exit);
 7687 }
 7688 
 7689 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 7690   switch(type) {
 7691     case T_BYTE:
 7692     case T_BOOLEAN:
 7693       evpbroadcastb(dst, src, vector_len);
 7694       break;
 7695     case T_SHORT:
 7696     case T_CHAR:
 7697       evpbroadcastw(dst, src, vector_len);
 7698       break;
 7699     case T_INT:
 7700     case T_FLOAT:
 7701       evpbroadcastd(dst, src, vector_len);
 7702       break;
 7703     case T_LONG:
 7704     case T_DOUBLE:
 7705       evpbroadcastq(dst, src, vector_len);
 7706       break;
 7707     default:
 7708       fatal("Unhandled type : %s", type2name(type));
 7709       break;
 7710   }
 7711 }
 7712 
 7713 // encode char[] to byte[] in ISO_8859_1 or ASCII
 7714    //@IntrinsicCandidate
 7715    //private static int implEncodeISOArray(byte[] sa, int sp,
 7716    //byte[] da, int dp, int len) {
 7717    //  int i = 0;
 7718    //  for (; i < len; i++) {
 7719    //    char c = StringUTF16.getChar(sa, sp++);
 7720    //    if (c > '\u00FF')
 7721    //      break;
 7722    //    da[dp++] = (byte)c;
 7723    //  }
 7724    //  return i;
 7725    //}
 7726    //
 7727    //@IntrinsicCandidate
 7728    //private static int implEncodeAsciiArray(char[] sa, int sp,
 7729    //    byte[] da, int dp, int len) {
 7730    //  int i = 0;
 7731    //  for (; i < len; i++) {
 7732    //    char c = sa[sp++];
 7733    //    if (c >= '\u0080')
 7734    //      break;
 7735    //    da[dp++] = (byte)c;
 7736    //  }
 7737    //  return i;
 7738    //}
 7739 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 7740   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 7741   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 7742   Register tmp5, Register result, bool ascii) {
 7743 
 7744   // rsi: src
 7745   // rdi: dst
 7746   // rdx: len
 7747   // rcx: tmp5
 7748   // rax: result
 7749   ShortBranchVerifier sbv(this);
 7750   assert_different_registers(src, dst, len, tmp5, result);
 7751   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 7752 
 7753   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 7754   int short_mask = ascii ? 0xff80 : 0xff00;
 7755 
 7756   // set result
 7757   xorl(result, result);
 7758   // check for zero length
 7759   testl(len, len);
 7760   jcc(Assembler::zero, L_done);
 7761 
 7762   movl(result, len);
 7763 
 7764   // Setup pointers
 7765   lea(src, Address(src, len, Address::times_2)); // char[]
 7766   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 7767   negptr(len);
 7768 
 7769   if (UseSSE42Intrinsics || UseAVX >= 2) {
 7770     Label L_copy_8_chars, L_copy_8_chars_exit;
 7771     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 7772 
 7773     if (UseAVX >= 2) {
 7774       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 7775       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 7776       movdl(tmp1Reg, tmp5);
 7777       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 7778       jmp(L_chars_32_check);
 7779 
 7780       bind(L_copy_32_chars);
 7781       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 7782       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 7783       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7784       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7785       jccb(Assembler::notZero, L_copy_32_chars_exit);
 7786       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7787       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 7788       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 7789 
 7790       bind(L_chars_32_check);
 7791       addptr(len, 32);
 7792       jcc(Assembler::lessEqual, L_copy_32_chars);
 7793 
 7794       bind(L_copy_32_chars_exit);
 7795       subptr(len, 16);
 7796       jccb(Assembler::greater, L_copy_16_chars_exit);
 7797 
 7798     } else if (UseSSE42Intrinsics) {
 7799       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 7800       movdl(tmp1Reg, tmp5);
 7801       pshufd(tmp1Reg, tmp1Reg, 0);
 7802       jmpb(L_chars_16_check);
 7803     }
 7804 
 7805     bind(L_copy_16_chars);
 7806     if (UseAVX >= 2) {
 7807       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 7808       vptest(tmp2Reg, tmp1Reg);
 7809       jcc(Assembler::notZero, L_copy_16_chars_exit);
 7810       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 7811       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 7812     } else {
 7813       if (UseAVX > 0) {
 7814         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7815         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7816         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 7817       } else {
 7818         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7819         por(tmp2Reg, tmp3Reg);
 7820         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7821         por(tmp2Reg, tmp4Reg);
 7822       }
 7823       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7824       jccb(Assembler::notZero, L_copy_16_chars_exit);
 7825       packuswb(tmp3Reg, tmp4Reg);
 7826     }
 7827     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 7828 
 7829     bind(L_chars_16_check);
 7830     addptr(len, 16);
 7831     jcc(Assembler::lessEqual, L_copy_16_chars);
 7832 
 7833     bind(L_copy_16_chars_exit);
 7834     if (UseAVX >= 2) {
 7835       // clean upper bits of YMM registers
 7836       vpxor(tmp2Reg, tmp2Reg);
 7837       vpxor(tmp3Reg, tmp3Reg);
 7838       vpxor(tmp4Reg, tmp4Reg);
 7839       movdl(tmp1Reg, tmp5);
 7840       pshufd(tmp1Reg, tmp1Reg, 0);
 7841     }
 7842     subptr(len, 8);
 7843     jccb(Assembler::greater, L_copy_8_chars_exit);
 7844 
 7845     bind(L_copy_8_chars);
 7846     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 7847     ptest(tmp3Reg, tmp1Reg);
 7848     jccb(Assembler::notZero, L_copy_8_chars_exit);
 7849     packuswb(tmp3Reg, tmp1Reg);
 7850     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 7851     addptr(len, 8);
 7852     jccb(Assembler::lessEqual, L_copy_8_chars);
 7853 
 7854     bind(L_copy_8_chars_exit);
 7855     subptr(len, 8);
 7856     jccb(Assembler::zero, L_done);
 7857   }
 7858 
 7859   bind(L_copy_1_char);
 7860   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 7861   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 7862   jccb(Assembler::notZero, L_copy_1_char_exit);
 7863   movb(Address(dst, len, Address::times_1, 0), tmp5);
 7864   addptr(len, 1);
 7865   jccb(Assembler::less, L_copy_1_char);
 7866 
 7867   bind(L_copy_1_char_exit);
 7868   addptr(result, len); // len is negative count of not processed elements
 7869 
 7870   bind(L_done);
 7871 }
 7872 
 7873 #ifdef _LP64
 7874 /**
 7875  * Helper for multiply_to_len().
 7876  */
 7877 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 7878   addq(dest_lo, src1);
 7879   adcq(dest_hi, 0);
 7880   addq(dest_lo, src2);
 7881   adcq(dest_hi, 0);
 7882 }
 7883 
 7884 /**
 7885  * Multiply 64 bit by 64 bit first loop.
 7886  */
 7887 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 7888                                            Register y, Register y_idx, Register z,
 7889                                            Register carry, Register product,
 7890                                            Register idx, Register kdx) {
 7891   //
 7892   //  jlong carry, x[], y[], z[];
 7893   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7894   //    huge_128 product = y[idx] * x[xstart] + carry;
 7895   //    z[kdx] = (jlong)product;
 7896   //    carry  = (jlong)(product >>> 64);
 7897   //  }
 7898   //  z[xstart] = carry;
 7899   //
 7900 
 7901   Label L_first_loop, L_first_loop_exit;
 7902   Label L_one_x, L_one_y, L_multiply;
 7903 
 7904   decrementl(xstart);
 7905   jcc(Assembler::negative, L_one_x);
 7906 
 7907   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7908   rorq(x_xstart, 32); // convert big-endian to little-endian
 7909 
 7910   bind(L_first_loop);
 7911   decrementl(idx);
 7912   jcc(Assembler::negative, L_first_loop_exit);
 7913   decrementl(idx);
 7914   jcc(Assembler::negative, L_one_y);
 7915   movq(y_idx, Address(y, idx, Address::times_4,  0));
 7916   rorq(y_idx, 32); // convert big-endian to little-endian
 7917   bind(L_multiply);
 7918   movq(product, x_xstart);
 7919   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 7920   addq(product, carry);
 7921   adcq(rdx, 0);
 7922   subl(kdx, 2);
 7923   movl(Address(z, kdx, Address::times_4,  4), product);
 7924   shrq(product, 32);
 7925   movl(Address(z, kdx, Address::times_4,  0), product);
 7926   movq(carry, rdx);
 7927   jmp(L_first_loop);
 7928 
 7929   bind(L_one_y);
 7930   movl(y_idx, Address(y,  0));
 7931   jmp(L_multiply);
 7932 
 7933   bind(L_one_x);
 7934   movl(x_xstart, Address(x,  0));
 7935   jmp(L_first_loop);
 7936 
 7937   bind(L_first_loop_exit);
 7938 }
 7939 
 7940 /**
 7941  * Multiply 64 bit by 64 bit and add 128 bit.
 7942  */
 7943 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 7944                                             Register yz_idx, Register idx,
 7945                                             Register carry, Register product, int offset) {
 7946   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 7947   //     z[kdx] = (jlong)product;
 7948 
 7949   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 7950   rorq(yz_idx, 32); // convert big-endian to little-endian
 7951   movq(product, x_xstart);
 7952   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 7953   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 7954   rorq(yz_idx, 32); // convert big-endian to little-endian
 7955 
 7956   add2_with_carry(rdx, product, carry, yz_idx);
 7957 
 7958   movl(Address(z, idx, Address::times_4,  offset+4), product);
 7959   shrq(product, 32);
 7960   movl(Address(z, idx, Address::times_4,  offset), product);
 7961 
 7962 }
 7963 
 7964 /**
 7965  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 7966  */
 7967 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 7968                                              Register yz_idx, Register idx, Register jdx,
 7969                                              Register carry, Register product,
 7970                                              Register carry2) {
 7971   //   jlong carry, x[], y[], z[];
 7972   //   int kdx = ystart+1;
 7973   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7974   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 7975   //     z[kdx+idx+1] = (jlong)product;
 7976   //     jlong carry2  = (jlong)(product >>> 64);
 7977   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 7978   //     z[kdx+idx] = (jlong)product;
 7979   //     carry  = (jlong)(product >>> 64);
 7980   //   }
 7981   //   idx += 2;
 7982   //   if (idx > 0) {
 7983   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 7984   //     z[kdx+idx] = (jlong)product;
 7985   //     carry  = (jlong)(product >>> 64);
 7986   //   }
 7987   //
 7988 
 7989   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7990 
 7991   movl(jdx, idx);
 7992   andl(jdx, 0xFFFFFFFC);
 7993   shrl(jdx, 2);
 7994 
 7995   bind(L_third_loop);
 7996   subl(jdx, 1);
 7997   jcc(Assembler::negative, L_third_loop_exit);
 7998   subl(idx, 4);
 7999 
 8000   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 8001   movq(carry2, rdx);
 8002 
 8003   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 8004   movq(carry, rdx);
 8005   jmp(L_third_loop);
 8006 
 8007   bind (L_third_loop_exit);
 8008 
 8009   andl (idx, 0x3);
 8010   jcc(Assembler::zero, L_post_third_loop_done);
 8011 
 8012   Label L_check_1;
 8013   subl(idx, 2);
 8014   jcc(Assembler::negative, L_check_1);
 8015 
 8016   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 8017   movq(carry, rdx);
 8018 
 8019   bind (L_check_1);
 8020   addl (idx, 0x2);
 8021   andl (idx, 0x1);
 8022   subl(idx, 1);
 8023   jcc(Assembler::negative, L_post_third_loop_done);
 8024 
 8025   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 8026   movq(product, x_xstart);
 8027   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 8028   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 8029 
 8030   add2_with_carry(rdx, product, yz_idx, carry);
 8031 
 8032   movl(Address(z, idx, Address::times_4,  0), product);
 8033   shrq(product, 32);
 8034 
 8035   shlq(rdx, 32);
 8036   orq(product, rdx);
 8037   movq(carry, product);
 8038 
 8039   bind(L_post_third_loop_done);
 8040 }
 8041 
 8042 /**
 8043  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 8044  *
 8045  */
 8046 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 8047                                                   Register carry, Register carry2,
 8048                                                   Register idx, Register jdx,
 8049                                                   Register yz_idx1, Register yz_idx2,
 8050                                                   Register tmp, Register tmp3, Register tmp4) {
 8051   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 8052 
 8053   //   jlong carry, x[], y[], z[];
 8054   //   int kdx = ystart+1;
 8055   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 8056   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 8057   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 8058   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 8059   //     carry  = (jlong)(tmp4 >>> 64);
 8060   //     z[kdx+idx+1] = (jlong)tmp3;
 8061   //     z[kdx+idx] = (jlong)tmp4;
 8062   //   }
 8063   //   idx += 2;
 8064   //   if (idx > 0) {
 8065   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 8066   //     z[kdx+idx] = (jlong)yz_idx1;
 8067   //     carry  = (jlong)(yz_idx1 >>> 64);
 8068   //   }
 8069   //
 8070 
 8071   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 8072 
 8073   movl(jdx, idx);
 8074   andl(jdx, 0xFFFFFFFC);
 8075   shrl(jdx, 2);
 8076 
 8077   bind(L_third_loop);
 8078   subl(jdx, 1);
 8079   jcc(Assembler::negative, L_third_loop_exit);
 8080   subl(idx, 4);
 8081 
 8082   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 8083   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 8084   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 8085   rorxq(yz_idx2, yz_idx2, 32);
 8086 
 8087   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 8088   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 8089 
 8090   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 8091   rorxq(yz_idx1, yz_idx1, 32);
 8092   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 8093   rorxq(yz_idx2, yz_idx2, 32);
 8094 
 8095   if (VM_Version::supports_adx()) {
 8096     adcxq(tmp3, carry);
 8097     adoxq(tmp3, yz_idx1);
 8098 
 8099     adcxq(tmp4, tmp);
 8100     adoxq(tmp4, yz_idx2);
 8101 
 8102     movl(carry, 0); // does not affect flags
 8103     adcxq(carry2, carry);
 8104     adoxq(carry2, carry);
 8105   } else {
 8106     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 8107     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 8108   }
 8109   movq(carry, carry2);
 8110 
 8111   movl(Address(z, idx, Address::times_4, 12), tmp3);
 8112   shrq(tmp3, 32);
 8113   movl(Address(z, idx, Address::times_4,  8), tmp3);
 8114 
 8115   movl(Address(z, idx, Address::times_4,  4), tmp4);
 8116   shrq(tmp4, 32);
 8117   movl(Address(z, idx, Address::times_4,  0), tmp4);
 8118 
 8119   jmp(L_third_loop);
 8120 
 8121   bind (L_third_loop_exit);
 8122 
 8123   andl (idx, 0x3);
 8124   jcc(Assembler::zero, L_post_third_loop_done);
 8125 
 8126   Label L_check_1;
 8127   subl(idx, 2);
 8128   jcc(Assembler::negative, L_check_1);
 8129 
 8130   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 8131   rorxq(yz_idx1, yz_idx1, 32);
 8132   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 8133   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 8134   rorxq(yz_idx2, yz_idx2, 32);
 8135 
 8136   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 8137 
 8138   movl(Address(z, idx, Address::times_4,  4), tmp3);
 8139   shrq(tmp3, 32);
 8140   movl(Address(z, idx, Address::times_4,  0), tmp3);
 8141   movq(carry, tmp4);
 8142 
 8143   bind (L_check_1);
 8144   addl (idx, 0x2);
 8145   andl (idx, 0x1);
 8146   subl(idx, 1);
 8147   jcc(Assembler::negative, L_post_third_loop_done);
 8148   movl(tmp4, Address(y, idx, Address::times_4,  0));
 8149   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 8150   movl(tmp4, Address(z, idx, Address::times_4,  0));
 8151 
 8152   add2_with_carry(carry2, tmp3, tmp4, carry);
 8153 
 8154   movl(Address(z, idx, Address::times_4,  0), tmp3);
 8155   shrq(tmp3, 32);
 8156 
 8157   shlq(carry2, 32);
 8158   orq(tmp3, carry2);
 8159   movq(carry, tmp3);
 8160 
 8161   bind(L_post_third_loop_done);
 8162 }
 8163 
 8164 /**
 8165  * Code for BigInteger::multiplyToLen() intrinsic.
 8166  *
 8167  * rdi: x
 8168  * rax: xlen
 8169  * rsi: y
 8170  * rcx: ylen
 8171  * r8:  z
 8172  * r11: tmp0
 8173  * r12: tmp1
 8174  * r13: tmp2
 8175  * r14: tmp3
 8176  * r15: tmp4
 8177  * rbx: tmp5
 8178  *
 8179  */
 8180 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
 8181                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 8182   ShortBranchVerifier sbv(this);
 8183   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 8184 
 8185   push(tmp0);
 8186   push(tmp1);
 8187   push(tmp2);
 8188   push(tmp3);
 8189   push(tmp4);
 8190   push(tmp5);
 8191 
 8192   push(xlen);
 8193 
 8194   const Register idx = tmp1;
 8195   const Register kdx = tmp2;
 8196   const Register xstart = tmp3;
 8197 
 8198   const Register y_idx = tmp4;
 8199   const Register carry = tmp5;
 8200   const Register product  = xlen;
 8201   const Register x_xstart = tmp0;
 8202 
 8203   // First Loop.
 8204   //
 8205   //  final static long LONG_MASK = 0xffffffffL;
 8206   //  int xstart = xlen - 1;
 8207   //  int ystart = ylen - 1;
 8208   //  long carry = 0;
 8209   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 8210   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 8211   //    z[kdx] = (int)product;
 8212   //    carry = product >>> 32;
 8213   //  }
 8214   //  z[xstart] = (int)carry;
 8215   //
 8216 
 8217   movl(idx, ylen);               // idx = ylen;
 8218   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
 8219   xorq(carry, carry);            // carry = 0;
 8220 
 8221   Label L_done;
 8222 
 8223   movl(xstart, xlen);
 8224   decrementl(xstart);
 8225   jcc(Assembler::negative, L_done);
 8226 
 8227   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 8228 
 8229   Label L_second_loop;
 8230   testl(kdx, kdx);
 8231   jcc(Assembler::zero, L_second_loop);
 8232 
 8233   Label L_carry;
 8234   subl(kdx, 1);
 8235   jcc(Assembler::zero, L_carry);
 8236 
 8237   movl(Address(z, kdx, Address::times_4,  0), carry);
 8238   shrq(carry, 32);
 8239   subl(kdx, 1);
 8240 
 8241   bind(L_carry);
 8242   movl(Address(z, kdx, Address::times_4,  0), carry);
 8243 
 8244   // Second and third (nested) loops.
 8245   //
 8246   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 8247   //   carry = 0;
 8248   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 8249   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 8250   //                    (z[k] & LONG_MASK) + carry;
 8251   //     z[k] = (int)product;
 8252   //     carry = product >>> 32;
 8253   //   }
 8254   //   z[i] = (int)carry;
 8255   // }
 8256   //
 8257   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 8258 
 8259   const Register jdx = tmp1;
 8260 
 8261   bind(L_second_loop);
 8262   xorl(carry, carry);    // carry = 0;
 8263   movl(jdx, ylen);       // j = ystart+1
 8264 
 8265   subl(xstart, 1);       // i = xstart-1;
 8266   jcc(Assembler::negative, L_done);
 8267 
 8268   push (z);
 8269 
 8270   Label L_last_x;
 8271   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 8272   subl(xstart, 1);       // i = xstart-1;
 8273   jcc(Assembler::negative, L_last_x);
 8274 
 8275   if (UseBMI2Instructions) {
 8276     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 8277     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 8278   } else {
 8279     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 8280     rorq(x_xstart, 32);  // convert big-endian to little-endian
 8281   }
 8282 
 8283   Label L_third_loop_prologue;
 8284   bind(L_third_loop_prologue);
 8285 
 8286   push (x);
 8287   push (xstart);
 8288   push (ylen);
 8289 
 8290 
 8291   if (UseBMI2Instructions) {
 8292     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 8293   } else { // !UseBMI2Instructions
 8294     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 8295   }
 8296 
 8297   pop(ylen);
 8298   pop(xlen);
 8299   pop(x);
 8300   pop(z);
 8301 
 8302   movl(tmp3, xlen);
 8303   addl(tmp3, 1);
 8304   movl(Address(z, tmp3, Address::times_4,  0), carry);
 8305   subl(tmp3, 1);
 8306   jccb(Assembler::negative, L_done);
 8307 
 8308   shrq(carry, 32);
 8309   movl(Address(z, tmp3, Address::times_4,  0), carry);
 8310   jmp(L_second_loop);
 8311 
 8312   // Next infrequent code is moved outside loops.
 8313   bind(L_last_x);
 8314   if (UseBMI2Instructions) {
 8315     movl(rdx, Address(x,  0));
 8316   } else {
 8317     movl(x_xstart, Address(x,  0));
 8318   }
 8319   jmp(L_third_loop_prologue);
 8320 
 8321   bind(L_done);
 8322 
 8323   pop(xlen);
 8324 
 8325   pop(tmp5);
 8326   pop(tmp4);
 8327   pop(tmp3);
 8328   pop(tmp2);
 8329   pop(tmp1);
 8330   pop(tmp0);
 8331 }
 8332 
 8333 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 8334   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 8335   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 8336   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 8337   Label VECTOR8_TAIL, VECTOR4_TAIL;
 8338   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 8339   Label SAME_TILL_END, DONE;
 8340   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 8341 
 8342   //scale is in rcx in both Win64 and Unix
 8343   ShortBranchVerifier sbv(this);
 8344 
 8345   shlq(length);
 8346   xorq(result, result);
 8347 
 8348   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 8349       VM_Version::supports_avx512vlbw()) {
 8350     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 8351 
 8352     cmpq(length, 64);
 8353     jcc(Assembler::less, VECTOR32_TAIL);
 8354 
 8355     movq(tmp1, length);
 8356     andq(tmp1, 0x3F);      // tail count
 8357     andq(length, ~(0x3F)); //vector count
 8358 
 8359     bind(VECTOR64_LOOP);
 8360     // AVX512 code to compare 64 byte vectors.
 8361     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 8362     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 8363     kortestql(k7, k7);
 8364     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 8365     addq(result, 64);
 8366     subq(length, 64);
 8367     jccb(Assembler::notZero, VECTOR64_LOOP);
 8368 
 8369     //bind(VECTOR64_TAIL);
 8370     testq(tmp1, tmp1);
 8371     jcc(Assembler::zero, SAME_TILL_END);
 8372 
 8373     //bind(VECTOR64_TAIL);
 8374     // AVX512 code to compare up to 63 byte vectors.
 8375     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 8376     shlxq(tmp2, tmp2, tmp1);
 8377     notq(tmp2);
 8378     kmovql(k3, tmp2);
 8379 
 8380     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 8381     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 8382 
 8383     ktestql(k7, k3);
 8384     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 8385 
 8386     bind(VECTOR64_NOT_EQUAL);
 8387     kmovql(tmp1, k7);
 8388     notq(tmp1);
 8389     tzcntq(tmp1, tmp1);
 8390     addq(result, tmp1);
 8391     shrq(result);
 8392     jmp(DONE);
 8393     bind(VECTOR32_TAIL);
 8394   }
 8395 
 8396   cmpq(length, 8);
 8397   jcc(Assembler::equal, VECTOR8_LOOP);
 8398   jcc(Assembler::less, VECTOR4_TAIL);
 8399 
 8400   if (UseAVX >= 2) {
 8401     Label VECTOR16_TAIL, VECTOR32_LOOP;
 8402 
 8403     cmpq(length, 16);
 8404     jcc(Assembler::equal, VECTOR16_LOOP);
 8405     jcc(Assembler::less, VECTOR8_LOOP);
 8406 
 8407     cmpq(length, 32);
 8408     jccb(Assembler::less, VECTOR16_TAIL);
 8409 
 8410     subq(length, 32);
 8411     bind(VECTOR32_LOOP);
 8412     vmovdqu(rymm0, Address(obja, result));
 8413     vmovdqu(rymm1, Address(objb, result));
 8414     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 8415     vptest(rymm2, rymm2);
 8416     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 8417     addq(result, 32);
 8418     subq(length, 32);
 8419     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 8420     addq(length, 32);
 8421     jcc(Assembler::equal, SAME_TILL_END);
 8422     //falling through if less than 32 bytes left //close the branch here.
 8423 
 8424     bind(VECTOR16_TAIL);
 8425     cmpq(length, 16);
 8426     jccb(Assembler::less, VECTOR8_TAIL);
 8427     bind(VECTOR16_LOOP);
 8428     movdqu(rymm0, Address(obja, result));
 8429     movdqu(rymm1, Address(objb, result));
 8430     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 8431     ptest(rymm2, rymm2);
 8432     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 8433     addq(result, 16);
 8434     subq(length, 16);
 8435     jcc(Assembler::equal, SAME_TILL_END);
 8436     //falling through if less than 16 bytes left
 8437   } else {//regular intrinsics
 8438 
 8439     cmpq(length, 16);
 8440     jccb(Assembler::less, VECTOR8_TAIL);
 8441 
 8442     subq(length, 16);
 8443     bind(VECTOR16_LOOP);
 8444     movdqu(rymm0, Address(obja, result));
 8445     movdqu(rymm1, Address(objb, result));
 8446     pxor(rymm0, rymm1);
 8447     ptest(rymm0, rymm0);
 8448     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 8449     addq(result, 16);
 8450     subq(length, 16);
 8451     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 8452     addq(length, 16);
 8453     jcc(Assembler::equal, SAME_TILL_END);
 8454     //falling through if less than 16 bytes left
 8455   }
 8456 
 8457   bind(VECTOR8_TAIL);
 8458   cmpq(length, 8);
 8459   jccb(Assembler::less, VECTOR4_TAIL);
 8460   bind(VECTOR8_LOOP);
 8461   movq(tmp1, Address(obja, result));
 8462   movq(tmp2, Address(objb, result));
 8463   xorq(tmp1, tmp2);
 8464   testq(tmp1, tmp1);
 8465   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 8466   addq(result, 8);
 8467   subq(length, 8);
 8468   jcc(Assembler::equal, SAME_TILL_END);
 8469   //falling through if less than 8 bytes left
 8470 
 8471   bind(VECTOR4_TAIL);
 8472   cmpq(length, 4);
 8473   jccb(Assembler::less, BYTES_TAIL);
 8474   bind(VECTOR4_LOOP);
 8475   movl(tmp1, Address(obja, result));
 8476   xorl(tmp1, Address(objb, result));
 8477   testl(tmp1, tmp1);
 8478   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 8479   addq(result, 4);
 8480   subq(length, 4);
 8481   jcc(Assembler::equal, SAME_TILL_END);
 8482   //falling through if less than 4 bytes left
 8483 
 8484   bind(BYTES_TAIL);
 8485   bind(BYTES_LOOP);
 8486   load_unsigned_byte(tmp1, Address(obja, result));
 8487   load_unsigned_byte(tmp2, Address(objb, result));
 8488   xorl(tmp1, tmp2);
 8489   testl(tmp1, tmp1);
 8490   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 8491   decq(length);
 8492   jcc(Assembler::zero, SAME_TILL_END);
 8493   incq(result);
 8494   load_unsigned_byte(tmp1, Address(obja, result));
 8495   load_unsigned_byte(tmp2, Address(objb, result));
 8496   xorl(tmp1, tmp2);
 8497   testl(tmp1, tmp1);
 8498   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 8499   decq(length);
 8500   jcc(Assembler::zero, SAME_TILL_END);
 8501   incq(result);
 8502   load_unsigned_byte(tmp1, Address(obja, result));
 8503   load_unsigned_byte(tmp2, Address(objb, result));
 8504   xorl(tmp1, tmp2);
 8505   testl(tmp1, tmp1);
 8506   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 8507   jmp(SAME_TILL_END);
 8508 
 8509   if (UseAVX >= 2) {
 8510     bind(VECTOR32_NOT_EQUAL);
 8511     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 8512     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 8513     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 8514     vpmovmskb(tmp1, rymm0);
 8515     bsfq(tmp1, tmp1);
 8516     addq(result, tmp1);
 8517     shrq(result);
 8518     jmp(DONE);
 8519   }
 8520 
 8521   bind(VECTOR16_NOT_EQUAL);
 8522   if (UseAVX >= 2) {
 8523     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 8524     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 8525     pxor(rymm0, rymm2);
 8526   } else {
 8527     pcmpeqb(rymm2, rymm2);
 8528     pxor(rymm0, rymm1);
 8529     pcmpeqb(rymm0, rymm1);
 8530     pxor(rymm0, rymm2);
 8531   }
 8532   pmovmskb(tmp1, rymm0);
 8533   bsfq(tmp1, tmp1);
 8534   addq(result, tmp1);
 8535   shrq(result);
 8536   jmpb(DONE);
 8537 
 8538   bind(VECTOR8_NOT_EQUAL);
 8539   bind(VECTOR4_NOT_EQUAL);
 8540   bsfq(tmp1, tmp1);
 8541   shrq(tmp1, 3);
 8542   addq(result, tmp1);
 8543   bind(BYTES_NOT_EQUAL);
 8544   shrq(result);
 8545   jmpb(DONE);
 8546 
 8547   bind(SAME_TILL_END);
 8548   mov64(result, -1);
 8549 
 8550   bind(DONE);
 8551 }
 8552 
 8553 //Helper functions for square_to_len()
 8554 
 8555 /**
 8556  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 8557  * Preserves x and z and modifies rest of the registers.
 8558  */
 8559 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8560   // Perform square and right shift by 1
 8561   // Handle odd xlen case first, then for even xlen do the following
 8562   // jlong carry = 0;
 8563   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 8564   //     huge_128 product = x[j:j+1] * x[j:j+1];
 8565   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 8566   //     z[i+2:i+3] = (jlong)(product >>> 1);
 8567   //     carry = (jlong)product;
 8568   // }
 8569 
 8570   xorq(tmp5, tmp5);     // carry
 8571   xorq(rdxReg, rdxReg);
 8572   xorl(tmp1, tmp1);     // index for x
 8573   xorl(tmp4, tmp4);     // index for z
 8574 
 8575   Label L_first_loop, L_first_loop_exit;
 8576 
 8577   testl(xlen, 1);
 8578   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 8579 
 8580   // Square and right shift by 1 the odd element using 32 bit multiply
 8581   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 8582   imulq(raxReg, raxReg);
 8583   shrq(raxReg, 1);
 8584   adcq(tmp5, 0);
 8585   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 8586   incrementl(tmp1);
 8587   addl(tmp4, 2);
 8588 
 8589   // Square and  right shift by 1 the rest using 64 bit multiply
 8590   bind(L_first_loop);
 8591   cmpptr(tmp1, xlen);
 8592   jccb(Assembler::equal, L_first_loop_exit);
 8593 
 8594   // Square
 8595   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 8596   rorq(raxReg, 32);    // convert big-endian to little-endian
 8597   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 8598 
 8599   // Right shift by 1 and save carry
 8600   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 8601   rcrq(rdxReg, 1);
 8602   rcrq(raxReg, 1);
 8603   adcq(tmp5, 0);
 8604 
 8605   // Store result in z
 8606   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 8607   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 8608 
 8609   // Update indices for x and z
 8610   addl(tmp1, 2);
 8611   addl(tmp4, 4);
 8612   jmp(L_first_loop);
 8613 
 8614   bind(L_first_loop_exit);
 8615 }
 8616 
 8617 
 8618 /**
 8619  * Perform the following multiply add operation using BMI2 instructions
 8620  * carry:sum = sum + op1*op2 + carry
 8621  * op2 should be in rdx
 8622  * op2 is preserved, all other registers are modified
 8623  */
 8624 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 8625   // assert op2 is rdx
 8626   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 8627   addq(sum, carry);
 8628   adcq(tmp2, 0);
 8629   addq(sum, op1);
 8630   adcq(tmp2, 0);
 8631   movq(carry, tmp2);
 8632 }
 8633 
 8634 /**
 8635  * Perform the following multiply add operation:
 8636  * carry:sum = sum + op1*op2 + carry
 8637  * Preserves op1, op2 and modifies rest of registers
 8638  */
 8639 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 8640   // rdx:rax = op1 * op2
 8641   movq(raxReg, op2);
 8642   mulq(op1);
 8643 
 8644   //  rdx:rax = sum + carry + rdx:rax
 8645   addq(sum, carry);
 8646   adcq(rdxReg, 0);
 8647   addq(sum, raxReg);
 8648   adcq(rdxReg, 0);
 8649 
 8650   // carry:sum = rdx:sum
 8651   movq(carry, rdxReg);
 8652 }
 8653 
 8654 /**
 8655  * Add 64 bit long carry into z[] with carry propagation.
 8656  * Preserves z and carry register values and modifies rest of registers.
 8657  *
 8658  */
 8659 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 8660   Label L_fourth_loop, L_fourth_loop_exit;
 8661 
 8662   movl(tmp1, 1);
 8663   subl(zlen, 2);
 8664   addq(Address(z, zlen, Address::times_4, 0), carry);
 8665 
 8666   bind(L_fourth_loop);
 8667   jccb(Assembler::carryClear, L_fourth_loop_exit);
 8668   subl(zlen, 2);
 8669   jccb(Assembler::negative, L_fourth_loop_exit);
 8670   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 8671   jmp(L_fourth_loop);
 8672   bind(L_fourth_loop_exit);
 8673 }
 8674 
 8675 /**
 8676  * Shift z[] left by 1 bit.
 8677  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 8678  *
 8679  */
 8680 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 8681 
 8682   Label L_fifth_loop, L_fifth_loop_exit;
 8683 
 8684   // Fifth loop
 8685   // Perform primitiveLeftShift(z, zlen, 1)
 8686 
 8687   const Register prev_carry = tmp1;
 8688   const Register new_carry = tmp4;
 8689   const Register value = tmp2;
 8690   const Register zidx = tmp3;
 8691 
 8692   // int zidx, carry;
 8693   // long value;
 8694   // carry = 0;
 8695   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 8696   //    (carry:value)  = (z[i] << 1) | carry ;
 8697   //    z[i] = value;
 8698   // }
 8699 
 8700   movl(zidx, zlen);
 8701   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 8702 
 8703   bind(L_fifth_loop);
 8704   decl(zidx);  // Use decl to preserve carry flag
 8705   decl(zidx);
 8706   jccb(Assembler::negative, L_fifth_loop_exit);
 8707 
 8708   if (UseBMI2Instructions) {
 8709      movq(value, Address(z, zidx, Address::times_4, 0));
 8710      rclq(value, 1);
 8711      rorxq(value, value, 32);
 8712      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 8713   }
 8714   else {
 8715     // clear new_carry
 8716     xorl(new_carry, new_carry);
 8717 
 8718     // Shift z[i] by 1, or in previous carry and save new carry
 8719     movq(value, Address(z, zidx, Address::times_4, 0));
 8720     shlq(value, 1);
 8721     adcl(new_carry, 0);
 8722 
 8723     orq(value, prev_carry);
 8724     rorq(value, 0x20);
 8725     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 8726 
 8727     // Set previous carry = new carry
 8728     movl(prev_carry, new_carry);
 8729   }
 8730   jmp(L_fifth_loop);
 8731 
 8732   bind(L_fifth_loop_exit);
 8733 }
 8734 
 8735 
 8736 /**
 8737  * Code for BigInteger::squareToLen() intrinsic
 8738  *
 8739  * rdi: x
 8740  * rsi: len
 8741  * r8:  z
 8742  * rcx: zlen
 8743  * r12: tmp1
 8744  * r13: tmp2
 8745  * r14: tmp3
 8746  * r15: tmp4
 8747  * rbx: tmp5
 8748  *
 8749  */
 8750 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8751 
 8752   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 8753   push(tmp1);
 8754   push(tmp2);
 8755   push(tmp3);
 8756   push(tmp4);
 8757   push(tmp5);
 8758 
 8759   // First loop
 8760   // Store the squares, right shifted one bit (i.e., divided by 2).
 8761   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8762 
 8763   // Add in off-diagonal sums.
 8764   //
 8765   // Second, third (nested) and fourth loops.
 8766   // zlen +=2;
 8767   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 8768   //    carry = 0;
 8769   //    long op2 = x[xidx:xidx+1];
 8770   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 8771   //       k -= 2;
 8772   //       long op1 = x[j:j+1];
 8773   //       long sum = z[k:k+1];
 8774   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 8775   //       z[k:k+1] = sum;
 8776   //    }
 8777   //    add_one_64(z, k, carry, tmp_regs);
 8778   // }
 8779 
 8780   const Register carry = tmp5;
 8781   const Register sum = tmp3;
 8782   const Register op1 = tmp4;
 8783   Register op2 = tmp2;
 8784 
 8785   push(zlen);
 8786   push(len);
 8787   addl(zlen,2);
 8788   bind(L_second_loop);
 8789   xorq(carry, carry);
 8790   subl(zlen, 4);
 8791   subl(len, 2);
 8792   push(zlen);
 8793   push(len);
 8794   cmpl(len, 0);
 8795   jccb(Assembler::lessEqual, L_second_loop_exit);
 8796 
 8797   // Multiply an array by one 64 bit long.
 8798   if (UseBMI2Instructions) {
 8799     op2 = rdxReg;
 8800     movq(op2, Address(x, len, Address::times_4,  0));
 8801     rorxq(op2, op2, 32);
 8802   }
 8803   else {
 8804     movq(op2, Address(x, len, Address::times_4,  0));
 8805     rorq(op2, 32);
 8806   }
 8807 
 8808   bind(L_third_loop);
 8809   decrementl(len);
 8810   jccb(Assembler::negative, L_third_loop_exit);
 8811   decrementl(len);
 8812   jccb(Assembler::negative, L_last_x);
 8813 
 8814   movq(op1, Address(x, len, Address::times_4,  0));
 8815   rorq(op1, 32);
 8816 
 8817   bind(L_multiply);
 8818   subl(zlen, 2);
 8819   movq(sum, Address(z, zlen, Address::times_4,  0));
 8820 
 8821   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 8822   if (UseBMI2Instructions) {
 8823     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 8824   }
 8825   else {
 8826     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8827   }
 8828 
 8829   movq(Address(z, zlen, Address::times_4, 0), sum);
 8830 
 8831   jmp(L_third_loop);
 8832   bind(L_third_loop_exit);
 8833 
 8834   // Fourth loop
 8835   // Add 64 bit long carry into z with carry propagation.
 8836   // Uses offsetted zlen.
 8837   add_one_64(z, zlen, carry, tmp1);
 8838 
 8839   pop(len);
 8840   pop(zlen);
 8841   jmp(L_second_loop);
 8842 
 8843   // Next infrequent code is moved outside loops.
 8844   bind(L_last_x);
 8845   movl(op1, Address(x, 0));
 8846   jmp(L_multiply);
 8847 
 8848   bind(L_second_loop_exit);
 8849   pop(len);
 8850   pop(zlen);
 8851   pop(len);
 8852   pop(zlen);
 8853 
 8854   // Fifth loop
 8855   // Shift z left 1 bit.
 8856   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 8857 
 8858   // z[zlen-1] |= x[len-1] & 1;
 8859   movl(tmp3, Address(x, len, Address::times_4, -4));
 8860   andl(tmp3, 1);
 8861   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 8862 
 8863   pop(tmp5);
 8864   pop(tmp4);
 8865   pop(tmp3);
 8866   pop(tmp2);
 8867   pop(tmp1);
 8868 }
 8869 
 8870 /**
 8871  * Helper function for mul_add()
 8872  * Multiply the in[] by int k and add to out[] starting at offset offs using
 8873  * 128 bit by 32 bit multiply and return the carry in tmp5.
 8874  * Only quad int aligned length of in[] is operated on in this function.
 8875  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 8876  * This function preserves out, in and k registers.
 8877  * len and offset point to the appropriate index in "in" & "out" correspondingly
 8878  * tmp5 has the carry.
 8879  * other registers are temporary and are modified.
 8880  *
 8881  */
 8882 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 8883   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 8884   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8885 
 8886   Label L_first_loop, L_first_loop_exit;
 8887 
 8888   movl(tmp1, len);
 8889   shrl(tmp1, 2);
 8890 
 8891   bind(L_first_loop);
 8892   subl(tmp1, 1);
 8893   jccb(Assembler::negative, L_first_loop_exit);
 8894 
 8895   subl(len, 4);
 8896   subl(offset, 4);
 8897 
 8898   Register op2 = tmp2;
 8899   const Register sum = tmp3;
 8900   const Register op1 = tmp4;
 8901   const Register carry = tmp5;
 8902 
 8903   if (UseBMI2Instructions) {
 8904     op2 = rdxReg;
 8905   }
 8906 
 8907   movq(op1, Address(in, len, Address::times_4,  8));
 8908   rorq(op1, 32);
 8909   movq(sum, Address(out, offset, Address::times_4,  8));
 8910   rorq(sum, 32);
 8911   if (UseBMI2Instructions) {
 8912     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8913   }
 8914   else {
 8915     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8916   }
 8917   // Store back in big endian from little endian
 8918   rorq(sum, 0x20);
 8919   movq(Address(out, offset, Address::times_4,  8), sum);
 8920 
 8921   movq(op1, Address(in, len, Address::times_4,  0));
 8922   rorq(op1, 32);
 8923   movq(sum, Address(out, offset, Address::times_4,  0));
 8924   rorq(sum, 32);
 8925   if (UseBMI2Instructions) {
 8926     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8927   }
 8928   else {
 8929     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8930   }
 8931   // Store back in big endian from little endian
 8932   rorq(sum, 0x20);
 8933   movq(Address(out, offset, Address::times_4,  0), sum);
 8934 
 8935   jmp(L_first_loop);
 8936   bind(L_first_loop_exit);
 8937 }
 8938 
 8939 /**
 8940  * Code for BigInteger::mulAdd() intrinsic
 8941  *
 8942  * rdi: out
 8943  * rsi: in
 8944  * r11: offs (out.length - offset)
 8945  * rcx: len
 8946  * r8:  k
 8947  * r12: tmp1
 8948  * r13: tmp2
 8949  * r14: tmp3
 8950  * r15: tmp4
 8951  * rbx: tmp5
 8952  * Multiply the in[] by word k and add to out[], return the carry in rax
 8953  */
 8954 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 8955    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 8956    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8957 
 8958   Label L_carry, L_last_in, L_done;
 8959 
 8960 // carry = 0;
 8961 // for (int j=len-1; j >= 0; j--) {
 8962 //    long product = (in[j] & LONG_MASK) * kLong +
 8963 //                   (out[offs] & LONG_MASK) + carry;
 8964 //    out[offs--] = (int)product;
 8965 //    carry = product >>> 32;
 8966 // }
 8967 //
 8968   push(tmp1);
 8969   push(tmp2);
 8970   push(tmp3);
 8971   push(tmp4);
 8972   push(tmp5);
 8973 
 8974   Register op2 = tmp2;
 8975   const Register sum = tmp3;
 8976   const Register op1 = tmp4;
 8977   const Register carry =  tmp5;
 8978 
 8979   if (UseBMI2Instructions) {
 8980     op2 = rdxReg;
 8981     movl(op2, k);
 8982   }
 8983   else {
 8984     movl(op2, k);
 8985   }
 8986 
 8987   xorq(carry, carry);
 8988 
 8989   //First loop
 8990 
 8991   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 8992   //The carry is in tmp5
 8993   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8994 
 8995   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 8996   decrementl(len);
 8997   jccb(Assembler::negative, L_carry);
 8998   decrementl(len);
 8999   jccb(Assembler::negative, L_last_in);
 9000 
 9001   movq(op1, Address(in, len, Address::times_4,  0));
 9002   rorq(op1, 32);
 9003 
 9004   subl(offs, 2);
 9005   movq(sum, Address(out, offs, Address::times_4,  0));
 9006   rorq(sum, 32);
 9007 
 9008   if (UseBMI2Instructions) {
 9009     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 9010   }
 9011   else {
 9012     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 9013   }
 9014 
 9015   // Store back in big endian from little endian
 9016   rorq(sum, 0x20);
 9017   movq(Address(out, offs, Address::times_4,  0), sum);
 9018 
 9019   testl(len, len);
 9020   jccb(Assembler::zero, L_carry);
 9021 
 9022   //Multiply the last in[] entry, if any
 9023   bind(L_last_in);
 9024   movl(op1, Address(in, 0));
 9025   movl(sum, Address(out, offs, Address::times_4,  -4));
 9026 
 9027   movl(raxReg, k);
 9028   mull(op1); //tmp4 * eax -> edx:eax
 9029   addl(sum, carry);
 9030   adcl(rdxReg, 0);
 9031   addl(sum, raxReg);
 9032   adcl(rdxReg, 0);
 9033   movl(carry, rdxReg);
 9034 
 9035   movl(Address(out, offs, Address::times_4,  -4), sum);
 9036 
 9037   bind(L_carry);
 9038   //return tmp5/carry as carry in rax
 9039   movl(rax, carry);
 9040 
 9041   bind(L_done);
 9042   pop(tmp5);
 9043   pop(tmp4);
 9044   pop(tmp3);
 9045   pop(tmp2);
 9046   pop(tmp1);
 9047 }
 9048 #endif
 9049 
 9050 /**
 9051  * Emits code to update CRC-32 with a byte value according to constants in table
 9052  *
 9053  * @param [in,out]crc   Register containing the crc.
 9054  * @param [in]val       Register containing the byte to fold into the CRC.
 9055  * @param [in]table     Register containing the table of crc constants.
 9056  *
 9057  * uint32_t crc;
 9058  * val = crc_table[(val ^ crc) & 0xFF];
 9059  * crc = val ^ (crc >> 8);
 9060  *
 9061  */
 9062 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 9063   xorl(val, crc);
 9064   andl(val, 0xFF);
 9065   shrl(crc, 8); // unsigned shift
 9066   xorl(crc, Address(table, val, Address::times_4, 0));
 9067 }
 9068 
 9069 /**
 9070  * Fold 128-bit data chunk
 9071  */
 9072 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 9073   if (UseAVX > 0) {
 9074     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 9075     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 9076     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 9077     pxor(xcrc, xtmp);
 9078   } else {
 9079     movdqa(xtmp, xcrc);
 9080     pclmulhdq(xtmp, xK);   // [123:64]
 9081     pclmulldq(xcrc, xK);   // [63:0]
 9082     pxor(xcrc, xtmp);
 9083     movdqu(xtmp, Address(buf, offset));
 9084     pxor(xcrc, xtmp);
 9085   }
 9086 }
 9087 
 9088 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 9089   if (UseAVX > 0) {
 9090     vpclmulhdq(xtmp, xK, xcrc);
 9091     vpclmulldq(xcrc, xK, xcrc);
 9092     pxor(xcrc, xbuf);
 9093     pxor(xcrc, xtmp);
 9094   } else {
 9095     movdqa(xtmp, xcrc);
 9096     pclmulhdq(xtmp, xK);
 9097     pclmulldq(xcrc, xK);
 9098     pxor(xcrc, xbuf);
 9099     pxor(xcrc, xtmp);
 9100   }
 9101 }
 9102 
 9103 /**
 9104  * 8-bit folds to compute 32-bit CRC
 9105  *
 9106  * uint64_t xcrc;
 9107  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 9108  */
 9109 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 9110   movdl(tmp, xcrc);
 9111   andl(tmp, 0xFF);
 9112   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 9113   psrldq(xcrc, 1); // unsigned shift one byte
 9114   pxor(xcrc, xtmp);
 9115 }
 9116 
 9117 /**
 9118  * uint32_t crc;
 9119  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 9120  */
 9121 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 9122   movl(tmp, crc);
 9123   andl(tmp, 0xFF);
 9124   shrl(crc, 8);
 9125   xorl(crc, Address(table, tmp, Address::times_4, 0));
 9126 }
 9127 
 9128 /**
 9129  * @param crc   register containing existing CRC (32-bit)
 9130  * @param buf   register pointing to input byte buffer (byte*)
 9131  * @param len   register containing number of bytes
 9132  * @param table register that will contain address of CRC table
 9133  * @param tmp   scratch register
 9134  */
 9135 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 9136   assert_different_registers(crc, buf, len, table, tmp, rax);
 9137 
 9138   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 9139   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 9140 
 9141   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 9142   // context for the registers used, where all instructions below are using 128-bit mode
 9143   // On EVEX without VL and BW, these instructions will all be AVX.
 9144   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 9145   notl(crc); // ~crc
 9146   cmpl(len, 16);
 9147   jcc(Assembler::less, L_tail);
 9148 
 9149   // Align buffer to 16 bytes
 9150   movl(tmp, buf);
 9151   andl(tmp, 0xF);
 9152   jccb(Assembler::zero, L_aligned);
 9153   subl(tmp,  16);
 9154   addl(len, tmp);
 9155 
 9156   align(4);
 9157   BIND(L_align_loop);
 9158   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 9159   update_byte_crc32(crc, rax, table);
 9160   increment(buf);
 9161   incrementl(tmp);
 9162   jccb(Assembler::less, L_align_loop);
 9163 
 9164   BIND(L_aligned);
 9165   movl(tmp, len); // save
 9166   shrl(len, 4);
 9167   jcc(Assembler::zero, L_tail_restore);
 9168 
 9169   // Fold crc into first bytes of vector
 9170   movdqa(xmm1, Address(buf, 0));
 9171   movdl(rax, xmm1);
 9172   xorl(crc, rax);
 9173   if (VM_Version::supports_sse4_1()) {
 9174     pinsrd(xmm1, crc, 0);
 9175   } else {
 9176     pinsrw(xmm1, crc, 0);
 9177     shrl(crc, 16);
 9178     pinsrw(xmm1, crc, 1);
 9179   }
 9180   addptr(buf, 16);
 9181   subl(len, 4); // len > 0
 9182   jcc(Assembler::less, L_fold_tail);
 9183 
 9184   movdqa(xmm2, Address(buf,  0));
 9185   movdqa(xmm3, Address(buf, 16));
 9186   movdqa(xmm4, Address(buf, 32));
 9187   addptr(buf, 48);
 9188   subl(len, 3);
 9189   jcc(Assembler::lessEqual, L_fold_512b);
 9190 
 9191   // Fold total 512 bits of polynomial on each iteration,
 9192   // 128 bits per each of 4 parallel streams.
 9193   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 9194 
 9195   align32();
 9196   BIND(L_fold_512b_loop);
 9197   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 9198   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 9199   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 9200   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 9201   addptr(buf, 64);
 9202   subl(len, 4);
 9203   jcc(Assembler::greater, L_fold_512b_loop);
 9204 
 9205   // Fold 512 bits to 128 bits.
 9206   BIND(L_fold_512b);
 9207   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 9208   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 9209   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 9210   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 9211 
 9212   // Fold the rest of 128 bits data chunks
 9213   BIND(L_fold_tail);
 9214   addl(len, 3);
 9215   jccb(Assembler::lessEqual, L_fold_128b);
 9216   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 9217 
 9218   BIND(L_fold_tail_loop);
 9219   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 9220   addptr(buf, 16);
 9221   decrementl(len);
 9222   jccb(Assembler::greater, L_fold_tail_loop);
 9223 
 9224   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 9225   BIND(L_fold_128b);
 9226   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 9227   if (UseAVX > 0) {
 9228     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 9229     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 9230     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 9231   } else {
 9232     movdqa(xmm2, xmm0);
 9233     pclmulqdq(xmm2, xmm1, 0x1);
 9234     movdqa(xmm3, xmm0);
 9235     pand(xmm3, xmm2);
 9236     pclmulqdq(xmm0, xmm3, 0x1);
 9237   }
 9238   psrldq(xmm1, 8);
 9239   psrldq(xmm2, 4);
 9240   pxor(xmm0, xmm1);
 9241   pxor(xmm0, xmm2);
 9242 
 9243   // 8 8-bit folds to compute 32-bit CRC.
 9244   for (int j = 0; j < 4; j++) {
 9245     fold_8bit_crc32(xmm0, table, xmm1, rax);
 9246   }
 9247   movdl(crc, xmm0); // mov 32 bits to general register
 9248   for (int j = 0; j < 4; j++) {
 9249     fold_8bit_crc32(crc, table, rax);
 9250   }
 9251 
 9252   BIND(L_tail_restore);
 9253   movl(len, tmp); // restore
 9254   BIND(L_tail);
 9255   andl(len, 0xf);
 9256   jccb(Assembler::zero, L_exit);
 9257 
 9258   // Fold the rest of bytes
 9259   align(4);
 9260   BIND(L_tail_loop);
 9261   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 9262   update_byte_crc32(crc, rax, table);
 9263   increment(buf);
 9264   decrementl(len);
 9265   jccb(Assembler::greater, L_tail_loop);
 9266 
 9267   BIND(L_exit);
 9268   notl(crc); // ~c
 9269 }
 9270 
 9271 #ifdef _LP64
 9272 // Helper function for AVX 512 CRC32
 9273 // Fold 512-bit data chunks
 9274 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 9275                                              Register pos, int offset) {
 9276   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 9277   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 9278   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 9279   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 9280   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 9281 }
 9282 
 9283 // Helper function for AVX 512 CRC32
 9284 // Compute CRC32 for < 256B buffers
 9285 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 9286                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 9287                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 9288 
 9289   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 9290   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 9291   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 9292 
 9293   // check if there is enough buffer to be able to fold 16B at a time
 9294   cmpl(len, 32);
 9295   jcc(Assembler::less, L_less_than_32);
 9296 
 9297   // if there is, load the constants
 9298   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 9299   movdl(xmm0, crc);                        // get the initial crc value
 9300   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 9301   pxor(xmm7, xmm0);
 9302 
 9303   // update the buffer pointer
 9304   addl(pos, 16);
 9305   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 9306   subl(len, 32);
 9307   jmp(L_16B_reduction_loop);
 9308 
 9309   bind(L_less_than_32);
 9310   //mov initial crc to the return value. this is necessary for zero - length buffers.
 9311   movl(rax, crc);
 9312   testl(len, len);
 9313   jcc(Assembler::equal, L_cleanup);
 9314 
 9315   movdl(xmm0, crc);                        //get the initial crc value
 9316 
 9317   cmpl(len, 16);
 9318   jcc(Assembler::equal, L_exact_16_left);
 9319   jcc(Assembler::less, L_less_than_16_left);
 9320 
 9321   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 9322   pxor(xmm7, xmm0);                       //xor the initial crc value
 9323   addl(pos, 16);
 9324   subl(len, 16);
 9325   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 9326   jmp(L_get_last_two_xmms);
 9327 
 9328   bind(L_less_than_16_left);
 9329   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 9330   pxor(xmm1, xmm1);
 9331   movptr(tmp1, rsp);
 9332   movdqu(Address(tmp1, 0 * 16), xmm1);
 9333 
 9334   cmpl(len, 4);
 9335   jcc(Assembler::less, L_only_less_than_4);
 9336 
 9337   //backup the counter value
 9338   movl(tmp2, len);
 9339   cmpl(len, 8);
 9340   jcc(Assembler::less, L_less_than_8_left);
 9341 
 9342   //load 8 Bytes
 9343   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 9344   movq(Address(tmp1, 0 * 16), rax);
 9345   addptr(tmp1, 8);
 9346   subl(len, 8);
 9347   addl(pos, 8);
 9348 
 9349   bind(L_less_than_8_left);
 9350   cmpl(len, 4);
 9351   jcc(Assembler::less, L_less_than_4_left);
 9352 
 9353   //load 4 Bytes
 9354   movl(rax, Address(buf, pos, Address::times_1, 0));
 9355   movl(Address(tmp1, 0 * 16), rax);
 9356   addptr(tmp1, 4);
 9357   subl(len, 4);
 9358   addl(pos, 4);
 9359 
 9360   bind(L_less_than_4_left);
 9361   cmpl(len, 2);
 9362   jcc(Assembler::less, L_less_than_2_left);
 9363 
 9364   // load 2 Bytes
 9365   movw(rax, Address(buf, pos, Address::times_1, 0));
 9366   movl(Address(tmp1, 0 * 16), rax);
 9367   addptr(tmp1, 2);
 9368   subl(len, 2);
 9369   addl(pos, 2);
 9370 
 9371   bind(L_less_than_2_left);
 9372   cmpl(len, 1);
 9373   jcc(Assembler::less, L_zero_left);
 9374 
 9375   // load 1 Byte
 9376   movb(rax, Address(buf, pos, Address::times_1, 0));
 9377   movb(Address(tmp1, 0 * 16), rax);
 9378 
 9379   bind(L_zero_left);
 9380   movdqu(xmm7, Address(rsp, 0));
 9381   pxor(xmm7, xmm0);                       //xor the initial crc value
 9382 
 9383   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 9384   movdqu(xmm0, Address(rax, tmp2));
 9385   pshufb(xmm7, xmm0);
 9386   jmp(L_128_done);
 9387 
 9388   bind(L_exact_16_left);
 9389   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 9390   pxor(xmm7, xmm0);                       //xor the initial crc value
 9391   jmp(L_128_done);
 9392 
 9393   bind(L_only_less_than_4);
 9394   cmpl(len, 3);
 9395   jcc(Assembler::less, L_only_less_than_3);
 9396 
 9397   // load 3 Bytes
 9398   movb(rax, Address(buf, pos, Address::times_1, 0));
 9399   movb(Address(tmp1, 0), rax);
 9400 
 9401   movb(rax, Address(buf, pos, Address::times_1, 1));
 9402   movb(Address(tmp1, 1), rax);
 9403 
 9404   movb(rax, Address(buf, pos, Address::times_1, 2));
 9405   movb(Address(tmp1, 2), rax);
 9406 
 9407   movdqu(xmm7, Address(rsp, 0));
 9408   pxor(xmm7, xmm0);                     //xor the initial crc value
 9409 
 9410   pslldq(xmm7, 0x5);
 9411   jmp(L_barrett);
 9412   bind(L_only_less_than_3);
 9413   cmpl(len, 2);
 9414   jcc(Assembler::less, L_only_less_than_2);
 9415 
 9416   // load 2 Bytes
 9417   movb(rax, Address(buf, pos, Address::times_1, 0));
 9418   movb(Address(tmp1, 0), rax);
 9419 
 9420   movb(rax, Address(buf, pos, Address::times_1, 1));
 9421   movb(Address(tmp1, 1), rax);
 9422 
 9423   movdqu(xmm7, Address(rsp, 0));
 9424   pxor(xmm7, xmm0);                     //xor the initial crc value
 9425 
 9426   pslldq(xmm7, 0x6);
 9427   jmp(L_barrett);
 9428 
 9429   bind(L_only_less_than_2);
 9430   //load 1 Byte
 9431   movb(rax, Address(buf, pos, Address::times_1, 0));
 9432   movb(Address(tmp1, 0), rax);
 9433 
 9434   movdqu(xmm7, Address(rsp, 0));
 9435   pxor(xmm7, xmm0);                     //xor the initial crc value
 9436 
 9437   pslldq(xmm7, 0x7);
 9438 }
 9439 
 9440 /**
 9441 * Compute CRC32 using AVX512 instructions
 9442 * param crc   register containing existing CRC (32-bit)
 9443 * param buf   register pointing to input byte buffer (byte*)
 9444 * param len   register containing number of bytes
 9445 * param table address of crc or crc32c table
 9446 * param tmp1  scratch register
 9447 * param tmp2  scratch register
 9448 * return rax  result register
 9449 *
 9450 * This routine is identical for crc32c with the exception of the precomputed constant
 9451 * table which will be passed as the table argument.  The calculation steps are
 9452 * the same for both variants.
 9453 */
 9454 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 9455   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 9456 
 9457   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 9458   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 9459   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 9460   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 9461   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 9462 
 9463   const Register pos = r12;
 9464   push(r12);
 9465   subptr(rsp, 16 * 2 + 8);
 9466 
 9467   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 9468   // context for the registers used, where all instructions below are using 128-bit mode
 9469   // On EVEX without VL and BW, these instructions will all be AVX.
 9470   movl(pos, 0);
 9471 
 9472   // check if smaller than 256B
 9473   cmpl(len, 256);
 9474   jcc(Assembler::less, L_less_than_256);
 9475 
 9476   // load the initial crc value
 9477   movdl(xmm10, crc);
 9478 
 9479   // receive the initial 64B data, xor the initial crc value
 9480   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 9481   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 9482   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 9483   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 9484 
 9485   subl(len, 256);
 9486   cmpl(len, 256);
 9487   jcc(Assembler::less, L_fold_128_B_loop);
 9488 
 9489   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 9490   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 9491   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 9492   subl(len, 256);
 9493 
 9494   bind(L_fold_256_B_loop);
 9495   addl(pos, 256);
 9496   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 9497   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 9498   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 9499   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 9500 
 9501   subl(len, 256);
 9502   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 9503 
 9504   // Fold 256 into 128
 9505   addl(pos, 256);
 9506   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 9507   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 9508   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 9509 
 9510   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 9511   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 9512   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 9513 
 9514   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 9515   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 9516 
 9517   addl(len, 128);
 9518   jmp(L_fold_128_B_register);
 9519 
 9520   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 9521   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 9522 
 9523   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 9524   bind(L_fold_128_B_loop);
 9525   addl(pos, 128);
 9526   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 9527   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 9528 
 9529   subl(len, 128);
 9530   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 9531 
 9532   addl(pos, 128);
 9533 
 9534   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 9535   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 9536   bind(L_fold_128_B_register);
 9537   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 9538   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 9539   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 9540   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 9541   // save last that has no multiplicand
 9542   vextracti64x2(xmm7, xmm4, 3);
 9543 
 9544   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 9545   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 9546   // Needed later in reduction loop
 9547   movdqu(xmm10, Address(table, 1 * 16));
 9548   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 9549   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 9550 
 9551   // Swap 1,0,3,2 - 01 00 11 10
 9552   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 9553   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 9554   vextracti128(xmm5, xmm8, 1);
 9555   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 9556 
 9557   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 9558   // instead of a cmp instruction, we use the negative flag with the jl instruction
 9559   addl(len, 128 - 16);
 9560   jcc(Assembler::less, L_final_reduction_for_128);
 9561 
 9562   bind(L_16B_reduction_loop);
 9563   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 9564   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 9565   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 9566   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 9567   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 9568   addl(pos, 16);
 9569   subl(len, 16);
 9570   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 9571 
 9572   bind(L_final_reduction_for_128);
 9573   addl(len, 16);
 9574   jcc(Assembler::equal, L_128_done);
 9575 
 9576   bind(L_get_last_two_xmms);
 9577   movdqu(xmm2, xmm7);
 9578   addl(pos, len);
 9579   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 9580   subl(pos, len);
 9581 
 9582   // get rid of the extra data that was loaded before
 9583   // load the shift constant
 9584   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 9585   movdqu(xmm0, Address(rax, len));
 9586   addl(rax, len);
 9587 
 9588   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 9589   //Change mask to 512
 9590   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 9591   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 9592 
 9593   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 9594   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 9595   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 9596   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 9597   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 9598 
 9599   bind(L_128_done);
 9600   // compute crc of a 128-bit value
 9601   movdqu(xmm10, Address(table, 3 * 16));
 9602   movdqu(xmm0, xmm7);
 9603 
 9604   // 64b fold
 9605   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 9606   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 9607   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 9608 
 9609   // 32b fold
 9610   movdqu(xmm0, xmm7);
 9611   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 9612   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 9613   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 9614   jmp(L_barrett);
 9615 
 9616   bind(L_less_than_256);
 9617   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 9618 
 9619   //barrett reduction
 9620   bind(L_barrett);
 9621   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 9622   movdqu(xmm1, xmm7);
 9623   movdqu(xmm2, xmm7);
 9624   movdqu(xmm10, Address(table, 4 * 16));
 9625 
 9626   pclmulqdq(xmm7, xmm10, 0x0);
 9627   pxor(xmm7, xmm2);
 9628   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 9629   movdqu(xmm2, xmm7);
 9630   pclmulqdq(xmm7, xmm10, 0x10);
 9631   pxor(xmm7, xmm2);
 9632   pxor(xmm7, xmm1);
 9633   pextrd(crc, xmm7, 2);
 9634 
 9635   bind(L_cleanup);
 9636   addptr(rsp, 16 * 2 + 8);
 9637   pop(r12);
 9638 }
 9639 
 9640 // S. Gueron / Information Processing Letters 112 (2012) 184
 9641 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 9642 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 9643 // Output: the 64-bit carry-less product of B * CONST
 9644 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 9645                                      Register tmp1, Register tmp2, Register tmp3) {
 9646   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 9647   if (n > 0) {
 9648     addq(tmp3, n * 256 * 8);
 9649   }
 9650   //    Q1 = TABLEExt[n][B & 0xFF];
 9651   movl(tmp1, in);
 9652   andl(tmp1, 0x000000FF);
 9653   shll(tmp1, 3);
 9654   addq(tmp1, tmp3);
 9655   movq(tmp1, Address(tmp1, 0));
 9656 
 9657   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 9658   movl(tmp2, in);
 9659   shrl(tmp2, 8);
 9660   andl(tmp2, 0x000000FF);
 9661   shll(tmp2, 3);
 9662   addq(tmp2, tmp3);
 9663   movq(tmp2, Address(tmp2, 0));
 9664 
 9665   shlq(tmp2, 8);
 9666   xorq(tmp1, tmp2);
 9667 
 9668   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 9669   movl(tmp2, in);
 9670   shrl(tmp2, 16);
 9671   andl(tmp2, 0x000000FF);
 9672   shll(tmp2, 3);
 9673   addq(tmp2, tmp3);
 9674   movq(tmp2, Address(tmp2, 0));
 9675 
 9676   shlq(tmp2, 16);
 9677   xorq(tmp1, tmp2);
 9678 
 9679   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 9680   shrl(in, 24);
 9681   andl(in, 0x000000FF);
 9682   shll(in, 3);
 9683   addq(in, tmp3);
 9684   movq(in, Address(in, 0));
 9685 
 9686   shlq(in, 24);
 9687   xorq(in, tmp1);
 9688   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 9689 }
 9690 
 9691 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 9692                                       Register in_out,
 9693                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 9694                                       XMMRegister w_xtmp2,
 9695                                       Register tmp1,
 9696                                       Register n_tmp2, Register n_tmp3) {
 9697   if (is_pclmulqdq_supported) {
 9698     movdl(w_xtmp1, in_out); // modified blindly
 9699 
 9700     movl(tmp1, const_or_pre_comp_const_index);
 9701     movdl(w_xtmp2, tmp1);
 9702     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 9703 
 9704     movdq(in_out, w_xtmp1);
 9705   } else {
 9706     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 9707   }
 9708 }
 9709 
 9710 // Recombination Alternative 2: No bit-reflections
 9711 // T1 = (CRC_A * U1) << 1
 9712 // T2 = (CRC_B * U2) << 1
 9713 // C1 = T1 >> 32
 9714 // C2 = T2 >> 32
 9715 // T1 = T1 & 0xFFFFFFFF
 9716 // T2 = T2 & 0xFFFFFFFF
 9717 // T1 = CRC32(0, T1)
 9718 // T2 = CRC32(0, T2)
 9719 // C1 = C1 ^ T1
 9720 // C2 = C2 ^ T2
 9721 // CRC = C1 ^ C2 ^ CRC_C
 9722 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9723                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9724                                      Register tmp1, Register tmp2,
 9725                                      Register n_tmp3) {
 9726   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9727   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9728   shlq(in_out, 1);
 9729   movl(tmp1, in_out);
 9730   shrq(in_out, 32);
 9731   xorl(tmp2, tmp2);
 9732   crc32(tmp2, tmp1, 4);
 9733   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 9734   shlq(in1, 1);
 9735   movl(tmp1, in1);
 9736   shrq(in1, 32);
 9737   xorl(tmp2, tmp2);
 9738   crc32(tmp2, tmp1, 4);
 9739   xorl(in1, tmp2);
 9740   xorl(in_out, in1);
 9741   xorl(in_out, in2);
 9742 }
 9743 
 9744 // Set N to predefined value
 9745 // Subtract from a length of a buffer
 9746 // execute in a loop:
 9747 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 9748 // for i = 1 to N do
 9749 //  CRC_A = CRC32(CRC_A, A[i])
 9750 //  CRC_B = CRC32(CRC_B, B[i])
 9751 //  CRC_C = CRC32(CRC_C, C[i])
 9752 // end for
 9753 // Recombine
 9754 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9755                                        Register in_out1, Register in_out2, Register in_out3,
 9756                                        Register tmp1, Register tmp2, Register tmp3,
 9757                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9758                                        Register tmp4, Register tmp5,
 9759                                        Register n_tmp6) {
 9760   Label L_processPartitions;
 9761   Label L_processPartition;
 9762   Label L_exit;
 9763 
 9764   bind(L_processPartitions);
 9765   cmpl(in_out1, 3 * size);
 9766   jcc(Assembler::less, L_exit);
 9767     xorl(tmp1, tmp1);
 9768     xorl(tmp2, tmp2);
 9769     movq(tmp3, in_out2);
 9770     addq(tmp3, size);
 9771 
 9772     bind(L_processPartition);
 9773       crc32(in_out3, Address(in_out2, 0), 8);
 9774       crc32(tmp1, Address(in_out2, size), 8);
 9775       crc32(tmp2, Address(in_out2, size * 2), 8);
 9776       addq(in_out2, 8);
 9777       cmpq(in_out2, tmp3);
 9778       jcc(Assembler::less, L_processPartition);
 9779     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9780             w_xtmp1, w_xtmp2, w_xtmp3,
 9781             tmp4, tmp5,
 9782             n_tmp6);
 9783     addq(in_out2, 2 * size);
 9784     subl(in_out1, 3 * size);
 9785     jmp(L_processPartitions);
 9786 
 9787   bind(L_exit);
 9788 }
 9789 #else
 9790 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 9791                                      Register tmp1, Register tmp2, Register tmp3,
 9792                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 9793   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 9794   if (n > 0) {
 9795     addl(tmp3, n * 256 * 8);
 9796   }
 9797   //    Q1 = TABLEExt[n][B & 0xFF];
 9798   movl(tmp1, in_out);
 9799   andl(tmp1, 0x000000FF);
 9800   shll(tmp1, 3);
 9801   addl(tmp1, tmp3);
 9802   movq(xtmp1, Address(tmp1, 0));
 9803 
 9804   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 9805   movl(tmp2, in_out);
 9806   shrl(tmp2, 8);
 9807   andl(tmp2, 0x000000FF);
 9808   shll(tmp2, 3);
 9809   addl(tmp2, tmp3);
 9810   movq(xtmp2, Address(tmp2, 0));
 9811 
 9812   psllq(xtmp2, 8);
 9813   pxor(xtmp1, xtmp2);
 9814 
 9815   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 9816   movl(tmp2, in_out);
 9817   shrl(tmp2, 16);
 9818   andl(tmp2, 0x000000FF);
 9819   shll(tmp2, 3);
 9820   addl(tmp2, tmp3);
 9821   movq(xtmp2, Address(tmp2, 0));
 9822 
 9823   psllq(xtmp2, 16);
 9824   pxor(xtmp1, xtmp2);
 9825 
 9826   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 9827   shrl(in_out, 24);
 9828   andl(in_out, 0x000000FF);
 9829   shll(in_out, 3);
 9830   addl(in_out, tmp3);
 9831   movq(xtmp2, Address(in_out, 0));
 9832 
 9833   psllq(xtmp2, 24);
 9834   pxor(xtmp1, xtmp2); // Result in CXMM
 9835   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 9836 }
 9837 
 9838 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 9839                                       Register in_out,
 9840                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 9841                                       XMMRegister w_xtmp2,
 9842                                       Register tmp1,
 9843                                       Register n_tmp2, Register n_tmp3) {
 9844   if (is_pclmulqdq_supported) {
 9845     movdl(w_xtmp1, in_out);
 9846 
 9847     movl(tmp1, const_or_pre_comp_const_index);
 9848     movdl(w_xtmp2, tmp1);
 9849     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 9850     // Keep result in XMM since GPR is 32 bit in length
 9851   } else {
 9852     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 9853   }
 9854 }
 9855 
 9856 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9857                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9858                                      Register tmp1, Register tmp2,
 9859                                      Register n_tmp3) {
 9860   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9861   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9862 
 9863   psllq(w_xtmp1, 1);
 9864   movdl(tmp1, w_xtmp1);
 9865   psrlq(w_xtmp1, 32);
 9866   movdl(in_out, w_xtmp1);
 9867 
 9868   xorl(tmp2, tmp2);
 9869   crc32(tmp2, tmp1, 4);
 9870   xorl(in_out, tmp2);
 9871 
 9872   psllq(w_xtmp2, 1);
 9873   movdl(tmp1, w_xtmp2);
 9874   psrlq(w_xtmp2, 32);
 9875   movdl(in1, w_xtmp2);
 9876 
 9877   xorl(tmp2, tmp2);
 9878   crc32(tmp2, tmp1, 4);
 9879   xorl(in1, tmp2);
 9880   xorl(in_out, in1);
 9881   xorl(in_out, in2);
 9882 }
 9883 
 9884 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9885                                        Register in_out1, Register in_out2, Register in_out3,
 9886                                        Register tmp1, Register tmp2, Register tmp3,
 9887                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9888                                        Register tmp4, Register tmp5,
 9889                                        Register n_tmp6) {
 9890   Label L_processPartitions;
 9891   Label L_processPartition;
 9892   Label L_exit;
 9893 
 9894   bind(L_processPartitions);
 9895   cmpl(in_out1, 3 * size);
 9896   jcc(Assembler::less, L_exit);
 9897     xorl(tmp1, tmp1);
 9898     xorl(tmp2, tmp2);
 9899     movl(tmp3, in_out2);
 9900     addl(tmp3, size);
 9901 
 9902     bind(L_processPartition);
 9903       crc32(in_out3, Address(in_out2, 0), 4);
 9904       crc32(tmp1, Address(in_out2, size), 4);
 9905       crc32(tmp2, Address(in_out2, size*2), 4);
 9906       crc32(in_out3, Address(in_out2, 0+4), 4);
 9907       crc32(tmp1, Address(in_out2, size+4), 4);
 9908       crc32(tmp2, Address(in_out2, size*2+4), 4);
 9909       addl(in_out2, 8);
 9910       cmpl(in_out2, tmp3);
 9911       jcc(Assembler::less, L_processPartition);
 9912 
 9913         push(tmp3);
 9914         push(in_out1);
 9915         push(in_out2);
 9916         tmp4 = tmp3;
 9917         tmp5 = in_out1;
 9918         n_tmp6 = in_out2;
 9919 
 9920       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9921             w_xtmp1, w_xtmp2, w_xtmp3,
 9922             tmp4, tmp5,
 9923             n_tmp6);
 9924 
 9925         pop(in_out2);
 9926         pop(in_out1);
 9927         pop(tmp3);
 9928 
 9929     addl(in_out2, 2 * size);
 9930     subl(in_out1, 3 * size);
 9931     jmp(L_processPartitions);
 9932 
 9933   bind(L_exit);
 9934 }
 9935 #endif //LP64
 9936 
 9937 #ifdef _LP64
 9938 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 9939 // Input: A buffer I of L bytes.
 9940 // Output: the CRC32C value of the buffer.
 9941 // Notations:
 9942 // Write L = 24N + r, with N = floor (L/24).
 9943 // r = L mod 24 (0 <= r < 24).
 9944 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 9945 // N quadwords, and R consists of r bytes.
 9946 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 9947 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 9948 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 9949 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 9950 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9951                                           Register tmp1, Register tmp2, Register tmp3,
 9952                                           Register tmp4, Register tmp5, Register tmp6,
 9953                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9954                                           bool is_pclmulqdq_supported) {
 9955   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9956   Label L_wordByWord;
 9957   Label L_byteByByteProlog;
 9958   Label L_byteByByte;
 9959   Label L_exit;
 9960 
 9961   if (is_pclmulqdq_supported ) {
 9962     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
 9963     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
 9964 
 9965     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
 9966     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
 9967 
 9968     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
 9969     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
 9970     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 9971   } else {
 9972     const_or_pre_comp_const_index[0] = 1;
 9973     const_or_pre_comp_const_index[1] = 0;
 9974 
 9975     const_or_pre_comp_const_index[2] = 3;
 9976     const_or_pre_comp_const_index[3] = 2;
 9977 
 9978     const_or_pre_comp_const_index[4] = 5;
 9979     const_or_pre_comp_const_index[5] = 4;
 9980    }
 9981   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9982                     in2, in1, in_out,
 9983                     tmp1, tmp2, tmp3,
 9984                     w_xtmp1, w_xtmp2, w_xtmp3,
 9985                     tmp4, tmp5,
 9986                     tmp6);
 9987   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9988                     in2, in1, in_out,
 9989                     tmp1, tmp2, tmp3,
 9990                     w_xtmp1, w_xtmp2, w_xtmp3,
 9991                     tmp4, tmp5,
 9992                     tmp6);
 9993   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9994                     in2, in1, in_out,
 9995                     tmp1, tmp2, tmp3,
 9996                     w_xtmp1, w_xtmp2, w_xtmp3,
 9997                     tmp4, tmp5,
 9998                     tmp6);
 9999   movl(tmp1, in2);
10000   andl(tmp1, 0x00000007);
10001   negl(tmp1);
10002   addl(tmp1, in2);
10003   addq(tmp1, in1);
10004 
10005   cmpq(in1, tmp1);
10006   jccb(Assembler::greaterEqual, L_byteByByteProlog);
10007   align(16);
10008   BIND(L_wordByWord);
10009     crc32(in_out, Address(in1, 0), 8);
10010     addq(in1, 8);
10011     cmpq(in1, tmp1);
10012     jcc(Assembler::less, L_wordByWord);
10013 
10014   BIND(L_byteByByteProlog);
10015   andl(in2, 0x00000007);
10016   movl(tmp2, 1);
10017 
10018   cmpl(tmp2, in2);
10019   jccb(Assembler::greater, L_exit);
10020   BIND(L_byteByByte);
10021     crc32(in_out, Address(in1, 0), 1);
10022     incq(in1);
10023     incl(tmp2);
10024     cmpl(tmp2, in2);
10025     jcc(Assembler::lessEqual, L_byteByByte);
10026 
10027   BIND(L_exit);
10028 }
10029 #else
10030 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10031                                           Register tmp1, Register  tmp2, Register tmp3,
10032                                           Register tmp4, Register  tmp5, Register tmp6,
10033                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10034                                           bool is_pclmulqdq_supported) {
10035   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10036   Label L_wordByWord;
10037   Label L_byteByByteProlog;
10038   Label L_byteByByte;
10039   Label L_exit;
10040 
10041   if (is_pclmulqdq_supported) {
10042     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
10043     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
10044 
10045     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
10046     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
10047 
10048     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
10049     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
10050   } else {
10051     const_or_pre_comp_const_index[0] = 1;
10052     const_or_pre_comp_const_index[1] = 0;
10053 
10054     const_or_pre_comp_const_index[2] = 3;
10055     const_or_pre_comp_const_index[3] = 2;
10056 
10057     const_or_pre_comp_const_index[4] = 5;
10058     const_or_pre_comp_const_index[5] = 4;
10059   }
10060   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10061                     in2, in1, in_out,
10062                     tmp1, tmp2, tmp3,
10063                     w_xtmp1, w_xtmp2, w_xtmp3,
10064                     tmp4, tmp5,
10065                     tmp6);
10066   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10067                     in2, in1, in_out,
10068                     tmp1, tmp2, tmp3,
10069                     w_xtmp1, w_xtmp2, w_xtmp3,
10070                     tmp4, tmp5,
10071                     tmp6);
10072   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10073                     in2, in1, in_out,
10074                     tmp1, tmp2, tmp3,
10075                     w_xtmp1, w_xtmp2, w_xtmp3,
10076                     tmp4, tmp5,
10077                     tmp6);
10078   movl(tmp1, in2);
10079   andl(tmp1, 0x00000007);
10080   negl(tmp1);
10081   addl(tmp1, in2);
10082   addl(tmp1, in1);
10083 
10084   BIND(L_wordByWord);
10085   cmpl(in1, tmp1);
10086   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10087     crc32(in_out, Address(in1,0), 4);
10088     addl(in1, 4);
10089     jmp(L_wordByWord);
10090 
10091   BIND(L_byteByByteProlog);
10092   andl(in2, 0x00000007);
10093   movl(tmp2, 1);
10094 
10095   BIND(L_byteByByte);
10096   cmpl(tmp2, in2);
10097   jccb(Assembler::greater, L_exit);
10098     movb(tmp1, Address(in1, 0));
10099     crc32(in_out, tmp1, 1);
10100     incl(in1);
10101     incl(tmp2);
10102     jmp(L_byteByByte);
10103 
10104   BIND(L_exit);
10105 }
10106 #endif // LP64
10107 #undef BIND
10108 #undef BLOCK_COMMENT
10109 
10110 // Compress char[] array to byte[].
10111 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
10112 // Return the array length if every element in array can be encoded,
10113 // otherwise, the index of first non-latin1 (> 0xff) character.
10114 //   @IntrinsicCandidate
10115 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10116 //     for (int i = 0; i < len; i++) {
10117 //       char c = src[srcOff];
10118 //       if (c > 0xff) {
10119 //           return i;  // return index of non-latin1 char
10120 //       }
10121 //       dst[dstOff] = (byte)c;
10122 //       srcOff++;
10123 //       dstOff++;
10124 //     }
10125 //     return len;
10126 //   }
10127 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10128   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10129   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10130   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
10131   Label copy_chars_loop, done, reset_sp, copy_tail;
10132 
10133   // rsi: src
10134   // rdi: dst
10135   // rdx: len
10136   // rcx: tmp5
10137   // rax: result
10138 
10139   // rsi holds start addr of source char[] to be compressed
10140   // rdi holds start addr of destination byte[]
10141   // rdx holds length
10142 
10143   assert(len != result, "");
10144 
10145   // save length for return
10146   movl(result, len);
10147 
10148   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
10149     VM_Version::supports_avx512vlbw() &&
10150     VM_Version::supports_bmi2()) {
10151 
10152     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
10153 
10154     // alignment
10155     Label post_alignment;
10156 
10157     // if length of the string is less than 32, handle it the old fashioned way
10158     testl(len, -32);
10159     jcc(Assembler::zero, below_threshold);
10160 
10161     // First check whether a character is compressible ( <= 0xFF).
10162     // Create mask to test for Unicode chars inside zmm vector
10163     movl(tmp5, 0x00FF);
10164     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
10165 
10166     testl(len, -64);
10167     jccb(Assembler::zero, post_alignment);
10168 
10169     movl(tmp5, dst);
10170     andl(tmp5, (32 - 1));
10171     negl(tmp5);
10172     andl(tmp5, (32 - 1));
10173 
10174     // bail out when there is nothing to be done
10175     testl(tmp5, 0xFFFFFFFF);
10176     jccb(Assembler::zero, post_alignment);
10177 
10178     // ~(~0 << len), where len is the # of remaining elements to process
10179     movl(len, 0xFFFFFFFF);
10180     shlxl(len, len, tmp5);
10181     notl(len);
10182     kmovdl(mask2, len);
10183     movl(len, result);
10184 
10185     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
10186     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
10187     ktestd(mask1, mask2);
10188     jcc(Assembler::carryClear, copy_tail);
10189 
10190     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
10191 
10192     addptr(src, tmp5);
10193     addptr(src, tmp5);
10194     addptr(dst, tmp5);
10195     subl(len, tmp5);
10196 
10197     bind(post_alignment);
10198     // end of alignment
10199 
10200     movl(tmp5, len);
10201     andl(tmp5, (32 - 1));    // tail count (in chars)
10202     andl(len, ~(32 - 1));    // vector count (in chars)
10203     jccb(Assembler::zero, copy_loop_tail);
10204 
10205     lea(src, Address(src, len, Address::times_2));
10206     lea(dst, Address(dst, len, Address::times_1));
10207     negptr(len);
10208 
10209     bind(copy_32_loop);
10210     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10211     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10212     kortestdl(mask1, mask1);
10213     jccb(Assembler::carryClear, reset_for_copy_tail);
10214 
10215     // All elements in current processed chunk are valid candidates for
10216     // compression. Write a truncated byte elements to the memory.
10217     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10218     addptr(len, 32);
10219     jccb(Assembler::notZero, copy_32_loop);
10220 
10221     bind(copy_loop_tail);
10222     // bail out when there is nothing to be done
10223     testl(tmp5, 0xFFFFFFFF);
10224     jcc(Assembler::zero, done);
10225 
10226     movl(len, tmp5);
10227 
10228     // ~(~0 << len), where len is the # of remaining elements to process
10229     movl(tmp5, 0xFFFFFFFF);
10230     shlxl(tmp5, tmp5, len);
10231     notl(tmp5);
10232 
10233     kmovdl(mask2, tmp5);
10234 
10235     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
10236     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
10237     ktestd(mask1, mask2);
10238     jcc(Assembler::carryClear, copy_tail);
10239 
10240     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
10241     jmp(done);
10242 
10243     bind(reset_for_copy_tail);
10244     lea(src, Address(src, tmp5, Address::times_2));
10245     lea(dst, Address(dst, tmp5, Address::times_1));
10246     subptr(len, tmp5);
10247     jmp(copy_chars_loop);
10248 
10249     bind(below_threshold);
10250   }
10251 
10252   if (UseSSE42Intrinsics) {
10253     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
10254 
10255     // vectored compression
10256     testl(len, 0xfffffff8);
10257     jcc(Assembler::zero, copy_tail);
10258 
10259     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10260     movdl(tmp1Reg, tmp5);
10261     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10262 
10263     andl(len, 0xfffffff0);
10264     jccb(Assembler::zero, copy_16);
10265 
10266     // compress 16 chars per iter
10267     pxor(tmp4Reg, tmp4Reg);
10268 
10269     lea(src, Address(src, len, Address::times_2));
10270     lea(dst, Address(dst, len, Address::times_1));
10271     negptr(len);
10272 
10273     bind(copy_32_loop);
10274     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10275     por(tmp4Reg, tmp2Reg);
10276     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10277     por(tmp4Reg, tmp3Reg);
10278     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10279     jccb(Assembler::notZero, reset_for_copy_tail);
10280     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10281     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10282     addptr(len, 16);
10283     jccb(Assembler::notZero, copy_32_loop);
10284 
10285     // compress next vector of 8 chars (if any)
10286     bind(copy_16);
10287     // len = 0
10288     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
10289     jccb(Assembler::zero, copy_tail_sse);
10290 
10291     pxor(tmp3Reg, tmp3Reg);
10292 
10293     movdqu(tmp2Reg, Address(src, 0));
10294     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
10295     jccb(Assembler::notZero, reset_for_copy_tail);
10296     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
10297     movq(Address(dst, 0), tmp2Reg);
10298     addptr(src, 16);
10299     addptr(dst, 8);
10300     jmpb(copy_tail_sse);
10301 
10302     bind(reset_for_copy_tail);
10303     movl(tmp5, result);
10304     andl(tmp5, 0x0000000f);
10305     lea(src, Address(src, tmp5, Address::times_2));
10306     lea(dst, Address(dst, tmp5, Address::times_1));
10307     subptr(len, tmp5);
10308     jmpb(copy_chars_loop);
10309 
10310     bind(copy_tail_sse);
10311     movl(len, result);
10312     andl(len, 0x00000007);    // tail count (in chars)
10313   }
10314   // compress 1 char per iter
10315   bind(copy_tail);
10316   testl(len, len);
10317   jccb(Assembler::zero, done);
10318   lea(src, Address(src, len, Address::times_2));
10319   lea(dst, Address(dst, len, Address::times_1));
10320   negptr(len);
10321 
10322   bind(copy_chars_loop);
10323   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
10324   testl(tmp5, 0xff00);      // check if Unicode char
10325   jccb(Assembler::notZero, reset_sp);
10326   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
10327   increment(len);
10328   jccb(Assembler::notZero, copy_chars_loop);
10329 
10330   // add len then return (len will be zero if compress succeeded, otherwise negative)
10331   bind(reset_sp);
10332   addl(result, len);
10333 
10334   bind(done);
10335 }
10336 
10337 // Inflate byte[] array to char[].
10338 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
10339 //   @IntrinsicCandidate
10340 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
10341 //     for (int i = 0; i < len; i++) {
10342 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
10343 //     }
10344 //   }
10345 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
10346   XMMRegister tmp1, Register tmp2, KRegister mask) {
10347   Label copy_chars_loop, done, below_threshold, avx3_threshold;
10348   // rsi: src
10349   // rdi: dst
10350   // rdx: len
10351   // rcx: tmp2
10352 
10353   // rsi holds start addr of source byte[] to be inflated
10354   // rdi holds start addr of destination char[]
10355   // rdx holds length
10356   assert_different_registers(src, dst, len, tmp2);
10357   movl(tmp2, len);
10358   if ((UseAVX > 2) && // AVX512
10359     VM_Version::supports_avx512vlbw() &&
10360     VM_Version::supports_bmi2()) {
10361 
10362     Label copy_32_loop, copy_tail;
10363     Register tmp3_aliased = len;
10364 
10365     // if length of the string is less than 16, handle it in an old fashioned way
10366     testl(len, -16);
10367     jcc(Assembler::zero, below_threshold);
10368 
10369     testl(len, -1 * AVX3Threshold);
10370     jcc(Assembler::zero, avx3_threshold);
10371 
10372     // In order to use only one arithmetic operation for the main loop we use
10373     // this pre-calculation
10374     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
10375     andl(len, -32);     // vector count
10376     jccb(Assembler::zero, copy_tail);
10377 
10378     lea(src, Address(src, len, Address::times_1));
10379     lea(dst, Address(dst, len, Address::times_2));
10380     negptr(len);
10381 
10382 
10383     // inflate 32 chars per iter
10384     bind(copy_32_loop);
10385     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
10386     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
10387     addptr(len, 32);
10388     jcc(Assembler::notZero, copy_32_loop);
10389 
10390     bind(copy_tail);
10391     // bail out when there is nothing to be done
10392     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
10393     jcc(Assembler::zero, done);
10394 
10395     // ~(~0 << length), where length is the # of remaining elements to process
10396     movl(tmp3_aliased, -1);
10397     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
10398     notl(tmp3_aliased);
10399     kmovdl(mask, tmp3_aliased);
10400     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
10401     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
10402 
10403     jmp(done);
10404     bind(avx3_threshold);
10405   }
10406   if (UseSSE42Intrinsics) {
10407     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
10408 
10409     if (UseAVX > 1) {
10410       andl(tmp2, (16 - 1));
10411       andl(len, -16);
10412       jccb(Assembler::zero, copy_new_tail);
10413     } else {
10414       andl(tmp2, 0x00000007);   // tail count (in chars)
10415       andl(len, 0xfffffff8);    // vector count (in chars)
10416       jccb(Assembler::zero, copy_tail);
10417     }
10418 
10419     // vectored inflation
10420     lea(src, Address(src, len, Address::times_1));
10421     lea(dst, Address(dst, len, Address::times_2));
10422     negptr(len);
10423 
10424     if (UseAVX > 1) {
10425       bind(copy_16_loop);
10426       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
10427       vmovdqu(Address(dst, len, Address::times_2), tmp1);
10428       addptr(len, 16);
10429       jcc(Assembler::notZero, copy_16_loop);
10430 
10431       bind(below_threshold);
10432       bind(copy_new_tail);
10433       movl(len, tmp2);
10434       andl(tmp2, 0x00000007);
10435       andl(len, 0xFFFFFFF8);
10436       jccb(Assembler::zero, copy_tail);
10437 
10438       pmovzxbw(tmp1, Address(src, 0));
10439       movdqu(Address(dst, 0), tmp1);
10440       addptr(src, 8);
10441       addptr(dst, 2 * 8);
10442 
10443       jmp(copy_tail, true);
10444     }
10445 
10446     // inflate 8 chars per iter
10447     bind(copy_8_loop);
10448     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
10449     movdqu(Address(dst, len, Address::times_2), tmp1);
10450     addptr(len, 8);
10451     jcc(Assembler::notZero, copy_8_loop);
10452 
10453     bind(copy_tail);
10454     movl(len, tmp2);
10455 
10456     cmpl(len, 4);
10457     jccb(Assembler::less, copy_bytes);
10458 
10459     movdl(tmp1, Address(src, 0));  // load 4 byte chars
10460     pmovzxbw(tmp1, tmp1);
10461     movq(Address(dst, 0), tmp1);
10462     subptr(len, 4);
10463     addptr(src, 4);
10464     addptr(dst, 8);
10465 
10466     bind(copy_bytes);
10467   } else {
10468     bind(below_threshold);
10469   }
10470 
10471   testl(len, len);
10472   jccb(Assembler::zero, done);
10473   lea(src, Address(src, len, Address::times_1));
10474   lea(dst, Address(dst, len, Address::times_2));
10475   negptr(len);
10476 
10477   // inflate 1 char per iter
10478   bind(copy_chars_loop);
10479   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
10480   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
10481   increment(len);
10482   jcc(Assembler::notZero, copy_chars_loop);
10483 
10484   bind(done);
10485 }
10486 
10487 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
10488   switch(type) {
10489     case T_BYTE:
10490     case T_BOOLEAN:
10491       evmovdqub(dst, kmask, src, merge, vector_len);
10492       break;
10493     case T_CHAR:
10494     case T_SHORT:
10495       evmovdquw(dst, kmask, src, merge, vector_len);
10496       break;
10497     case T_INT:
10498     case T_FLOAT:
10499       evmovdqul(dst, kmask, src, merge, vector_len);
10500       break;
10501     case T_LONG:
10502     case T_DOUBLE:
10503       evmovdquq(dst, kmask, src, merge, vector_len);
10504       break;
10505     default:
10506       fatal("Unexpected type argument %s", type2name(type));
10507       break;
10508   }
10509 }
10510 
10511 
10512 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
10513   switch(type) {
10514     case T_BYTE:
10515     case T_BOOLEAN:
10516       evmovdqub(dst, kmask, src, merge, vector_len);
10517       break;
10518     case T_CHAR:
10519     case T_SHORT:
10520       evmovdquw(dst, kmask, src, merge, vector_len);
10521       break;
10522     case T_INT:
10523     case T_FLOAT:
10524       evmovdqul(dst, kmask, src, merge, vector_len);
10525       break;
10526     case T_LONG:
10527     case T_DOUBLE:
10528       evmovdquq(dst, kmask, src, merge, vector_len);
10529       break;
10530     default:
10531       fatal("Unexpected type argument %s", type2name(type));
10532       break;
10533   }
10534 }
10535 
10536 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
10537   switch(type) {
10538     case T_BYTE:
10539     case T_BOOLEAN:
10540       evmovdqub(dst, kmask, src, merge, vector_len);
10541       break;
10542     case T_CHAR:
10543     case T_SHORT:
10544       evmovdquw(dst, kmask, src, merge, vector_len);
10545       break;
10546     case T_INT:
10547     case T_FLOAT:
10548       evmovdqul(dst, kmask, src, merge, vector_len);
10549       break;
10550     case T_LONG:
10551     case T_DOUBLE:
10552       evmovdquq(dst, kmask, src, merge, vector_len);
10553       break;
10554     default:
10555       fatal("Unexpected type argument %s", type2name(type));
10556       break;
10557   }
10558 }
10559 
10560 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
10561   switch(masklen) {
10562     case 2:
10563        knotbl(dst, src);
10564        movl(rtmp, 3);
10565        kmovbl(ktmp, rtmp);
10566        kandbl(dst, ktmp, dst);
10567        break;
10568     case 4:
10569        knotbl(dst, src);
10570        movl(rtmp, 15);
10571        kmovbl(ktmp, rtmp);
10572        kandbl(dst, ktmp, dst);
10573        break;
10574     case 8:
10575        knotbl(dst, src);
10576        break;
10577     case 16:
10578        knotwl(dst, src);
10579        break;
10580     case 32:
10581        knotdl(dst, src);
10582        break;
10583     case 64:
10584        knotql(dst, src);
10585        break;
10586     default:
10587       fatal("Unexpected vector length %d", masklen);
10588       break;
10589   }
10590 }
10591 
10592 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
10593   switch(type) {
10594     case T_BOOLEAN:
10595     case T_BYTE:
10596        kandbl(dst, src1, src2);
10597        break;
10598     case T_CHAR:
10599     case T_SHORT:
10600        kandwl(dst, src1, src2);
10601        break;
10602     case T_INT:
10603     case T_FLOAT:
10604        kanddl(dst, src1, src2);
10605        break;
10606     case T_LONG:
10607     case T_DOUBLE:
10608        kandql(dst, src1, src2);
10609        break;
10610     default:
10611       fatal("Unexpected type argument %s", type2name(type));
10612       break;
10613   }
10614 }
10615 
10616 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
10617   switch(type) {
10618     case T_BOOLEAN:
10619     case T_BYTE:
10620        korbl(dst, src1, src2);
10621        break;
10622     case T_CHAR:
10623     case T_SHORT:
10624        korwl(dst, src1, src2);
10625        break;
10626     case T_INT:
10627     case T_FLOAT:
10628        kordl(dst, src1, src2);
10629        break;
10630     case T_LONG:
10631     case T_DOUBLE:
10632        korql(dst, src1, src2);
10633        break;
10634     default:
10635       fatal("Unexpected type argument %s", type2name(type));
10636       break;
10637   }
10638 }
10639 
10640 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
10641   switch(type) {
10642     case T_BOOLEAN:
10643     case T_BYTE:
10644        kxorbl(dst, src1, src2);
10645        break;
10646     case T_CHAR:
10647     case T_SHORT:
10648        kxorwl(dst, src1, src2);
10649        break;
10650     case T_INT:
10651     case T_FLOAT:
10652        kxordl(dst, src1, src2);
10653        break;
10654     case T_LONG:
10655     case T_DOUBLE:
10656        kxorql(dst, src1, src2);
10657        break;
10658     default:
10659       fatal("Unexpected type argument %s", type2name(type));
10660       break;
10661   }
10662 }
10663 
10664 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10665   switch(type) {
10666     case T_BOOLEAN:
10667     case T_BYTE:
10668       evpermb(dst, mask, nds, src, merge, vector_len); break;
10669     case T_CHAR:
10670     case T_SHORT:
10671       evpermw(dst, mask, nds, src, merge, vector_len); break;
10672     case T_INT:
10673     case T_FLOAT:
10674       evpermd(dst, mask, nds, src, merge, vector_len); break;
10675     case T_LONG:
10676     case T_DOUBLE:
10677       evpermq(dst, mask, nds, src, merge, vector_len); break;
10678     default:
10679       fatal("Unexpected type argument %s", type2name(type)); break;
10680   }
10681 }
10682 
10683 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10684   switch(type) {
10685     case T_BOOLEAN:
10686     case T_BYTE:
10687       evpermb(dst, mask, nds, src, merge, vector_len); break;
10688     case T_CHAR:
10689     case T_SHORT:
10690       evpermw(dst, mask, nds, src, merge, vector_len); break;
10691     case T_INT:
10692     case T_FLOAT:
10693       evpermd(dst, mask, nds, src, merge, vector_len); break;
10694     case T_LONG:
10695     case T_DOUBLE:
10696       evpermq(dst, mask, nds, src, merge, vector_len); break;
10697     default:
10698       fatal("Unexpected type argument %s", type2name(type)); break;
10699   }
10700 }
10701 
10702 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10703   switch(type) {
10704     case T_BYTE:
10705       evpminub(dst, mask, nds, src, merge, vector_len); break;
10706     case T_SHORT:
10707       evpminuw(dst, mask, nds, src, merge, vector_len); break;
10708     case T_INT:
10709       evpminud(dst, mask, nds, src, merge, vector_len); break;
10710     case T_LONG:
10711       evpminuq(dst, mask, nds, src, merge, vector_len); break;
10712     default:
10713       fatal("Unexpected type argument %s", type2name(type)); break;
10714   }
10715 }
10716 
10717 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10718   switch(type) {
10719     case T_BYTE:
10720       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
10721     case T_SHORT:
10722       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
10723     case T_INT:
10724       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
10725     case T_LONG:
10726       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
10727     default:
10728       fatal("Unexpected type argument %s", type2name(type)); break;
10729   }
10730 }
10731 
10732 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10733   switch(type) {
10734     case T_BYTE:
10735       evpminub(dst, mask, nds, src, merge, vector_len); break;
10736     case T_SHORT:
10737       evpminuw(dst, mask, nds, src, merge, vector_len); break;
10738     case T_INT:
10739       evpminud(dst, mask, nds, src, merge, vector_len); break;
10740     case T_LONG:
10741       evpminuq(dst, mask, nds, src, merge, vector_len); break;
10742     default:
10743       fatal("Unexpected type argument %s", type2name(type)); break;
10744   }
10745 }
10746 
10747 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10748   switch(type) {
10749     case T_BYTE:
10750       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
10751     case T_SHORT:
10752       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
10753     case T_INT:
10754       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
10755     case T_LONG:
10756       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
10757     default:
10758       fatal("Unexpected type argument %s", type2name(type)); break;
10759   }
10760 }
10761 
10762 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10763   switch(type) {
10764     case T_BYTE:
10765       evpminsb(dst, mask, nds, src, merge, vector_len); break;
10766     case T_SHORT:
10767       evpminsw(dst, mask, nds, src, merge, vector_len); break;
10768     case T_INT:
10769       evpminsd(dst, mask, nds, src, merge, vector_len); break;
10770     case T_LONG:
10771       evpminsq(dst, mask, nds, src, merge, vector_len); break;
10772     default:
10773       fatal("Unexpected type argument %s", type2name(type)); break;
10774   }
10775 }
10776 
10777 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10778   switch(type) {
10779     case T_BYTE:
10780       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
10781     case T_SHORT:
10782       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
10783     case T_INT:
10784       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
10785     case T_LONG:
10786       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
10787     default:
10788       fatal("Unexpected type argument %s", type2name(type)); break;
10789   }
10790 }
10791 
10792 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10793   switch(type) {
10794     case T_BYTE:
10795       evpminsb(dst, mask, nds, src, merge, vector_len); break;
10796     case T_SHORT:
10797       evpminsw(dst, mask, nds, src, merge, vector_len); break;
10798     case T_INT:
10799       evpminsd(dst, mask, nds, src, merge, vector_len); break;
10800     case T_LONG:
10801       evpminsq(dst, mask, nds, src, merge, vector_len); break;
10802     default:
10803       fatal("Unexpected type argument %s", type2name(type)); break;
10804   }
10805 }
10806 
10807 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10808   switch(type) {
10809     case T_BYTE:
10810       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
10811     case T_SHORT:
10812       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
10813     case T_INT:
10814       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
10815     case T_LONG:
10816       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
10817     default:
10818       fatal("Unexpected type argument %s", type2name(type)); break;
10819   }
10820 }
10821 
10822 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10823   switch(type) {
10824     case T_INT:
10825       evpxord(dst, mask, nds, src, merge, vector_len); break;
10826     case T_LONG:
10827       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10828     default:
10829       fatal("Unexpected type argument %s", type2name(type)); break;
10830   }
10831 }
10832 
10833 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10834   switch(type) {
10835     case T_INT:
10836       evpxord(dst, mask, nds, src, merge, vector_len); break;
10837     case T_LONG:
10838       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10839     default:
10840       fatal("Unexpected type argument %s", type2name(type)); break;
10841   }
10842 }
10843 
10844 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10845   switch(type) {
10846     case T_INT:
10847       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10848     case T_LONG:
10849       evporq(dst, mask, nds, src, merge, vector_len); break;
10850     default:
10851       fatal("Unexpected type argument %s", type2name(type)); break;
10852   }
10853 }
10854 
10855 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10856   switch(type) {
10857     case T_INT:
10858       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10859     case T_LONG:
10860       evporq(dst, mask, nds, src, merge, vector_len); break;
10861     default:
10862       fatal("Unexpected type argument %s", type2name(type)); break;
10863   }
10864 }
10865 
10866 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10867   switch(type) {
10868     case T_INT:
10869       evpandd(dst, mask, nds, src, merge, vector_len); break;
10870     case T_LONG:
10871       evpandq(dst, mask, nds, src, merge, vector_len); break;
10872     default:
10873       fatal("Unexpected type argument %s", type2name(type)); break;
10874   }
10875 }
10876 
10877 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10878   switch(type) {
10879     case T_INT:
10880       evpandd(dst, mask, nds, src, merge, vector_len); break;
10881     case T_LONG:
10882       evpandq(dst, mask, nds, src, merge, vector_len); break;
10883     default:
10884       fatal("Unexpected type argument %s", type2name(type)); break;
10885   }
10886 }
10887 
10888 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
10889   switch(masklen) {
10890     case 8:
10891        kortestbl(src1, src2);
10892        break;
10893     case 16:
10894        kortestwl(src1, src2);
10895        break;
10896     case 32:
10897        kortestdl(src1, src2);
10898        break;
10899     case 64:
10900        kortestql(src1, src2);
10901        break;
10902     default:
10903       fatal("Unexpected mask length %d", masklen);
10904       break;
10905   }
10906 }
10907 
10908 
10909 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
10910   switch(masklen)  {
10911     case 8:
10912        ktestbl(src1, src2);
10913        break;
10914     case 16:
10915        ktestwl(src1, src2);
10916        break;
10917     case 32:
10918        ktestdl(src1, src2);
10919        break;
10920     case 64:
10921        ktestql(src1, src2);
10922        break;
10923     default:
10924       fatal("Unexpected mask length %d", masklen);
10925       break;
10926   }
10927 }
10928 
10929 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10930   switch(type) {
10931     case T_INT:
10932       evprold(dst, mask, src, shift, merge, vlen_enc); break;
10933     case T_LONG:
10934       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
10935     default:
10936       fatal("Unexpected type argument %s", type2name(type)); break;
10937       break;
10938   }
10939 }
10940 
10941 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10942   switch(type) {
10943     case T_INT:
10944       evprord(dst, mask, src, shift, merge, vlen_enc); break;
10945     case T_LONG:
10946       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
10947     default:
10948       fatal("Unexpected type argument %s", type2name(type)); break;
10949   }
10950 }
10951 
10952 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10953   switch(type) {
10954     case T_INT:
10955       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
10956     case T_LONG:
10957       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
10958     default:
10959       fatal("Unexpected type argument %s", type2name(type)); break;
10960   }
10961 }
10962 
10963 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10964   switch(type) {
10965     case T_INT:
10966       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
10967     case T_LONG:
10968       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
10969     default:
10970       fatal("Unexpected type argument %s", type2name(type)); break;
10971   }
10972 }
10973 
10974 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10975   assert(rscratch != noreg || always_reachable(src), "missing");
10976 
10977   if (reachable(src)) {
10978     evpandq(dst, nds, as_Address(src), vector_len);
10979   } else {
10980     lea(rscratch, src);
10981     evpandq(dst, nds, Address(rscratch, 0), vector_len);
10982   }
10983 }
10984 
10985 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
10986   assert(rscratch != noreg || always_reachable(src), "missing");
10987 
10988   if (reachable(src)) {
10989     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
10990   } else {
10991     lea(rscratch, src);
10992     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
10993   }
10994 }
10995 
10996 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10997   assert(rscratch != noreg || always_reachable(src), "missing");
10998 
10999   if (reachable(src)) {
11000     evporq(dst, nds, as_Address(src), vector_len);
11001   } else {
11002     lea(rscratch, src);
11003     evporq(dst, nds, Address(rscratch, 0), vector_len);
11004   }
11005 }
11006 
11007 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
11008   assert(rscratch != noreg || always_reachable(src), "missing");
11009 
11010   if (reachable(src)) {
11011     vpshufb(dst, nds, as_Address(src), vector_len);
11012   } else {
11013     lea(rscratch, src);
11014     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
11015   }
11016 }
11017 
11018 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
11019   assert(rscratch != noreg || always_reachable(src), "missing");
11020 
11021   if (reachable(src)) {
11022     Assembler::vpor(dst, nds, as_Address(src), vector_len);
11023   } else {
11024     lea(rscratch, src);
11025     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
11026   }
11027 }
11028 
11029 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
11030   assert(rscratch != noreg || always_reachable(src3), "missing");
11031 
11032   if (reachable(src3)) {
11033     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
11034   } else {
11035     lea(rscratch, src3);
11036     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
11037   }
11038 }
11039 
11040 #if COMPILER2_OR_JVMCI
11041 
11042 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
11043                                  Register length, Register temp, int vec_enc) {
11044   // Computing mask for predicated vector store.
11045   movptr(temp, -1);
11046   bzhiq(temp, temp, length);
11047   kmov(mask, temp);
11048   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
11049 }
11050 
11051 // Set memory operation for length "less than" 64 bytes.
11052 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
11053                                        XMMRegister xmm, KRegister mask, Register length,
11054                                        Register temp, bool use64byteVector) {
11055   assert(MaxVectorSize >= 32, "vector length should be >= 32");
11056   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
11057   if (!use64byteVector) {
11058     fill32(dst, disp, xmm);
11059     subptr(length, 32 >> shift);
11060     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
11061   } else {
11062     assert(MaxVectorSize == 64, "vector length != 64");
11063     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
11064   }
11065 }
11066 
11067 
11068 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
11069                                        XMMRegister xmm, KRegister mask, Register length,
11070                                        Register temp) {
11071   assert(MaxVectorSize >= 32, "vector length should be >= 32");
11072   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
11073   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
11074 }
11075 
11076 
11077 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
11078   assert(MaxVectorSize >= 32, "vector length should be >= 32");
11079   vmovdqu(dst, xmm);
11080 }
11081 
11082 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
11083   fill32(Address(dst, disp), xmm);
11084 }
11085 
11086 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
11087   assert(MaxVectorSize >= 32, "vector length should be >= 32");
11088   if (!use64byteVector) {
11089     fill32(dst, xmm);
11090     fill32(dst.plus_disp(32), xmm);
11091   } else {
11092     evmovdquq(dst, xmm, Assembler::AVX_512bit);
11093   }
11094 }
11095 
11096 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
11097   fill64(Address(dst, disp), xmm, use64byteVector);
11098 }
11099 
11100 #ifdef _LP64
11101 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
11102                                         Register count, Register rtmp, XMMRegister xtmp) {
11103   Label L_exit;
11104   Label L_fill_start;
11105   Label L_fill_64_bytes;
11106   Label L_fill_96_bytes;
11107   Label L_fill_128_bytes;
11108   Label L_fill_128_bytes_loop;
11109   Label L_fill_128_loop_header;
11110   Label L_fill_128_bytes_loop_header;
11111   Label L_fill_128_bytes_loop_pre_header;
11112   Label L_fill_zmm_sequence;
11113 
11114   int shift = -1;
11115   int avx3threshold = VM_Version::avx3_threshold();
11116   switch(type) {
11117     case T_BYTE:  shift = 0;
11118       break;
11119     case T_SHORT: shift = 1;
11120       break;
11121     case T_INT:   shift = 2;
11122       break;
11123     /* Uncomment when LONG fill stubs are supported.
11124     case T_LONG:  shift = 3;
11125       break;
11126     */
11127     default:
11128       fatal("Unhandled type: %s\n", type2name(type));
11129   }
11130 
11131   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
11132 
11133     if (MaxVectorSize == 64) {
11134       cmpq(count, avx3threshold >> shift);
11135       jcc(Assembler::greater, L_fill_zmm_sequence);
11136     }
11137 
11138     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
11139 
11140     bind(L_fill_start);
11141 
11142     cmpq(count, 32 >> shift);
11143     jccb(Assembler::greater, L_fill_64_bytes);
11144     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
11145     jmp(L_exit);
11146 
11147     bind(L_fill_64_bytes);
11148     cmpq(count, 64 >> shift);
11149     jccb(Assembler::greater, L_fill_96_bytes);
11150     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
11151     jmp(L_exit);
11152 
11153     bind(L_fill_96_bytes);
11154     cmpq(count, 96 >> shift);
11155     jccb(Assembler::greater, L_fill_128_bytes);
11156     fill64(to, 0, xtmp);
11157     subq(count, 64 >> shift);
11158     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
11159     jmp(L_exit);
11160 
11161     bind(L_fill_128_bytes);
11162     cmpq(count, 128 >> shift);
11163     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
11164     fill64(to, 0, xtmp);
11165     fill32(to, 64, xtmp);
11166     subq(count, 96 >> shift);
11167     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
11168     jmp(L_exit);
11169 
11170     bind(L_fill_128_bytes_loop_pre_header);
11171     {
11172       mov(rtmp, to);
11173       andq(rtmp, 31);
11174       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
11175       negq(rtmp);
11176       addq(rtmp, 32);
11177       mov64(r8, -1L);
11178       bzhiq(r8, r8, rtmp);
11179       kmovql(k2, r8);
11180       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
11181       addq(to, rtmp);
11182       shrq(rtmp, shift);
11183       subq(count, rtmp);
11184     }
11185 
11186     cmpq(count, 128 >> shift);
11187     jcc(Assembler::less, L_fill_start);
11188 
11189     bind(L_fill_128_bytes_loop_header);
11190     subq(count, 128 >> shift);
11191 
11192     align32();
11193     bind(L_fill_128_bytes_loop);
11194       fill64(to, 0, xtmp);
11195       fill64(to, 64, xtmp);
11196       addq(to, 128);
11197       subq(count, 128 >> shift);
11198       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
11199 
11200     addq(count, 128 >> shift);
11201     jcc(Assembler::zero, L_exit);
11202     jmp(L_fill_start);
11203   }
11204 
11205   if (MaxVectorSize == 64) {
11206     // Sequence using 64 byte ZMM register.
11207     Label L_fill_128_bytes_zmm;
11208     Label L_fill_192_bytes_zmm;
11209     Label L_fill_192_bytes_loop_zmm;
11210     Label L_fill_192_bytes_loop_header_zmm;
11211     Label L_fill_192_bytes_loop_pre_header_zmm;
11212     Label L_fill_start_zmm_sequence;
11213 
11214     bind(L_fill_zmm_sequence);
11215     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
11216 
11217     bind(L_fill_start_zmm_sequence);
11218     cmpq(count, 64 >> shift);
11219     jccb(Assembler::greater, L_fill_128_bytes_zmm);
11220     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
11221     jmp(L_exit);
11222 
11223     bind(L_fill_128_bytes_zmm);
11224     cmpq(count, 128 >> shift);
11225     jccb(Assembler::greater, L_fill_192_bytes_zmm);
11226     fill64(to, 0, xtmp, true);
11227     subq(count, 64 >> shift);
11228     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
11229     jmp(L_exit);
11230 
11231     bind(L_fill_192_bytes_zmm);
11232     cmpq(count, 192 >> shift);
11233     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
11234     fill64(to, 0, xtmp, true);
11235     fill64(to, 64, xtmp, true);
11236     subq(count, 128 >> shift);
11237     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
11238     jmp(L_exit);
11239 
11240     bind(L_fill_192_bytes_loop_pre_header_zmm);
11241     {
11242       movq(rtmp, to);
11243       andq(rtmp, 63);
11244       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
11245       negq(rtmp);
11246       addq(rtmp, 64);
11247       mov64(r8, -1L);
11248       bzhiq(r8, r8, rtmp);
11249       kmovql(k2, r8);
11250       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
11251       addq(to, rtmp);
11252       shrq(rtmp, shift);
11253       subq(count, rtmp);
11254     }
11255 
11256     cmpq(count, 192 >> shift);
11257     jcc(Assembler::less, L_fill_start_zmm_sequence);
11258 
11259     bind(L_fill_192_bytes_loop_header_zmm);
11260     subq(count, 192 >> shift);
11261 
11262     align32();
11263     bind(L_fill_192_bytes_loop_zmm);
11264       fill64(to, 0, xtmp, true);
11265       fill64(to, 64, xtmp, true);
11266       fill64(to, 128, xtmp, true);
11267       addq(to, 192);
11268       subq(count, 192 >> shift);
11269       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
11270 
11271     addq(count, 192 >> shift);
11272     jcc(Assembler::zero, L_exit);
11273     jmp(L_fill_start_zmm_sequence);
11274   }
11275   bind(L_exit);
11276 }
11277 #endif
11278 #endif //COMPILER2_OR_JVMCI
11279 
11280 
11281 #ifdef _LP64
11282 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
11283   Label done;
11284   cvttss2sil(dst, src);
11285   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
11286   cmpl(dst, 0x80000000); // float_sign_flip
11287   jccb(Assembler::notEqual, done);
11288   subptr(rsp, 8);
11289   movflt(Address(rsp, 0), src);
11290   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
11291   pop(dst);
11292   bind(done);
11293 }
11294 
11295 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
11296   Label done;
11297   cvttsd2sil(dst, src);
11298   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
11299   cmpl(dst, 0x80000000); // float_sign_flip
11300   jccb(Assembler::notEqual, done);
11301   subptr(rsp, 8);
11302   movdbl(Address(rsp, 0), src);
11303   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
11304   pop(dst);
11305   bind(done);
11306 }
11307 
11308 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
11309   Label done;
11310   cvttss2siq(dst, src);
11311   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
11312   jccb(Assembler::notEqual, done);
11313   subptr(rsp, 8);
11314   movflt(Address(rsp, 0), src);
11315   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
11316   pop(dst);
11317   bind(done);
11318 }
11319 
11320 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
11321   // Following code is line by line assembly translation rounding algorithm.
11322   // Please refer to java.lang.Math.round(float) algorithm for details.
11323   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
11324   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
11325   const int32_t FloatConsts_EXP_BIAS = 127;
11326   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
11327   const int32_t MINUS_32 = 0xFFFFFFE0;
11328   Label L_special_case, L_block1, L_exit;
11329   movl(rtmp, FloatConsts_EXP_BIT_MASK);
11330   movdl(dst, src);
11331   andl(dst, rtmp);
11332   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
11333   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
11334   subl(rtmp, dst);
11335   movl(rcx, rtmp);
11336   movl(dst, MINUS_32);
11337   testl(rtmp, dst);
11338   jccb(Assembler::notEqual, L_special_case);
11339   movdl(dst, src);
11340   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
11341   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
11342   movdl(rtmp, src);
11343   testl(rtmp, rtmp);
11344   jccb(Assembler::greaterEqual, L_block1);
11345   negl(dst);
11346   bind(L_block1);
11347   sarl(dst);
11348   addl(dst, 0x1);
11349   sarl(dst, 0x1);
11350   jmp(L_exit);
11351   bind(L_special_case);
11352   convert_f2i(dst, src);
11353   bind(L_exit);
11354 }
11355 
11356 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
11357   // Following code is line by line assembly translation rounding algorithm.
11358   // Please refer to java.lang.Math.round(double) algorithm for details.
11359   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
11360   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
11361   const int64_t DoubleConsts_EXP_BIAS = 1023;
11362   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
11363   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
11364   Label L_special_case, L_block1, L_exit;
11365   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
11366   movq(dst, src);
11367   andq(dst, rtmp);
11368   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
11369   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
11370   subq(rtmp, dst);
11371   movq(rcx, rtmp);
11372   mov64(dst, MINUS_64);
11373   testq(rtmp, dst);
11374   jccb(Assembler::notEqual, L_special_case);
11375   movq(dst, src);
11376   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
11377   andq(dst, rtmp);
11378   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
11379   orq(dst, rtmp);
11380   movq(rtmp, src);
11381   testq(rtmp, rtmp);
11382   jccb(Assembler::greaterEqual, L_block1);
11383   negq(dst);
11384   bind(L_block1);
11385   sarq(dst);
11386   addq(dst, 0x1);
11387   sarq(dst, 0x1);
11388   jmp(L_exit);
11389   bind(L_special_case);
11390   convert_d2l(dst, src);
11391   bind(L_exit);
11392 }
11393 
11394 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
11395   Label done;
11396   cvttsd2siq(dst, src);
11397   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
11398   jccb(Assembler::notEqual, done);
11399   subptr(rsp, 8);
11400   movdbl(Address(rsp, 0), src);
11401   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
11402   pop(dst);
11403   bind(done);
11404 }
11405 
11406 void MacroAssembler::cache_wb(Address line)
11407 {
11408   // 64 bit cpus always support clflush
11409   assert(VM_Version::supports_clflush(), "clflush should be available");
11410   bool optimized = VM_Version::supports_clflushopt();
11411   bool no_evict = VM_Version::supports_clwb();
11412 
11413   // prefer clwb (writeback without evict) otherwise
11414   // prefer clflushopt (potentially parallel writeback with evict)
11415   // otherwise fallback on clflush (serial writeback with evict)
11416 
11417   if (optimized) {
11418     if (no_evict) {
11419       clwb(line);
11420     } else {
11421       clflushopt(line);
11422     }
11423   } else {
11424     // no need for fence when using CLFLUSH
11425     clflush(line);
11426   }
11427 }
11428 
11429 void MacroAssembler::cache_wbsync(bool is_pre)
11430 {
11431   assert(VM_Version::supports_clflush(), "clflush should be available");
11432   bool optimized = VM_Version::supports_clflushopt();
11433   bool no_evict = VM_Version::supports_clwb();
11434 
11435   // pick the correct implementation
11436 
11437   if (!is_pre && (optimized || no_evict)) {
11438     // need an sfence for post flush when using clflushopt or clwb
11439     // otherwise no no need for any synchroniaztion
11440 
11441     sfence();
11442   }
11443 }
11444 
11445 #endif // _LP64
11446 
11447 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
11448   switch (cond) {
11449     // Note some conditions are synonyms for others
11450     case Assembler::zero:         return Assembler::notZero;
11451     case Assembler::notZero:      return Assembler::zero;
11452     case Assembler::less:         return Assembler::greaterEqual;
11453     case Assembler::lessEqual:    return Assembler::greater;
11454     case Assembler::greater:      return Assembler::lessEqual;
11455     case Assembler::greaterEqual: return Assembler::less;
11456     case Assembler::below:        return Assembler::aboveEqual;
11457     case Assembler::belowEqual:   return Assembler::above;
11458     case Assembler::above:        return Assembler::belowEqual;
11459     case Assembler::aboveEqual:   return Assembler::below;
11460     case Assembler::overflow:     return Assembler::noOverflow;
11461     case Assembler::noOverflow:   return Assembler::overflow;
11462     case Assembler::negative:     return Assembler::positive;
11463     case Assembler::positive:     return Assembler::negative;
11464     case Assembler::parity:       return Assembler::noParity;
11465     case Assembler::noParity:     return Assembler::parity;
11466   }
11467   ShouldNotReachHere(); return Assembler::overflow;
11468 }
11469 
11470 // This is simply a call to Thread::current()
11471 void MacroAssembler::get_thread(Register thread) {
11472   if (thread != rax) {
11473     push(rax);
11474   }
11475   LP64_ONLY(push(rdi);)
11476   LP64_ONLY(push(rsi);)
11477   push(rdx);
11478   push(rcx);
11479 #ifdef _LP64
11480   push(r8);
11481   push(r9);
11482   push(r10);
11483   push(r11);
11484 #endif
11485 
11486   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
11487 
11488 #ifdef _LP64
11489   pop(r11);
11490   pop(r10);
11491   pop(r9);
11492   pop(r8);
11493 #endif
11494   pop(rcx);
11495   pop(rdx);
11496   LP64_ONLY(pop(rsi);)
11497   LP64_ONLY(pop(rdi);)
11498   if (thread != rax) {
11499     mov(thread, rax);
11500     pop(rax);
11501   }
11502 }
11503 
11504 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
11505   Label L_stack_ok;
11506   if (bias == 0) {
11507     testptr(sp, 2 * wordSize - 1);
11508   } else {
11509     // lea(tmp, Address(rsp, bias);
11510     mov(tmp, sp);
11511     addptr(tmp, bias);
11512     testptr(tmp, 2 * wordSize - 1);
11513   }
11514   jcc(Assembler::equal, L_stack_ok);
11515   block_comment(msg);
11516   stop(msg);
11517   bind(L_stack_ok);
11518 }
11519 
11520 // Implements lightweight-locking.
11521 //
11522 // obj: the object to be locked
11523 // reg_rax: rax
11524 // thread: the thread which attempts to lock obj
11525 // tmp: a temporary register
11526 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
11527   assert(reg_rax == rax, "");
11528   assert_different_registers(basic_lock, obj, reg_rax, thread, tmp);
11529 
11530   Label push;
11531   const Register top = tmp;
11532 
11533   // Preload the markWord. It is important that this is the first
11534   // instruction emitted as it is part of C1's null check semantics.
11535   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
11536 
11537   if (UseObjectMonitorTable) {
11538     // Clear cache in case fast locking succeeds.
11539     movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0);
11540   }
11541 
11542   // Load top.
11543   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11544 
11545   // Check if the lock-stack is full.
11546   cmpl(top, LockStack::end_offset());
11547   jcc(Assembler::greaterEqual, slow);
11548 
11549   // Check for recursion.
11550   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
11551   jcc(Assembler::equal, push);
11552 
11553   // Check header for monitor (0b10).
11554   testptr(reg_rax, markWord::monitor_value);
11555   jcc(Assembler::notZero, slow);
11556 
11557   // Try to lock. Transition lock bits 0b01 => 0b00
11558   movptr(tmp, reg_rax);
11559   andptr(tmp, ~(int32_t)markWord::unlocked_value);
11560   orptr(reg_rax, markWord::unlocked_value);
11561   if (EnableValhalla) {
11562     // Mask inline_type bit such that we go to the slow path if object is an inline type
11563     andptr(reg_rax, ~((int) markWord::inline_type_bit_in_place));
11564   }
11565   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
11566   jcc(Assembler::notEqual, slow);
11567 
11568   // Restore top, CAS clobbers register.
11569   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11570 
11571   bind(push);
11572   // After successful lock, push object on lock-stack.
11573   movptr(Address(thread, top), obj);
11574   incrementl(top, oopSize);
11575   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
11576 }
11577 
11578 // Implements lightweight-unlocking.
11579 //
11580 // obj: the object to be unlocked
11581 // reg_rax: rax
11582 // thread: the thread
11583 // tmp: a temporary register
11584 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
11585   assert(reg_rax == rax, "");
11586   assert_different_registers(obj, reg_rax, thread, tmp);
11587 
11588   Label unlocked, push_and_slow;
11589   const Register top = tmp;
11590 
11591   // Check if obj is top of lock-stack.
11592   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11593   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
11594   jcc(Assembler::notEqual, slow);
11595 
11596   // Pop lock-stack.
11597   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
11598   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
11599 
11600   // Check if recursive.
11601   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
11602   jcc(Assembler::equal, unlocked);
11603 
11604   // Not recursive. Check header for monitor (0b10).
11605   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
11606   testptr(reg_rax, markWord::monitor_value);
11607   jcc(Assembler::notZero, push_and_slow);
11608 
11609 #ifdef ASSERT
11610   // Check header not unlocked (0b01).
11611   Label not_unlocked;
11612   testptr(reg_rax, markWord::unlocked_value);
11613   jcc(Assembler::zero, not_unlocked);
11614   stop("lightweight_unlock already unlocked");
11615   bind(not_unlocked);
11616 #endif
11617 
11618   // Try to unlock. Transition lock bits 0b00 => 0b01
11619   movptr(tmp, reg_rax);
11620   orptr(tmp, markWord::unlocked_value);
11621   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
11622   jcc(Assembler::equal, unlocked);
11623 
11624   bind(push_and_slow);
11625   // Restore lock-stack and handle the unlock in runtime.
11626 #ifdef ASSERT
11627   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11628   movptr(Address(thread, top), obj);
11629 #endif
11630   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
11631   jmp(slow);
11632 
11633   bind(unlocked);
11634 }
11635 
11636 #ifdef _LP64
11637 // Saves legacy GPRs state on stack.
11638 void MacroAssembler::save_legacy_gprs() {
11639   subq(rsp, 16 * wordSize);
11640   movq(Address(rsp, 15 * wordSize), rax);
11641   movq(Address(rsp, 14 * wordSize), rcx);
11642   movq(Address(rsp, 13 * wordSize), rdx);
11643   movq(Address(rsp, 12 * wordSize), rbx);
11644   movq(Address(rsp, 10 * wordSize), rbp);
11645   movq(Address(rsp, 9 * wordSize), rsi);
11646   movq(Address(rsp, 8 * wordSize), rdi);
11647   movq(Address(rsp, 7 * wordSize), r8);
11648   movq(Address(rsp, 6 * wordSize), r9);
11649   movq(Address(rsp, 5 * wordSize), r10);
11650   movq(Address(rsp, 4 * wordSize), r11);
11651   movq(Address(rsp, 3 * wordSize), r12);
11652   movq(Address(rsp, 2 * wordSize), r13);
11653   movq(Address(rsp, wordSize), r14);
11654   movq(Address(rsp, 0), r15);
11655 }
11656 
11657 // Resotres back legacy GPRs state from stack.
11658 void MacroAssembler::restore_legacy_gprs() {
11659   movq(r15, Address(rsp, 0));
11660   movq(r14, Address(rsp, wordSize));
11661   movq(r13, Address(rsp, 2 * wordSize));
11662   movq(r12, Address(rsp, 3 * wordSize));
11663   movq(r11, Address(rsp, 4 * wordSize));
11664   movq(r10, Address(rsp, 5 * wordSize));
11665   movq(r9,  Address(rsp, 6 * wordSize));
11666   movq(r8,  Address(rsp, 7 * wordSize));
11667   movq(rdi, Address(rsp, 8 * wordSize));
11668   movq(rsi, Address(rsp, 9 * wordSize));
11669   movq(rbp, Address(rsp, 10 * wordSize));
11670   movq(rbx, Address(rsp, 12 * wordSize));
11671   movq(rdx, Address(rsp, 13 * wordSize));
11672   movq(rcx, Address(rsp, 14 * wordSize));
11673   movq(rax, Address(rsp, 15 * wordSize));
11674   addq(rsp, 16 * wordSize);
11675 }
11676 
11677 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
11678   if (VM_Version::supports_apx_f()) {
11679     esetzucc(comparison, dst);
11680   } else {
11681     setb(comparison, dst);
11682     movzbl(dst, dst);
11683   }
11684 }
11685 #endif