1 /*
    2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "compiler/compiler_globals.hpp"
   29 #include "compiler/disassembler.hpp"
   30 #include "ci/ciInlineKlass.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "jvm.h"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedKlass.inline.hpp"
   43 #include "oops/compressedOops.inline.hpp"
   44 #include "oops/klass.inline.hpp"
   45 #include "prims/methodHandles.hpp"
   46 #include "runtime/continuation.hpp"
   47 #include "runtime/interfaceSupport.inline.hpp"
   48 #include "runtime/javaThread.hpp"
   49 #include "runtime/jniHandles.hpp"
   50 #include "runtime/objectMonitor.hpp"
   51 #include "runtime/os.hpp"
   52 #include "runtime/safepoint.hpp"
   53 #include "runtime/safepointMechanism.hpp"
   54 #include "runtime/sharedRuntime.hpp"
   55 #include "runtime/signature_cc.hpp"
   56 #include "runtime/stubRoutines.hpp"
   57 #include "utilities/macros.hpp"
   58 #include "vmreg_x86.inline.hpp"
   59 #ifdef COMPILER2
   60 #include "opto/output.hpp"
   61 #endif
   62 
   63 #ifdef PRODUCT
   64 #define BLOCK_COMMENT(str) /* nothing */
   65 #define STOP(error) stop(error)
   66 #else
   67 #define BLOCK_COMMENT(str) block_comment(str)
   68 #define STOP(error) block_comment(error); stop(error)
   69 #endif
   70 
   71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   72 
   73 #ifdef ASSERT
   74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   75 #endif
   76 
   77 static const Assembler::Condition reverse[] = {
   78     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   79     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   80     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   81     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   82     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   83     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   84     Assembler::above          /* belowEqual    = 0x6 */ ,
   85     Assembler::belowEqual     /* above         = 0x7 */ ,
   86     Assembler::positive       /* negative      = 0x8 */ ,
   87     Assembler::negative       /* positive      = 0x9 */ ,
   88     Assembler::noParity       /* parity        = 0xa */ ,
   89     Assembler::parity         /* noParity      = 0xb */ ,
   90     Assembler::greaterEqual   /* less          = 0xc */ ,
   91     Assembler::less           /* greaterEqual  = 0xd */ ,
   92     Assembler::greater        /* lessEqual     = 0xe */ ,
   93     Assembler::lessEqual      /* greater       = 0xf, */
   94 
   95 };
   96 
   97 
   98 // Implementation of MacroAssembler
   99 
  100 // First all the versions that have distinct versions depending on 32/64 bit
  101 // Unless the difference is trivial (1 line or so).
  102 
  103 #ifndef _LP64
  104 
  105 // 32bit versions
  106 
  107 Address MacroAssembler::as_Address(AddressLiteral adr) {
  108   return Address(adr.target(), adr.rspec());
  109 }
  110 
  111 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  112   assert(rscratch == noreg, "");
  113   return Address::make_array(adr);
  114 }
  115 
  116 void MacroAssembler::call_VM_leaf_base(address entry_point,
  117                                        int number_of_arguments) {
  118   call(RuntimeAddress(entry_point));
  119   increment(rsp, number_of_arguments * wordSize);
  120 }
  121 
  122 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  123   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  124 }
  125 
  126 
  127 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  128   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  129 }
  130 
  131 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  132   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  133 }
  134 
  135 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  136   assert(rscratch == noreg, "redundant");
  137   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  138 }
  139 
  140 void MacroAssembler::extend_sign(Register hi, Register lo) {
  141   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  142   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  143     cdql();
  144   } else {
  145     movl(hi, lo);
  146     sarl(hi, 31);
  147   }
  148 }
  149 
  150 void MacroAssembler::jC2(Register tmp, Label& L) {
  151   // set parity bit if FPU flag C2 is set (via rax)
  152   save_rax(tmp);
  153   fwait(); fnstsw_ax();
  154   sahf();
  155   restore_rax(tmp);
  156   // branch
  157   jcc(Assembler::parity, L);
  158 }
  159 
  160 void MacroAssembler::jnC2(Register tmp, Label& L) {
  161   // set parity bit if FPU flag C2 is set (via rax)
  162   save_rax(tmp);
  163   fwait(); fnstsw_ax();
  164   sahf();
  165   restore_rax(tmp);
  166   // branch
  167   jcc(Assembler::noParity, L);
  168 }
  169 
  170 // 32bit can do a case table jump in one instruction but we no longer allow the base
  171 // to be installed in the Address class
  172 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  173   assert(rscratch == noreg, "not needed");
  174   jmp(as_Address(entry, noreg));
  175 }
  176 
  177 // Note: y_lo will be destroyed
  178 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  179   // Long compare for Java (semantics as described in JVM spec.)
  180   Label high, low, done;
  181 
  182   cmpl(x_hi, y_hi);
  183   jcc(Assembler::less, low);
  184   jcc(Assembler::greater, high);
  185   // x_hi is the return register
  186   xorl(x_hi, x_hi);
  187   cmpl(x_lo, y_lo);
  188   jcc(Assembler::below, low);
  189   jcc(Assembler::equal, done);
  190 
  191   bind(high);
  192   xorl(x_hi, x_hi);
  193   increment(x_hi);
  194   jmp(done);
  195 
  196   bind(low);
  197   xorl(x_hi, x_hi);
  198   decrementl(x_hi);
  199 
  200   bind(done);
  201 }
  202 
  203 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  204   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  205 }
  206 
  207 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  208   assert(rscratch == noreg, "not needed");
  209 
  210   // leal(dst, as_Address(adr));
  211   // see note in movl as to why we must use a move
  212   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  213 }
  214 
  215 void MacroAssembler::leave() {
  216   mov(rsp, rbp);
  217   pop(rbp);
  218 }
  219 
  220 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  221   // Multiplication of two Java long values stored on the stack
  222   // as illustrated below. Result is in rdx:rax.
  223   //
  224   // rsp ---> [  ??  ] \               \
  225   //            ....    | y_rsp_offset  |
  226   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  227   //          [ y_hi ]                  | (in bytes)
  228   //            ....                    |
  229   //          [ x_lo ]                 /
  230   //          [ x_hi ]
  231   //            ....
  232   //
  233   // Basic idea: lo(result) = lo(x_lo * y_lo)
  234   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  235   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  236   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  237   Label quick;
  238   // load x_hi, y_hi and check if quick
  239   // multiplication is possible
  240   movl(rbx, x_hi);
  241   movl(rcx, y_hi);
  242   movl(rax, rbx);
  243   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  244   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  245   // do full multiplication
  246   // 1st step
  247   mull(y_lo);                                    // x_hi * y_lo
  248   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  249   // 2nd step
  250   movl(rax, x_lo);
  251   mull(rcx);                                     // x_lo * y_hi
  252   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  253   // 3rd step
  254   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  255   movl(rax, x_lo);
  256   mull(y_lo);                                    // x_lo * y_lo
  257   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  258 }
  259 
  260 void MacroAssembler::lneg(Register hi, Register lo) {
  261   negl(lo);
  262   adcl(hi, 0);
  263   negl(hi);
  264 }
  265 
  266 void MacroAssembler::lshl(Register hi, Register lo) {
  267   // Java shift left long support (semantics as described in JVM spec., p.305)
  268   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  269   // shift value is in rcx !
  270   assert(hi != rcx, "must not use rcx");
  271   assert(lo != rcx, "must not use rcx");
  272   const Register s = rcx;                        // shift count
  273   const int      n = BitsPerWord;
  274   Label L;
  275   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  276   cmpl(s, n);                                    // if (s < n)
  277   jcc(Assembler::less, L);                       // else (s >= n)
  278   movl(hi, lo);                                  // x := x << n
  279   xorl(lo, lo);
  280   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  281   bind(L);                                       // s (mod n) < n
  282   shldl(hi, lo);                                 // x := x << s
  283   shll(lo);
  284 }
  285 
  286 
  287 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  288   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  289   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  290   assert(hi != rcx, "must not use rcx");
  291   assert(lo != rcx, "must not use rcx");
  292   const Register s = rcx;                        // shift count
  293   const int      n = BitsPerWord;
  294   Label L;
  295   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  296   cmpl(s, n);                                    // if (s < n)
  297   jcc(Assembler::less, L);                       // else (s >= n)
  298   movl(lo, hi);                                  // x := x >> n
  299   if (sign_extension) sarl(hi, 31);
  300   else                xorl(hi, hi);
  301   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  302   bind(L);                                       // s (mod n) < n
  303   shrdl(lo, hi);                                 // x := x >> s
  304   if (sign_extension) sarl(hi);
  305   else                shrl(hi);
  306 }
  307 
  308 void MacroAssembler::movoop(Register dst, jobject obj) {
  309   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  310 }
  311 
  312 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  313   assert(rscratch == noreg, "redundant");
  314   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  315 }
  316 
  317 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  318   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  319 }
  320 
  321 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  322   assert(rscratch == noreg, "redundant");
  323   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  324 }
  325 
  326 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  327   if (src.is_lval()) {
  328     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  329   } else {
  330     movl(dst, as_Address(src));
  331   }
  332 }
  333 
  334 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  335   assert(rscratch == noreg, "redundant");
  336   movl(as_Address(dst, noreg), src);
  337 }
  338 
  339 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  340   movl(dst, as_Address(src, noreg));
  341 }
  342 
  343 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  344   assert(rscratch == noreg, "redundant");
  345   movl(dst, src);
  346 }
  347 
  348 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  349   assert(rscratch == noreg, "redundant");
  350   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  351 }
  352 
  353 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  354   assert(rscratch == noreg, "redundant");
  355   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  356 }
  357 
  358 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  359   assert(rscratch == noreg, "redundant");
  360   if (src.is_lval()) {
  361     push_literal32((int32_t)src.target(), src.rspec());
  362   } else {
  363     pushl(as_Address(src));
  364   }
  365 }
  366 
  367 static void pass_arg0(MacroAssembler* masm, Register arg) {
  368   masm->push(arg);
  369 }
  370 
  371 static void pass_arg1(MacroAssembler* masm, Register arg) {
  372   masm->push(arg);
  373 }
  374 
  375 static void pass_arg2(MacroAssembler* masm, Register arg) {
  376   masm->push(arg);
  377 }
  378 
  379 static void pass_arg3(MacroAssembler* masm, Register arg) {
  380   masm->push(arg);
  381 }
  382 
  383 #ifndef PRODUCT
  384 extern "C" void findpc(intptr_t x);
  385 #endif
  386 
  387 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  388   // In order to get locks to work, we need to fake a in_VM state
  389   JavaThread* thread = JavaThread::current();
  390   JavaThreadState saved_state = thread->thread_state();
  391   thread->set_thread_state(_thread_in_vm);
  392   if (ShowMessageBoxOnError) {
  393     JavaThread* thread = JavaThread::current();
  394     JavaThreadState saved_state = thread->thread_state();
  395     thread->set_thread_state(_thread_in_vm);
  396     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  397       ttyLocker ttyl;
  398       BytecodeCounter::print();
  399     }
  400     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  401     // This is the value of eip which points to where verify_oop will return.
  402     if (os::message_box(msg, "Execution stopped, print registers?")) {
  403       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  404       BREAKPOINT;
  405     }
  406   }
  407   fatal("DEBUG MESSAGE: %s", msg);
  408 }
  409 
  410 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  411   ttyLocker ttyl;
  412   DebuggingContext debugging{};
  413   tty->print_cr("eip = 0x%08x", eip);
  414 #ifndef PRODUCT
  415   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  416     tty->cr();
  417     findpc(eip);
  418     tty->cr();
  419   }
  420 #endif
  421 #define PRINT_REG(rax) \
  422   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  423   PRINT_REG(rax);
  424   PRINT_REG(rbx);
  425   PRINT_REG(rcx);
  426   PRINT_REG(rdx);
  427   PRINT_REG(rdi);
  428   PRINT_REG(rsi);
  429   PRINT_REG(rbp);
  430   PRINT_REG(rsp);
  431 #undef PRINT_REG
  432   // Print some words near top of staack.
  433   int* dump_sp = (int*) rsp;
  434   for (int col1 = 0; col1 < 8; col1++) {
  435     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  436     os::print_location(tty, *dump_sp++);
  437   }
  438   for (int row = 0; row < 16; row++) {
  439     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  440     for (int col = 0; col < 8; col++) {
  441       tty->print(" 0x%08x", *dump_sp++);
  442     }
  443     tty->cr();
  444   }
  445   // Print some instructions around pc:
  446   Disassembler::decode((address)eip-64, (address)eip);
  447   tty->print_cr("--------");
  448   Disassembler::decode((address)eip, (address)eip+32);
  449 }
  450 
  451 void MacroAssembler::stop(const char* msg) {
  452   // push address of message
  453   ExternalAddress message((address)msg);
  454   pushptr(message.addr(), noreg);
  455   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  456   pusha();                                            // push registers
  457   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  458   hlt();
  459 }
  460 
  461 void MacroAssembler::warn(const char* msg) {
  462   push_CPU_state();
  463 
  464   // push address of message
  465   ExternalAddress message((address)msg);
  466   pushptr(message.addr(), noreg);
  467 
  468   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  469   addl(rsp, wordSize);       // discard argument
  470   pop_CPU_state();
  471 }
  472 
  473 void MacroAssembler::print_state() {
  474   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  475   pusha();                                            // push registers
  476 
  477   push_CPU_state();
  478   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  479   pop_CPU_state();
  480 
  481   popa();
  482   addl(rsp, wordSize);
  483 }
  484 
  485 #else // _LP64
  486 
  487 // 64 bit versions
  488 
  489 Address MacroAssembler::as_Address(AddressLiteral adr) {
  490   // amd64 always does this as a pc-rel
  491   // we can be absolute or disp based on the instruction type
  492   // jmp/call are displacements others are absolute
  493   assert(!adr.is_lval(), "must be rval");
  494   assert(reachable(adr), "must be");
  495   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  496 
  497 }
  498 
  499 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  500   AddressLiteral base = adr.base();
  501   lea(rscratch, base);
  502   Address index = adr.index();
  503   assert(index._disp == 0, "must not have disp"); // maybe it can?
  504   Address array(rscratch, index._index, index._scale, index._disp);
  505   return array;
  506 }
  507 
  508 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  509   Label L, E;
  510 
  511 #ifdef _WIN64
  512   // Windows always allocates space for it's register args
  513   assert(num_args <= 4, "only register arguments supported");
  514   subq(rsp,  frame::arg_reg_save_area_bytes);
  515 #endif
  516 
  517   // Align stack if necessary
  518   testl(rsp, 15);
  519   jcc(Assembler::zero, L);
  520 
  521   subq(rsp, 8);
  522   call(RuntimeAddress(entry_point));
  523   addq(rsp, 8);
  524   jmp(E);
  525 
  526   bind(L);
  527   call(RuntimeAddress(entry_point));
  528 
  529   bind(E);
  530 
  531 #ifdef _WIN64
  532   // restore stack pointer
  533   addq(rsp, frame::arg_reg_save_area_bytes);
  534 #endif
  535 
  536 }
  537 
  538 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  539   assert(!src2.is_lval(), "should use cmpptr");
  540   assert(rscratch != noreg || always_reachable(src2), "missing");
  541 
  542   if (reachable(src2)) {
  543     cmpq(src1, as_Address(src2));
  544   } else {
  545     lea(rscratch, src2);
  546     Assembler::cmpq(src1, Address(rscratch, 0));
  547   }
  548 }
  549 
  550 int MacroAssembler::corrected_idivq(Register reg) {
  551   // Full implementation of Java ldiv and lrem; checks for special
  552   // case as described in JVM spec., p.243 & p.271.  The function
  553   // returns the (pc) offset of the idivl instruction - may be needed
  554   // for implicit exceptions.
  555   //
  556   //         normal case                           special case
  557   //
  558   // input : rax: dividend                         min_long
  559   //         reg: divisor   (may not be eax/edx)   -1
  560   //
  561   // output: rax: quotient  (= rax idiv reg)       min_long
  562   //         rdx: remainder (= rax irem reg)       0
  563   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  564   static const int64_t min_long = 0x8000000000000000;
  565   Label normal_case, special_case;
  566 
  567   // check for special case
  568   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  569   jcc(Assembler::notEqual, normal_case);
  570   xorl(rdx, rdx); // prepare rdx for possible special case (where
  571                   // remainder = 0)
  572   cmpq(reg, -1);
  573   jcc(Assembler::equal, special_case);
  574 
  575   // handle normal case
  576   bind(normal_case);
  577   cdqq();
  578   int idivq_offset = offset();
  579   idivq(reg);
  580 
  581   // normal and special case exit
  582   bind(special_case);
  583 
  584   return idivq_offset;
  585 }
  586 
  587 void MacroAssembler::decrementq(Register reg, int value) {
  588   if (value == min_jint) { subq(reg, value); return; }
  589   if (value <  0) { incrementq(reg, -value); return; }
  590   if (value == 0) {                        ; return; }
  591   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  592   /* else */      { subq(reg, value)       ; return; }
  593 }
  594 
  595 void MacroAssembler::decrementq(Address dst, int value) {
  596   if (value == min_jint) { subq(dst, value); return; }
  597   if (value <  0) { incrementq(dst, -value); return; }
  598   if (value == 0) {                        ; return; }
  599   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  600   /* else */      { subq(dst, value)       ; return; }
  601 }
  602 
  603 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  604   assert(rscratch != noreg || always_reachable(dst), "missing");
  605 
  606   if (reachable(dst)) {
  607     incrementq(as_Address(dst));
  608   } else {
  609     lea(rscratch, dst);
  610     incrementq(Address(rscratch, 0));
  611   }
  612 }
  613 
  614 void MacroAssembler::incrementq(Register reg, int value) {
  615   if (value == min_jint) { addq(reg, value); return; }
  616   if (value <  0) { decrementq(reg, -value); return; }
  617   if (value == 0) {                        ; return; }
  618   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  619   /* else */      { addq(reg, value)       ; return; }
  620 }
  621 
  622 void MacroAssembler::incrementq(Address dst, int value) {
  623   if (value == min_jint) { addq(dst, value); return; }
  624   if (value <  0) { decrementq(dst, -value); return; }
  625   if (value == 0) {                        ; return; }
  626   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  627   /* else */      { addq(dst, value)       ; return; }
  628 }
  629 
  630 // 32bit can do a case table jump in one instruction but we no longer allow the base
  631 // to be installed in the Address class
  632 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  633   lea(rscratch, entry.base());
  634   Address dispatch = entry.index();
  635   assert(dispatch._base == noreg, "must be");
  636   dispatch._base = rscratch;
  637   jmp(dispatch);
  638 }
  639 
  640 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  641   ShouldNotReachHere(); // 64bit doesn't use two regs
  642   cmpq(x_lo, y_lo);
  643 }
  644 
  645 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  646   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  647 }
  648 
  649 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  650   lea(rscratch, adr);
  651   movptr(dst, rscratch);
  652 }
  653 
  654 void MacroAssembler::leave() {
  655   // %%% is this really better? Why not on 32bit too?
  656   emit_int8((unsigned char)0xC9); // LEAVE
  657 }
  658 
  659 void MacroAssembler::lneg(Register hi, Register lo) {
  660   ShouldNotReachHere(); // 64bit doesn't use two regs
  661   negq(lo);
  662 }
  663 
  664 void MacroAssembler::movoop(Register dst, jobject obj) {
  665   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  666 }
  667 
  668 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  669   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  670   movq(dst, rscratch);
  671 }
  672 
  673 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  674   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  675 }
  676 
  677 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  678   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  679   movq(dst, rscratch);
  680 }
  681 
  682 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  683   if (src.is_lval()) {
  684     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  685   } else {
  686     if (reachable(src)) {
  687       movq(dst, as_Address(src));
  688     } else {
  689       lea(dst, src);
  690       movq(dst, Address(dst, 0));
  691     }
  692   }
  693 }
  694 
  695 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  696   movq(as_Address(dst, rscratch), src);
  697 }
  698 
  699 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  700   movq(dst, as_Address(src, dst /*rscratch*/));
  701 }
  702 
  703 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  704 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  705   if (is_simm32(src)) {
  706     movptr(dst, checked_cast<int32_t>(src));
  707   } else {
  708     mov64(rscratch, src);
  709     movq(dst, rscratch);
  710   }
  711 }
  712 
  713 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  714   movoop(rscratch, obj);
  715   push(rscratch);
  716 }
  717 
  718 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  719   mov_metadata(rscratch, obj);
  720   push(rscratch);
  721 }
  722 
  723 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  724   lea(rscratch, src);
  725   if (src.is_lval()) {
  726     push(rscratch);
  727   } else {
  728     pushq(Address(rscratch, 0));
  729   }
  730 }
  731 
  732 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  733   reset_last_Java_frame(r15_thread, clear_fp);
  734 }
  735 
  736 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  737                                          Register last_java_fp,
  738                                          address  last_java_pc,
  739                                          Register rscratch) {
  740   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  741 }
  742 
  743 static void pass_arg0(MacroAssembler* masm, Register arg) {
  744   if (c_rarg0 != arg ) {
  745     masm->mov(c_rarg0, arg);
  746   }
  747 }
  748 
  749 static void pass_arg1(MacroAssembler* masm, Register arg) {
  750   if (c_rarg1 != arg ) {
  751     masm->mov(c_rarg1, arg);
  752   }
  753 }
  754 
  755 static void pass_arg2(MacroAssembler* masm, Register arg) {
  756   if (c_rarg2 != arg ) {
  757     masm->mov(c_rarg2, arg);
  758   }
  759 }
  760 
  761 static void pass_arg3(MacroAssembler* masm, Register arg) {
  762   if (c_rarg3 != arg ) {
  763     masm->mov(c_rarg3, arg);
  764   }
  765 }
  766 
  767 void MacroAssembler::stop(const char* msg) {
  768   if (ShowMessageBoxOnError) {
  769     address rip = pc();
  770     pusha(); // get regs on stack
  771     lea(c_rarg1, InternalAddress(rip));
  772     movq(c_rarg2, rsp); // pass pointer to regs array
  773   }
  774   lea(c_rarg0, ExternalAddress((address) msg));
  775   andq(rsp, -16); // align stack as required by ABI
  776   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  777   hlt();
  778 }
  779 
  780 void MacroAssembler::warn(const char* msg) {
  781   push(rbp);
  782   movq(rbp, rsp);
  783   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  784   push_CPU_state();   // keeps alignment at 16 bytes
  785 
  786   lea(c_rarg0, ExternalAddress((address) msg));
  787   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  788 
  789   pop_CPU_state();
  790   mov(rsp, rbp);
  791   pop(rbp);
  792 }
  793 
  794 void MacroAssembler::print_state() {
  795   address rip = pc();
  796   pusha();            // get regs on stack
  797   push(rbp);
  798   movq(rbp, rsp);
  799   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  800   push_CPU_state();   // keeps alignment at 16 bytes
  801 
  802   lea(c_rarg0, InternalAddress(rip));
  803   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  804   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  805 
  806   pop_CPU_state();
  807   mov(rsp, rbp);
  808   pop(rbp);
  809   popa();
  810 }
  811 
  812 #ifndef PRODUCT
  813 extern "C" void findpc(intptr_t x);
  814 #endif
  815 
  816 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  817   // In order to get locks to work, we need to fake a in_VM state
  818   if (ShowMessageBoxOnError) {
  819     JavaThread* thread = JavaThread::current();
  820     JavaThreadState saved_state = thread->thread_state();
  821     thread->set_thread_state(_thread_in_vm);
  822 #ifndef PRODUCT
  823     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  824       ttyLocker ttyl;
  825       BytecodeCounter::print();
  826     }
  827 #endif
  828     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  829     // XXX correct this offset for amd64
  830     // This is the value of eip which points to where verify_oop will return.
  831     if (os::message_box(msg, "Execution stopped, print registers?")) {
  832       print_state64(pc, regs);
  833       BREAKPOINT;
  834     }
  835   }
  836   fatal("DEBUG MESSAGE: %s", msg);
  837 }
  838 
  839 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  840   ttyLocker ttyl;
  841   DebuggingContext debugging{};
  842   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  843 #ifndef PRODUCT
  844   tty->cr();
  845   findpc(pc);
  846   tty->cr();
  847 #endif
  848 #define PRINT_REG(rax, value) \
  849   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  850   PRINT_REG(rax, regs[15]);
  851   PRINT_REG(rbx, regs[12]);
  852   PRINT_REG(rcx, regs[14]);
  853   PRINT_REG(rdx, regs[13]);
  854   PRINT_REG(rdi, regs[8]);
  855   PRINT_REG(rsi, regs[9]);
  856   PRINT_REG(rbp, regs[10]);
  857   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  858   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  859   PRINT_REG(r8 , regs[7]);
  860   PRINT_REG(r9 , regs[6]);
  861   PRINT_REG(r10, regs[5]);
  862   PRINT_REG(r11, regs[4]);
  863   PRINT_REG(r12, regs[3]);
  864   PRINT_REG(r13, regs[2]);
  865   PRINT_REG(r14, regs[1]);
  866   PRINT_REG(r15, regs[0]);
  867 #undef PRINT_REG
  868   // Print some words near the top of the stack.
  869   int64_t* rsp = &regs[16];
  870   int64_t* dump_sp = rsp;
  871   for (int col1 = 0; col1 < 8; col1++) {
  872     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  873     os::print_location(tty, *dump_sp++);
  874   }
  875   for (int row = 0; row < 25; row++) {
  876     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  877     for (int col = 0; col < 4; col++) {
  878       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  879     }
  880     tty->cr();
  881   }
  882   // Print some instructions around pc:
  883   Disassembler::decode((address)pc-64, (address)pc);
  884   tty->print_cr("--------");
  885   Disassembler::decode((address)pc, (address)pc+32);
  886 }
  887 
  888 // The java_calling_convention describes stack locations as ideal slots on
  889 // a frame with no abi restrictions. Since we must observe abi restrictions
  890 // (like the placement of the register window) the slots must be biased by
  891 // the following value.
  892 static int reg2offset_in(VMReg r) {
  893   // Account for saved rbp and return address
  894   // This should really be in_preserve_stack_slots
  895   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  896 }
  897 
  898 static int reg2offset_out(VMReg r) {
  899   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  900 }
  901 
  902 // A long move
  903 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  904 
  905   // The calling conventions assures us that each VMregpair is either
  906   // all really one physical register or adjacent stack slots.
  907 
  908   if (src.is_single_phys_reg() ) {
  909     if (dst.is_single_phys_reg()) {
  910       if (dst.first() != src.first()) {
  911         mov(dst.first()->as_Register(), src.first()->as_Register());
  912       }
  913     } else {
  914       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  915              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  916       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  917     }
  918   } else if (dst.is_single_phys_reg()) {
  919     assert(src.is_single_reg(),  "not a stack pair");
  920     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  921   } else {
  922     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  923     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  924     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  925   }
  926 }
  927 
  928 // A double move
  929 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  930 
  931   // The calling conventions assures us that each VMregpair is either
  932   // all really one physical register or adjacent stack slots.
  933 
  934   if (src.is_single_phys_reg() ) {
  935     if (dst.is_single_phys_reg()) {
  936       // In theory these overlap but the ordering is such that this is likely a nop
  937       if ( src.first() != dst.first()) {
  938         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  939       }
  940     } else {
  941       assert(dst.is_single_reg(), "not a stack pair");
  942       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  943     }
  944   } else if (dst.is_single_phys_reg()) {
  945     assert(src.is_single_reg(),  "not a stack pair");
  946     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  947   } else {
  948     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  949     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  950     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  951   }
  952 }
  953 
  954 
  955 // A float arg may have to do float reg int reg conversion
  956 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  957   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  958 
  959   // The calling conventions assures us that each VMregpair is either
  960   // all really one physical register or adjacent stack slots.
  961 
  962   if (src.first()->is_stack()) {
  963     if (dst.first()->is_stack()) {
  964       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  965       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  966     } else {
  967       // stack to reg
  968       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  969       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  970     }
  971   } else if (dst.first()->is_stack()) {
  972     // reg to stack
  973     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  974     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  975   } else {
  976     // reg to reg
  977     // In theory these overlap but the ordering is such that this is likely a nop
  978     if ( src.first() != dst.first()) {
  979       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  980     }
  981   }
  982 }
  983 
  984 // On 64 bit we will store integer like items to the stack as
  985 // 64 bits items (x86_32/64 abi) even though java would only store
  986 // 32bits for a parameter. On 32bit it will simply be 32 bits
  987 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  988 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  989   if (src.first()->is_stack()) {
  990     if (dst.first()->is_stack()) {
  991       // stack to stack
  992       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  993       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  994     } else {
  995       // stack to reg
  996       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  997     }
  998   } else if (dst.first()->is_stack()) {
  999     // reg to stack
 1000     // Do we really have to sign extend???
 1001     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 1002     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 1003   } else {
 1004     // Do we really have to sign extend???
 1005     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1006     if (dst.first() != src.first()) {
 1007       movq(dst.first()->as_Register(), src.first()->as_Register());
 1008     }
 1009   }
 1010 }
 1011 
 1012 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1013   if (src.first()->is_stack()) {
 1014     if (dst.first()->is_stack()) {
 1015       // stack to stack
 1016       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1017       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1018     } else {
 1019       // stack to reg
 1020       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1021     }
 1022   } else if (dst.first()->is_stack()) {
 1023     // reg to stack
 1024     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1025   } else {
 1026     if (dst.first() != src.first()) {
 1027       movq(dst.first()->as_Register(), src.first()->as_Register());
 1028     }
 1029   }
 1030 }
 1031 
 1032 // An oop arg. Must pass a handle not the oop itself
 1033 void MacroAssembler::object_move(OopMap* map,
 1034                         int oop_handle_offset,
 1035                         int framesize_in_slots,
 1036                         VMRegPair src,
 1037                         VMRegPair dst,
 1038                         bool is_receiver,
 1039                         int* receiver_offset) {
 1040 
 1041   // must pass a handle. First figure out the location we use as a handle
 1042 
 1043   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1044 
 1045   // See if oop is null if it is we need no handle
 1046 
 1047   if (src.first()->is_stack()) {
 1048 
 1049     // Oop is already on the stack as an argument
 1050     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1051     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1052     if (is_receiver) {
 1053       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1054     }
 1055 
 1056     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1057     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1058     // conditionally move a null
 1059     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1060   } else {
 1061 
 1062     // Oop is in a register we must store it to the space we reserve
 1063     // on the stack for oop_handles and pass a handle if oop is non-null
 1064 
 1065     const Register rOop = src.first()->as_Register();
 1066     int oop_slot;
 1067     if (rOop == j_rarg0)
 1068       oop_slot = 0;
 1069     else if (rOop == j_rarg1)
 1070       oop_slot = 1;
 1071     else if (rOop == j_rarg2)
 1072       oop_slot = 2;
 1073     else if (rOop == j_rarg3)
 1074       oop_slot = 3;
 1075     else if (rOop == j_rarg4)
 1076       oop_slot = 4;
 1077     else {
 1078       assert(rOop == j_rarg5, "wrong register");
 1079       oop_slot = 5;
 1080     }
 1081 
 1082     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1083     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1084 
 1085     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1086     // Store oop in handle area, may be null
 1087     movptr(Address(rsp, offset), rOop);
 1088     if (is_receiver) {
 1089       *receiver_offset = offset;
 1090     }
 1091 
 1092     cmpptr(rOop, NULL_WORD);
 1093     lea(rHandle, Address(rsp, offset));
 1094     // conditionally move a null from the handle area where it was just stored
 1095     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1096   }
 1097 
 1098   // If arg is on the stack then place it otherwise it is already in correct reg.
 1099   if (dst.first()->is_stack()) {
 1100     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1101   }
 1102 }
 1103 
 1104 #endif // _LP64
 1105 
 1106 // Now versions that are common to 32/64 bit
 1107 
 1108 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1109   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1110 }
 1111 
 1112 void MacroAssembler::addptr(Register dst, Register src) {
 1113   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1114 }
 1115 
 1116 void MacroAssembler::addptr(Address dst, Register src) {
 1117   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1118 }
 1119 
 1120 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1121   assert(rscratch != noreg || always_reachable(src), "missing");
 1122 
 1123   if (reachable(src)) {
 1124     Assembler::addsd(dst, as_Address(src));
 1125   } else {
 1126     lea(rscratch, src);
 1127     Assembler::addsd(dst, Address(rscratch, 0));
 1128   }
 1129 }
 1130 
 1131 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1132   assert(rscratch != noreg || always_reachable(src), "missing");
 1133 
 1134   if (reachable(src)) {
 1135     addss(dst, as_Address(src));
 1136   } else {
 1137     lea(rscratch, src);
 1138     addss(dst, Address(rscratch, 0));
 1139   }
 1140 }
 1141 
 1142 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1143   assert(rscratch != noreg || always_reachable(src), "missing");
 1144 
 1145   if (reachable(src)) {
 1146     Assembler::addpd(dst, as_Address(src));
 1147   } else {
 1148     lea(rscratch, src);
 1149     Assembler::addpd(dst, Address(rscratch, 0));
 1150   }
 1151 }
 1152 
 1153 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1154 // Stub code is generated once and never copied.
 1155 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1156 void MacroAssembler::align64() {
 1157   align(64, (unsigned long long) pc());
 1158 }
 1159 
 1160 void MacroAssembler::align32() {
 1161   align(32, (unsigned long long) pc());
 1162 }
 1163 
 1164 void MacroAssembler::align(int modulus) {
 1165   // 8273459: Ensure alignment is possible with current segment alignment
 1166   assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1167   align(modulus, offset());
 1168 }
 1169 
 1170 void MacroAssembler::align(int modulus, int target) {
 1171   if (target % modulus != 0) {
 1172     nop(modulus - (target % modulus));
 1173   }
 1174 }
 1175 
 1176 void MacroAssembler::push_f(XMMRegister r) {
 1177   subptr(rsp, wordSize);
 1178   movflt(Address(rsp, 0), r);
 1179 }
 1180 
 1181 void MacroAssembler::pop_f(XMMRegister r) {
 1182   movflt(r, Address(rsp, 0));
 1183   addptr(rsp, wordSize);
 1184 }
 1185 
 1186 void MacroAssembler::push_d(XMMRegister r) {
 1187   subptr(rsp, 2 * wordSize);
 1188   movdbl(Address(rsp, 0), r);
 1189 }
 1190 
 1191 void MacroAssembler::pop_d(XMMRegister r) {
 1192   movdbl(r, Address(rsp, 0));
 1193   addptr(rsp, 2 * Interpreter::stackElementSize);
 1194 }
 1195 
 1196 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1197   // Used in sign-masking with aligned address.
 1198   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1199   assert(rscratch != noreg || always_reachable(src), "missing");
 1200 
 1201   if (reachable(src)) {
 1202     Assembler::andpd(dst, as_Address(src));
 1203   } else {
 1204     lea(rscratch, src);
 1205     Assembler::andpd(dst, Address(rscratch, 0));
 1206   }
 1207 }
 1208 
 1209 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1210   // Used in sign-masking with aligned address.
 1211   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1212   assert(rscratch != noreg || always_reachable(src), "missing");
 1213 
 1214   if (reachable(src)) {
 1215     Assembler::andps(dst, as_Address(src));
 1216   } else {
 1217     lea(rscratch, src);
 1218     Assembler::andps(dst, Address(rscratch, 0));
 1219   }
 1220 }
 1221 
 1222 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1223   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1224 }
 1225 
 1226 #ifdef _LP64
 1227 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1228   assert(rscratch != noreg || always_reachable(src), "missing");
 1229 
 1230   if (reachable(src)) {
 1231     andq(dst, as_Address(src));
 1232   } else {
 1233     lea(rscratch, src);
 1234     andq(dst, Address(rscratch, 0));
 1235   }
 1236 }
 1237 #endif
 1238 
 1239 void MacroAssembler::atomic_incl(Address counter_addr) {
 1240   lock();
 1241   incrementl(counter_addr);
 1242 }
 1243 
 1244 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1245   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1246 
 1247   if (reachable(counter_addr)) {
 1248     atomic_incl(as_Address(counter_addr));
 1249   } else {
 1250     lea(rscratch, counter_addr);
 1251     atomic_incl(Address(rscratch, 0));
 1252   }
 1253 }
 1254 
 1255 #ifdef _LP64
 1256 void MacroAssembler::atomic_incq(Address counter_addr) {
 1257   lock();
 1258   incrementq(counter_addr);
 1259 }
 1260 
 1261 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1262   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1263 
 1264   if (reachable(counter_addr)) {
 1265     atomic_incq(as_Address(counter_addr));
 1266   } else {
 1267     lea(rscratch, counter_addr);
 1268     atomic_incq(Address(rscratch, 0));
 1269   }
 1270 }
 1271 #endif
 1272 
 1273 // Writes to stack successive pages until offset reached to check for
 1274 // stack overflow + shadow pages.  This clobbers tmp.
 1275 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1276   movptr(tmp, rsp);
 1277   // Bang stack for total size given plus shadow page size.
 1278   // Bang one page at a time because large size can bang beyond yellow and
 1279   // red zones.
 1280   Label loop;
 1281   bind(loop);
 1282   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1283   subptr(tmp, (int)os::vm_page_size());
 1284   subl(size, (int)os::vm_page_size());
 1285   jcc(Assembler::greater, loop);
 1286 
 1287   // Bang down shadow pages too.
 1288   // At this point, (tmp-0) is the last address touched, so don't
 1289   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1290   // was post-decremented.)  Skip this address by starting at i=1, and
 1291   // touch a few more pages below.  N.B.  It is important to touch all
 1292   // the way down including all pages in the shadow zone.
 1293   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1294     // this could be any sized move but this is can be a debugging crumb
 1295     // so the bigger the better.
 1296     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1297   }
 1298 }
 1299 
 1300 void MacroAssembler::reserved_stack_check() {
 1301   // testing if reserved zone needs to be enabled
 1302   Label no_reserved_zone_enabling;
 1303   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1304   NOT_LP64(get_thread(rsi);)
 1305 
 1306   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1307   jcc(Assembler::below, no_reserved_zone_enabling);
 1308 
 1309   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1310   jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
 1311   should_not_reach_here();
 1312 
 1313   bind(no_reserved_zone_enabling);
 1314 }
 1315 
 1316 void MacroAssembler::c2bool(Register x) {
 1317   // implements x == 0 ? 0 : 1
 1318   // note: must only look at least-significant byte of x
 1319   //       since C-style booleans are stored in one byte
 1320   //       only! (was bug)
 1321   andl(x, 0xFF);
 1322   setb(Assembler::notZero, x);
 1323 }
 1324 
 1325 // Wouldn't need if AddressLiteral version had new name
 1326 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1327   Assembler::call(L, rtype);
 1328 }
 1329 
 1330 void MacroAssembler::call(Register entry) {
 1331   Assembler::call(entry);
 1332 }
 1333 
 1334 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1335   assert(rscratch != noreg || always_reachable(entry), "missing");
 1336 
 1337   if (reachable(entry)) {
 1338     Assembler::call_literal(entry.target(), entry.rspec());
 1339   } else {
 1340     lea(rscratch, entry);
 1341     Assembler::call(rscratch);
 1342   }
 1343 }
 1344 
 1345 void MacroAssembler::ic_call(address entry, jint method_index) {
 1346   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1347   movptr(rax, (intptr_t)Universe::non_oop_word());
 1348   call(AddressLiteral(entry, rh));
 1349 }
 1350 
 1351 void MacroAssembler::emit_static_call_stub() {
 1352   // Static stub relocation also tags the Method* in the code-stream.
 1353   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1354   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1355   jump(RuntimeAddress(pc()));
 1356 }
 1357 
 1358 // Implementation of call_VM versions
 1359 
 1360 void MacroAssembler::call_VM(Register oop_result,
 1361                              address entry_point,
 1362                              bool check_exceptions) {
 1363   Label C, E;
 1364   call(C, relocInfo::none);
 1365   jmp(E);
 1366 
 1367   bind(C);
 1368   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1369   ret(0);
 1370 
 1371   bind(E);
 1372 }
 1373 
 1374 void MacroAssembler::call_VM(Register oop_result,
 1375                              address entry_point,
 1376                              Register arg_1,
 1377                              bool check_exceptions) {
 1378   Label C, E;
 1379   call(C, relocInfo::none);
 1380   jmp(E);
 1381 
 1382   bind(C);
 1383   pass_arg1(this, arg_1);
 1384   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1385   ret(0);
 1386 
 1387   bind(E);
 1388 }
 1389 
 1390 void MacroAssembler::call_VM(Register oop_result,
 1391                              address entry_point,
 1392                              Register arg_1,
 1393                              Register arg_2,
 1394                              bool check_exceptions) {
 1395   Label C, E;
 1396   call(C, relocInfo::none);
 1397   jmp(E);
 1398 
 1399   bind(C);
 1400 
 1401   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1402 
 1403   pass_arg2(this, arg_2);
 1404   pass_arg1(this, arg_1);
 1405   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1406   ret(0);
 1407 
 1408   bind(E);
 1409 }
 1410 
 1411 void MacroAssembler::call_VM(Register oop_result,
 1412                              address entry_point,
 1413                              Register arg_1,
 1414                              Register arg_2,
 1415                              Register arg_3,
 1416                              bool check_exceptions) {
 1417   Label C, E;
 1418   call(C, relocInfo::none);
 1419   jmp(E);
 1420 
 1421   bind(C);
 1422 
 1423   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1424   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1425   pass_arg3(this, arg_3);
 1426   pass_arg2(this, arg_2);
 1427   pass_arg1(this, arg_1);
 1428   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1429   ret(0);
 1430 
 1431   bind(E);
 1432 }
 1433 
 1434 void MacroAssembler::call_VM(Register oop_result,
 1435                              Register last_java_sp,
 1436                              address entry_point,
 1437                              int number_of_arguments,
 1438                              bool check_exceptions) {
 1439   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1440   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1441 }
 1442 
 1443 void MacroAssembler::call_VM(Register oop_result,
 1444                              Register last_java_sp,
 1445                              address entry_point,
 1446                              Register arg_1,
 1447                              bool check_exceptions) {
 1448   pass_arg1(this, arg_1);
 1449   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1450 }
 1451 
 1452 void MacroAssembler::call_VM(Register oop_result,
 1453                              Register last_java_sp,
 1454                              address entry_point,
 1455                              Register arg_1,
 1456                              Register arg_2,
 1457                              bool check_exceptions) {
 1458 
 1459   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1460   pass_arg2(this, arg_2);
 1461   pass_arg1(this, arg_1);
 1462   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1463 }
 1464 
 1465 void MacroAssembler::call_VM(Register oop_result,
 1466                              Register last_java_sp,
 1467                              address entry_point,
 1468                              Register arg_1,
 1469                              Register arg_2,
 1470                              Register arg_3,
 1471                              bool check_exceptions) {
 1472   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1473   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1474   pass_arg3(this, arg_3);
 1475   pass_arg2(this, arg_2);
 1476   pass_arg1(this, arg_1);
 1477   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1478 }
 1479 
 1480 void MacroAssembler::super_call_VM(Register oop_result,
 1481                                    Register last_java_sp,
 1482                                    address entry_point,
 1483                                    int number_of_arguments,
 1484                                    bool check_exceptions) {
 1485   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1486   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1487 }
 1488 
 1489 void MacroAssembler::super_call_VM(Register oop_result,
 1490                                    Register last_java_sp,
 1491                                    address entry_point,
 1492                                    Register arg_1,
 1493                                    bool check_exceptions) {
 1494   pass_arg1(this, arg_1);
 1495   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1496 }
 1497 
 1498 void MacroAssembler::super_call_VM(Register oop_result,
 1499                                    Register last_java_sp,
 1500                                    address entry_point,
 1501                                    Register arg_1,
 1502                                    Register arg_2,
 1503                                    bool check_exceptions) {
 1504 
 1505   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1506   pass_arg2(this, arg_2);
 1507   pass_arg1(this, arg_1);
 1508   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1509 }
 1510 
 1511 void MacroAssembler::super_call_VM(Register oop_result,
 1512                                    Register last_java_sp,
 1513                                    address entry_point,
 1514                                    Register arg_1,
 1515                                    Register arg_2,
 1516                                    Register arg_3,
 1517                                    bool check_exceptions) {
 1518   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1519   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1520   pass_arg3(this, arg_3);
 1521   pass_arg2(this, arg_2);
 1522   pass_arg1(this, arg_1);
 1523   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1524 }
 1525 
 1526 void MacroAssembler::call_VM_base(Register oop_result,
 1527                                   Register java_thread,
 1528                                   Register last_java_sp,
 1529                                   address  entry_point,
 1530                                   int      number_of_arguments,
 1531                                   bool     check_exceptions) {
 1532   // determine java_thread register
 1533   if (!java_thread->is_valid()) {
 1534 #ifdef _LP64
 1535     java_thread = r15_thread;
 1536 #else
 1537     java_thread = rdi;
 1538     get_thread(java_thread);
 1539 #endif // LP64
 1540   }
 1541   // determine last_java_sp register
 1542   if (!last_java_sp->is_valid()) {
 1543     last_java_sp = rsp;
 1544   }
 1545   // debugging support
 1546   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1547   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1548 #ifdef ASSERT
 1549   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1550   // r12 is the heapbase.
 1551   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1552 #endif // ASSERT
 1553 
 1554   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1555   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1556 
 1557   // push java thread (becomes first argument of C function)
 1558 
 1559   NOT_LP64(push(java_thread); number_of_arguments++);
 1560   LP64_ONLY(mov(c_rarg0, r15_thread));
 1561 
 1562   // set last Java frame before call
 1563   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1564 
 1565   // Only interpreter should have to set fp
 1566   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1567 
 1568   // do the call, remove parameters
 1569   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1570 
 1571   // restore the thread (cannot use the pushed argument since arguments
 1572   // may be overwritten by C code generated by an optimizing compiler);
 1573   // however can use the register value directly if it is callee saved.
 1574   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1575     // rdi & rsi (also r15) are callee saved -> nothing to do
 1576 #ifdef ASSERT
 1577     guarantee(java_thread != rax, "change this code");
 1578     push(rax);
 1579     { Label L;
 1580       get_thread(rax);
 1581       cmpptr(java_thread, rax);
 1582       jcc(Assembler::equal, L);
 1583       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1584       bind(L);
 1585     }
 1586     pop(rax);
 1587 #endif
 1588   } else {
 1589     get_thread(java_thread);
 1590   }
 1591   // reset last Java frame
 1592   // Only interpreter should have to clear fp
 1593   reset_last_Java_frame(java_thread, true);
 1594 
 1595    // C++ interp handles this in the interpreter
 1596   check_and_handle_popframe(java_thread);
 1597   check_and_handle_earlyret(java_thread);
 1598 
 1599   if (check_exceptions) {
 1600     // check for pending exceptions (java_thread is set upon return)
 1601     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1602 #ifndef _LP64
 1603     jump_cc(Assembler::notEqual,
 1604             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1605 #else
 1606     // This used to conditionally jump to forward_exception however it is
 1607     // possible if we relocate that the branch will not reach. So we must jump
 1608     // around so we can always reach
 1609 
 1610     Label ok;
 1611     jcc(Assembler::equal, ok);
 1612     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1613     bind(ok);
 1614 #endif // LP64
 1615   }
 1616 
 1617   // get oop result if there is one and reset the value in the thread
 1618   if (oop_result->is_valid()) {
 1619     get_vm_result(oop_result, java_thread);
 1620   }
 1621 }
 1622 
 1623 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1624 
 1625   // Calculate the value for last_Java_sp
 1626   // somewhat subtle. call_VM does an intermediate call
 1627   // which places a return address on the stack just under the
 1628   // stack pointer as the user finished with it. This allows
 1629   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1630   // On 32bit we then have to push additional args on the stack to accomplish
 1631   // the actual requested call. On 64bit call_VM only can use register args
 1632   // so the only extra space is the return address that call_VM created.
 1633   // This hopefully explains the calculations here.
 1634 
 1635 #ifdef _LP64
 1636   // We've pushed one address, correct last_Java_sp
 1637   lea(rax, Address(rsp, wordSize));
 1638 #else
 1639   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1640 #endif // LP64
 1641 
 1642   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1643 
 1644 }
 1645 
 1646 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1647 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1648   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1649 }
 1650 
 1651 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1652   call_VM_leaf_base(entry_point, number_of_arguments);
 1653 }
 1654 
 1655 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1656   pass_arg0(this, arg_0);
 1657   call_VM_leaf(entry_point, 1);
 1658 }
 1659 
 1660 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1661 
 1662   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1663   pass_arg1(this, arg_1);
 1664   pass_arg0(this, arg_0);
 1665   call_VM_leaf(entry_point, 2);
 1666 }
 1667 
 1668 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1669   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1670   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1671   pass_arg2(this, arg_2);
 1672   pass_arg1(this, arg_1);
 1673   pass_arg0(this, arg_0);
 1674   call_VM_leaf(entry_point, 3);
 1675 }
 1676 
 1677 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1678   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1679   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1680   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1681   pass_arg3(this, arg_3);
 1682   pass_arg2(this, arg_2);
 1683   pass_arg1(this, arg_1);
 1684   pass_arg0(this, arg_0);
 1685   call_VM_leaf(entry_point, 3);
 1686 }
 1687 
 1688 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 1689   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1690 }
 1691 
 1692 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1693   pass_arg0(this, arg_0);
 1694   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1695 }
 1696 
 1697 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1698   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1699   pass_arg1(this, arg_1);
 1700   pass_arg0(this, arg_0);
 1701   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1702 }
 1703 
 1704 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1705   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1706   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1707   pass_arg2(this, arg_2);
 1708   pass_arg1(this, arg_1);
 1709   pass_arg0(this, arg_0);
 1710   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1711 }
 1712 
 1713 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1714   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1715   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1716   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1717   pass_arg3(this, arg_3);
 1718   pass_arg2(this, arg_2);
 1719   pass_arg1(this, arg_1);
 1720   pass_arg0(this, arg_0);
 1721   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1722 }
 1723 
 1724 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1725   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1726   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1727   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1728 }
 1729 
 1730 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1731   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1732   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1733 }
 1734 
 1735 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1736 }
 1737 
 1738 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1739 }
 1740 
 1741 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1742   assert(rscratch != noreg || always_reachable(src1), "missing");
 1743 
 1744   if (reachable(src1)) {
 1745     cmpl(as_Address(src1), imm);
 1746   } else {
 1747     lea(rscratch, src1);
 1748     cmpl(Address(rscratch, 0), imm);
 1749   }
 1750 }
 1751 
 1752 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1753   assert(!src2.is_lval(), "use cmpptr");
 1754   assert(rscratch != noreg || always_reachable(src2), "missing");
 1755 
 1756   if (reachable(src2)) {
 1757     cmpl(src1, as_Address(src2));
 1758   } else {
 1759     lea(rscratch, src2);
 1760     cmpl(src1, Address(rscratch, 0));
 1761   }
 1762 }
 1763 
 1764 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1765   Assembler::cmpl(src1, imm);
 1766 }
 1767 
 1768 void MacroAssembler::cmp32(Register src1, Address src2) {
 1769   Assembler::cmpl(src1, src2);
 1770 }
 1771 
 1772 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1773   ucomisd(opr1, opr2);
 1774 
 1775   Label L;
 1776   if (unordered_is_less) {
 1777     movl(dst, -1);
 1778     jcc(Assembler::parity, L);
 1779     jcc(Assembler::below , L);
 1780     movl(dst, 0);
 1781     jcc(Assembler::equal , L);
 1782     increment(dst);
 1783   } else { // unordered is greater
 1784     movl(dst, 1);
 1785     jcc(Assembler::parity, L);
 1786     jcc(Assembler::above , L);
 1787     movl(dst, 0);
 1788     jcc(Assembler::equal , L);
 1789     decrementl(dst);
 1790   }
 1791   bind(L);
 1792 }
 1793 
 1794 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1795   ucomiss(opr1, opr2);
 1796 
 1797   Label L;
 1798   if (unordered_is_less) {
 1799     movl(dst, -1);
 1800     jcc(Assembler::parity, L);
 1801     jcc(Assembler::below , L);
 1802     movl(dst, 0);
 1803     jcc(Assembler::equal , L);
 1804     increment(dst);
 1805   } else { // unordered is greater
 1806     movl(dst, 1);
 1807     jcc(Assembler::parity, L);
 1808     jcc(Assembler::above , L);
 1809     movl(dst, 0);
 1810     jcc(Assembler::equal , L);
 1811     decrementl(dst);
 1812   }
 1813   bind(L);
 1814 }
 1815 
 1816 
 1817 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1818   assert(rscratch != noreg || always_reachable(src1), "missing");
 1819 
 1820   if (reachable(src1)) {
 1821     cmpb(as_Address(src1), imm);
 1822   } else {
 1823     lea(rscratch, src1);
 1824     cmpb(Address(rscratch, 0), imm);
 1825   }
 1826 }
 1827 
 1828 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1829 #ifdef _LP64
 1830   assert(rscratch != noreg || always_reachable(src2), "missing");
 1831 
 1832   if (src2.is_lval()) {
 1833     movptr(rscratch, src2);
 1834     Assembler::cmpq(src1, rscratch);
 1835   } else if (reachable(src2)) {
 1836     cmpq(src1, as_Address(src2));
 1837   } else {
 1838     lea(rscratch, src2);
 1839     Assembler::cmpq(src1, Address(rscratch, 0));
 1840   }
 1841 #else
 1842   assert(rscratch == noreg, "not needed");
 1843   if (src2.is_lval()) {
 1844     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1845   } else {
 1846     cmpl(src1, as_Address(src2));
 1847   }
 1848 #endif // _LP64
 1849 }
 1850 
 1851 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1852   assert(src2.is_lval(), "not a mem-mem compare");
 1853 #ifdef _LP64
 1854   // moves src2's literal address
 1855   movptr(rscratch, src2);
 1856   Assembler::cmpq(src1, rscratch);
 1857 #else
 1858   assert(rscratch == noreg, "not needed");
 1859   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1860 #endif // _LP64
 1861 }
 1862 
 1863 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1864   cmpptr(src1, src2);
 1865 }
 1866 
 1867 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1868   cmpptr(src1, src2);
 1869 }
 1870 
 1871 #ifdef _LP64
 1872 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1873   movoop(rscratch, src2);
 1874   cmpptr(src1, rscratch);
 1875 }
 1876 #endif
 1877 
 1878 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1879   assert(rscratch != noreg || always_reachable(adr), "missing");
 1880 
 1881   if (reachable(adr)) {
 1882     lock();
 1883     cmpxchgptr(reg, as_Address(adr));
 1884   } else {
 1885     lea(rscratch, adr);
 1886     lock();
 1887     cmpxchgptr(reg, Address(rscratch, 0));
 1888   }
 1889 }
 1890 
 1891 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1892   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1893 }
 1894 
 1895 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1896   assert(rscratch != noreg || always_reachable(src), "missing");
 1897 
 1898   if (reachable(src)) {
 1899     Assembler::comisd(dst, as_Address(src));
 1900   } else {
 1901     lea(rscratch, src);
 1902     Assembler::comisd(dst, Address(rscratch, 0));
 1903   }
 1904 }
 1905 
 1906 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1907   assert(rscratch != noreg || always_reachable(src), "missing");
 1908 
 1909   if (reachable(src)) {
 1910     Assembler::comiss(dst, as_Address(src));
 1911   } else {
 1912     lea(rscratch, src);
 1913     Assembler::comiss(dst, Address(rscratch, 0));
 1914   }
 1915 }
 1916 
 1917 
 1918 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1919   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1920 
 1921   Condition negated_cond = negate_condition(cond);
 1922   Label L;
 1923   jcc(negated_cond, L);
 1924   pushf(); // Preserve flags
 1925   atomic_incl(counter_addr, rscratch);
 1926   popf();
 1927   bind(L);
 1928 }
 1929 
 1930 int MacroAssembler::corrected_idivl(Register reg) {
 1931   // Full implementation of Java idiv and irem; checks for
 1932   // special case as described in JVM spec., p.243 & p.271.
 1933   // The function returns the (pc) offset of the idivl
 1934   // instruction - may be needed for implicit exceptions.
 1935   //
 1936   //         normal case                           special case
 1937   //
 1938   // input : rax,: dividend                         min_int
 1939   //         reg: divisor   (may not be rax,/rdx)   -1
 1940   //
 1941   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1942   //         rdx: remainder (= rax, irem reg)       0
 1943   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1944   const int min_int = 0x80000000;
 1945   Label normal_case, special_case;
 1946 
 1947   // check for special case
 1948   cmpl(rax, min_int);
 1949   jcc(Assembler::notEqual, normal_case);
 1950   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1951   cmpl(reg, -1);
 1952   jcc(Assembler::equal, special_case);
 1953 
 1954   // handle normal case
 1955   bind(normal_case);
 1956   cdql();
 1957   int idivl_offset = offset();
 1958   idivl(reg);
 1959 
 1960   // normal and special case exit
 1961   bind(special_case);
 1962 
 1963   return idivl_offset;
 1964 }
 1965 
 1966 
 1967 
 1968 void MacroAssembler::decrementl(Register reg, int value) {
 1969   if (value == min_jint) {subl(reg, value) ; return; }
 1970   if (value <  0) { incrementl(reg, -value); return; }
 1971   if (value == 0) {                        ; return; }
 1972   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 1973   /* else */      { subl(reg, value)       ; return; }
 1974 }
 1975 
 1976 void MacroAssembler::decrementl(Address dst, int value) {
 1977   if (value == min_jint) {subl(dst, value) ; return; }
 1978   if (value <  0) { incrementl(dst, -value); return; }
 1979   if (value == 0) {                        ; return; }
 1980   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 1981   /* else */      { subl(dst, value)       ; return; }
 1982 }
 1983 
 1984 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 1985   assert(shift_value > 0, "illegal shift value");
 1986   Label _is_positive;
 1987   testl (reg, reg);
 1988   jcc (Assembler::positive, _is_positive);
 1989   int offset = (1 << shift_value) - 1 ;
 1990 
 1991   if (offset == 1) {
 1992     incrementl(reg);
 1993   } else {
 1994     addl(reg, offset);
 1995   }
 1996 
 1997   bind (_is_positive);
 1998   sarl(reg, shift_value);
 1999 }
 2000 
 2001 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2002   assert(rscratch != noreg || always_reachable(src), "missing");
 2003 
 2004   if (reachable(src)) {
 2005     Assembler::divsd(dst, as_Address(src));
 2006   } else {
 2007     lea(rscratch, src);
 2008     Assembler::divsd(dst, Address(rscratch, 0));
 2009   }
 2010 }
 2011 
 2012 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2013   assert(rscratch != noreg || always_reachable(src), "missing");
 2014 
 2015   if (reachable(src)) {
 2016     Assembler::divss(dst, as_Address(src));
 2017   } else {
 2018     lea(rscratch, src);
 2019     Assembler::divss(dst, Address(rscratch, 0));
 2020   }
 2021 }
 2022 
 2023 void MacroAssembler::enter() {
 2024   push(rbp);
 2025   mov(rbp, rsp);
 2026 }
 2027 
 2028 void MacroAssembler::post_call_nop() {
 2029   if (!Continuations::enabled()) {
 2030     return;
 2031   }
 2032   InstructionMark im(this);
 2033   relocate(post_call_nop_Relocation::spec());
 2034   InlineSkippedInstructionsCounter skipCounter(this);
 2035   emit_int8((uint8_t)0x0f);
 2036   emit_int8((uint8_t)0x1f);
 2037   emit_int8((uint8_t)0x84);
 2038   emit_int8((uint8_t)0x00);
 2039   emit_int32(0x00);
 2040 }
 2041 
 2042 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2043 void MacroAssembler::fat_nop() {
 2044   if (UseAddressNop) {
 2045     addr_nop_5();
 2046   } else {
 2047     emit_int8((uint8_t)0x26); // es:
 2048     emit_int8((uint8_t)0x2e); // cs:
 2049     emit_int8((uint8_t)0x64); // fs:
 2050     emit_int8((uint8_t)0x65); // gs:
 2051     emit_int8((uint8_t)0x90);
 2052   }
 2053 }
 2054 
 2055 #ifndef _LP64
 2056 void MacroAssembler::fcmp(Register tmp) {
 2057   fcmp(tmp, 1, true, true);
 2058 }
 2059 
 2060 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2061   assert(!pop_right || pop_left, "usage error");
 2062   if (VM_Version::supports_cmov()) {
 2063     assert(tmp == noreg, "unneeded temp");
 2064     if (pop_left) {
 2065       fucomip(index);
 2066     } else {
 2067       fucomi(index);
 2068     }
 2069     if (pop_right) {
 2070       fpop();
 2071     }
 2072   } else {
 2073     assert(tmp != noreg, "need temp");
 2074     if (pop_left) {
 2075       if (pop_right) {
 2076         fcompp();
 2077       } else {
 2078         fcomp(index);
 2079       }
 2080     } else {
 2081       fcom(index);
 2082     }
 2083     // convert FPU condition into eflags condition via rax,
 2084     save_rax(tmp);
 2085     fwait(); fnstsw_ax();
 2086     sahf();
 2087     restore_rax(tmp);
 2088   }
 2089   // condition codes set as follows:
 2090   //
 2091   // CF (corresponds to C0) if x < y
 2092   // PF (corresponds to C2) if unordered
 2093   // ZF (corresponds to C3) if x = y
 2094 }
 2095 
 2096 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2097   fcmp2int(dst, unordered_is_less, 1, true, true);
 2098 }
 2099 
 2100 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2101   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2102   Label L;
 2103   if (unordered_is_less) {
 2104     movl(dst, -1);
 2105     jcc(Assembler::parity, L);
 2106     jcc(Assembler::below , L);
 2107     movl(dst, 0);
 2108     jcc(Assembler::equal , L);
 2109     increment(dst);
 2110   } else { // unordered is greater
 2111     movl(dst, 1);
 2112     jcc(Assembler::parity, L);
 2113     jcc(Assembler::above , L);
 2114     movl(dst, 0);
 2115     jcc(Assembler::equal , L);
 2116     decrementl(dst);
 2117   }
 2118   bind(L);
 2119 }
 2120 
 2121 void MacroAssembler::fld_d(AddressLiteral src) {
 2122   fld_d(as_Address(src));
 2123 }
 2124 
 2125 void MacroAssembler::fld_s(AddressLiteral src) {
 2126   fld_s(as_Address(src));
 2127 }
 2128 
 2129 void MacroAssembler::fldcw(AddressLiteral src) {
 2130   fldcw(as_Address(src));
 2131 }
 2132 
 2133 void MacroAssembler::fpop() {
 2134   ffree();
 2135   fincstp();
 2136 }
 2137 
 2138 void MacroAssembler::fremr(Register tmp) {
 2139   save_rax(tmp);
 2140   { Label L;
 2141     bind(L);
 2142     fprem();
 2143     fwait(); fnstsw_ax();
 2144     sahf();
 2145     jcc(Assembler::parity, L);
 2146   }
 2147   restore_rax(tmp);
 2148   // Result is in ST0.
 2149   // Note: fxch & fpop to get rid of ST1
 2150   // (otherwise FPU stack could overflow eventually)
 2151   fxch(1);
 2152   fpop();
 2153 }
 2154 
 2155 void MacroAssembler::empty_FPU_stack() {
 2156   if (VM_Version::supports_mmx()) {
 2157     emms();
 2158   } else {
 2159     for (int i = 8; i-- > 0; ) ffree(i);
 2160   }
 2161 }
 2162 #endif // !LP64
 2163 
 2164 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2165   assert(rscratch != noreg || always_reachable(src), "missing");
 2166   if (reachable(src)) {
 2167     Assembler::mulpd(dst, as_Address(src));
 2168   } else {
 2169     lea(rscratch, src);
 2170     Assembler::mulpd(dst, Address(rscratch, 0));
 2171   }
 2172 }
 2173 
 2174 void MacroAssembler::load_float(Address src) {
 2175 #ifdef _LP64
 2176   movflt(xmm0, src);
 2177 #else
 2178   if (UseSSE >= 1) {
 2179     movflt(xmm0, src);
 2180   } else {
 2181     fld_s(src);
 2182   }
 2183 #endif // LP64
 2184 }
 2185 
 2186 void MacroAssembler::store_float(Address dst) {
 2187 #ifdef _LP64
 2188   movflt(dst, xmm0);
 2189 #else
 2190   if (UseSSE >= 1) {
 2191     movflt(dst, xmm0);
 2192   } else {
 2193     fstp_s(dst);
 2194   }
 2195 #endif // LP64
 2196 }
 2197 
 2198 void MacroAssembler::load_double(Address src) {
 2199 #ifdef _LP64
 2200   movdbl(xmm0, src);
 2201 #else
 2202   if (UseSSE >= 2) {
 2203     movdbl(xmm0, src);
 2204   } else {
 2205     fld_d(src);
 2206   }
 2207 #endif // LP64
 2208 }
 2209 
 2210 void MacroAssembler::store_double(Address dst) {
 2211 #ifdef _LP64
 2212   movdbl(dst, xmm0);
 2213 #else
 2214   if (UseSSE >= 2) {
 2215     movdbl(dst, xmm0);
 2216   } else {
 2217     fstp_d(dst);
 2218   }
 2219 #endif // LP64
 2220 }
 2221 
 2222 // dst = c = a * b + c
 2223 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2224   Assembler::vfmadd231sd(c, a, b);
 2225   if (dst != c) {
 2226     movdbl(dst, c);
 2227   }
 2228 }
 2229 
 2230 // dst = c = a * b + c
 2231 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2232   Assembler::vfmadd231ss(c, a, b);
 2233   if (dst != c) {
 2234     movflt(dst, c);
 2235   }
 2236 }
 2237 
 2238 // dst = c = a * b + c
 2239 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2240   Assembler::vfmadd231pd(c, a, b, vector_len);
 2241   if (dst != c) {
 2242     vmovdqu(dst, c);
 2243   }
 2244 }
 2245 
 2246 // dst = c = a * b + c
 2247 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2248   Assembler::vfmadd231ps(c, a, b, vector_len);
 2249   if (dst != c) {
 2250     vmovdqu(dst, c);
 2251   }
 2252 }
 2253 
 2254 // dst = c = a * b + c
 2255 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2256   Assembler::vfmadd231pd(c, a, b, vector_len);
 2257   if (dst != c) {
 2258     vmovdqu(dst, c);
 2259   }
 2260 }
 2261 
 2262 // dst = c = a * b + c
 2263 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2264   Assembler::vfmadd231ps(c, a, b, vector_len);
 2265   if (dst != c) {
 2266     vmovdqu(dst, c);
 2267   }
 2268 }
 2269 
 2270 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2271   assert(rscratch != noreg || always_reachable(dst), "missing");
 2272 
 2273   if (reachable(dst)) {
 2274     incrementl(as_Address(dst));
 2275   } else {
 2276     lea(rscratch, dst);
 2277     incrementl(Address(rscratch, 0));
 2278   }
 2279 }
 2280 
 2281 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2282   incrementl(as_Address(dst, rscratch));
 2283 }
 2284 
 2285 void MacroAssembler::incrementl(Register reg, int value) {
 2286   if (value == min_jint) {addl(reg, value) ; return; }
 2287   if (value <  0) { decrementl(reg, -value); return; }
 2288   if (value == 0) {                        ; return; }
 2289   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2290   /* else */      { addl(reg, value)       ; return; }
 2291 }
 2292 
 2293 void MacroAssembler::incrementl(Address dst, int value) {
 2294   if (value == min_jint) {addl(dst, value) ; return; }
 2295   if (value <  0) { decrementl(dst, -value); return; }
 2296   if (value == 0) {                        ; return; }
 2297   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2298   /* else */      { addl(dst, value)       ; return; }
 2299 }
 2300 
 2301 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2302   assert(rscratch != noreg || always_reachable(dst), "missing");
 2303 
 2304   if (reachable(dst)) {
 2305     jmp_literal(dst.target(), dst.rspec());
 2306   } else {
 2307     lea(rscratch, dst);
 2308     jmp(rscratch);
 2309   }
 2310 }
 2311 
 2312 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2313   assert(rscratch != noreg || always_reachable(dst), "missing");
 2314 
 2315   if (reachable(dst)) {
 2316     InstructionMark im(this);
 2317     relocate(dst.reloc());
 2318     const int short_size = 2;
 2319     const int long_size = 6;
 2320     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2321     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2322       // 0111 tttn #8-bit disp
 2323       emit_int8(0x70 | cc);
 2324       emit_int8((offs - short_size) & 0xFF);
 2325     } else {
 2326       // 0000 1111 1000 tttn #32-bit disp
 2327       emit_int8(0x0F);
 2328       emit_int8((unsigned char)(0x80 | cc));
 2329       emit_int32(offs - long_size);
 2330     }
 2331   } else {
 2332 #ifdef ASSERT
 2333     warning("reversing conditional branch");
 2334 #endif /* ASSERT */
 2335     Label skip;
 2336     jccb(reverse[cc], skip);
 2337     lea(rscratch, dst);
 2338     Assembler::jmp(rscratch);
 2339     bind(skip);
 2340   }
 2341 }
 2342 
 2343 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2344   assert(rscratch != noreg || always_reachable(src), "missing");
 2345 
 2346   if (reachable(src)) {
 2347     Assembler::ldmxcsr(as_Address(src));
 2348   } else {
 2349     lea(rscratch, src);
 2350     Assembler::ldmxcsr(Address(rscratch, 0));
 2351   }
 2352 }
 2353 
 2354 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2355   int off;
 2356   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2357     off = offset();
 2358     movsbl(dst, src); // movsxb
 2359   } else {
 2360     off = load_unsigned_byte(dst, src);
 2361     shll(dst, 24);
 2362     sarl(dst, 24);
 2363   }
 2364   return off;
 2365 }
 2366 
 2367 // Note: load_signed_short used to be called load_signed_word.
 2368 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2369 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2370 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2371 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2372   int off;
 2373   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2374     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2375     // version but this is what 64bit has always done. This seems to imply
 2376     // that users are only using 32bits worth.
 2377     off = offset();
 2378     movswl(dst, src); // movsxw
 2379   } else {
 2380     off = load_unsigned_short(dst, src);
 2381     shll(dst, 16);
 2382     sarl(dst, 16);
 2383   }
 2384   return off;
 2385 }
 2386 
 2387 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2388   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2389   // and "3.9 Partial Register Penalties", p. 22).
 2390   int off;
 2391   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2392     off = offset();
 2393     movzbl(dst, src); // movzxb
 2394   } else {
 2395     xorl(dst, dst);
 2396     off = offset();
 2397     movb(dst, src);
 2398   }
 2399   return off;
 2400 }
 2401 
 2402 // Note: load_unsigned_short used to be called load_unsigned_word.
 2403 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2404   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2405   // and "3.9 Partial Register Penalties", p. 22).
 2406   int off;
 2407   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2408     off = offset();
 2409     movzwl(dst, src); // movzxw
 2410   } else {
 2411     xorl(dst, dst);
 2412     off = offset();
 2413     movw(dst, src);
 2414   }
 2415   return off;
 2416 }
 2417 
 2418 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2419   switch (size_in_bytes) {
 2420 #ifndef _LP64
 2421   case  8:
 2422     assert(dst2 != noreg, "second dest register required");
 2423     movl(dst,  src);
 2424     movl(dst2, src.plus_disp(BytesPerInt));
 2425     break;
 2426 #else
 2427   case  8:  movq(dst, src); break;
 2428 #endif
 2429   case  4:  movl(dst, src); break;
 2430   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2431   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2432   default:  ShouldNotReachHere();
 2433   }
 2434 }
 2435 
 2436 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2437   switch (size_in_bytes) {
 2438 #ifndef _LP64
 2439   case  8:
 2440     assert(src2 != noreg, "second source register required");
 2441     movl(dst,                        src);
 2442     movl(dst.plus_disp(BytesPerInt), src2);
 2443     break;
 2444 #else
 2445   case  8:  movq(dst, src); break;
 2446 #endif
 2447   case  4:  movl(dst, src); break;
 2448   case  2:  movw(dst, src); break;
 2449   case  1:  movb(dst, src); break;
 2450   default:  ShouldNotReachHere();
 2451   }
 2452 }
 2453 
 2454 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2455   assert(rscratch != noreg || always_reachable(dst), "missing");
 2456 
 2457   if (reachable(dst)) {
 2458     movl(as_Address(dst), src);
 2459   } else {
 2460     lea(rscratch, dst);
 2461     movl(Address(rscratch, 0), src);
 2462   }
 2463 }
 2464 
 2465 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2466   if (reachable(src)) {
 2467     movl(dst, as_Address(src));
 2468   } else {
 2469     lea(dst, src);
 2470     movl(dst, Address(dst, 0));
 2471   }
 2472 }
 2473 
 2474 // C++ bool manipulation
 2475 
 2476 void MacroAssembler::movbool(Register dst, Address src) {
 2477   if(sizeof(bool) == 1)
 2478     movb(dst, src);
 2479   else if(sizeof(bool) == 2)
 2480     movw(dst, src);
 2481   else if(sizeof(bool) == 4)
 2482     movl(dst, src);
 2483   else
 2484     // unsupported
 2485     ShouldNotReachHere();
 2486 }
 2487 
 2488 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2489   if(sizeof(bool) == 1)
 2490     movb(dst, (int) boolconst);
 2491   else if(sizeof(bool) == 2)
 2492     movw(dst, (int) boolconst);
 2493   else if(sizeof(bool) == 4)
 2494     movl(dst, (int) boolconst);
 2495   else
 2496     // unsupported
 2497     ShouldNotReachHere();
 2498 }
 2499 
 2500 void MacroAssembler::movbool(Address dst, Register src) {
 2501   if(sizeof(bool) == 1)
 2502     movb(dst, src);
 2503   else if(sizeof(bool) == 2)
 2504     movw(dst, src);
 2505   else if(sizeof(bool) == 4)
 2506     movl(dst, src);
 2507   else
 2508     // unsupported
 2509     ShouldNotReachHere();
 2510 }
 2511 
 2512 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2513   assert(rscratch != noreg || always_reachable(src), "missing");
 2514 
 2515   if (reachable(src)) {
 2516     movdl(dst, as_Address(src));
 2517   } else {
 2518     lea(rscratch, src);
 2519     movdl(dst, Address(rscratch, 0));
 2520   }
 2521 }
 2522 
 2523 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2524   assert(rscratch != noreg || always_reachable(src), "missing");
 2525 
 2526   if (reachable(src)) {
 2527     movq(dst, as_Address(src));
 2528   } else {
 2529     lea(rscratch, src);
 2530     movq(dst, Address(rscratch, 0));
 2531   }
 2532 }
 2533 
 2534 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2535   assert(rscratch != noreg || always_reachable(src), "missing");
 2536 
 2537   if (reachable(src)) {
 2538     if (UseXmmLoadAndClearUpper) {
 2539       movsd (dst, as_Address(src));
 2540     } else {
 2541       movlpd(dst, as_Address(src));
 2542     }
 2543   } else {
 2544     lea(rscratch, src);
 2545     if (UseXmmLoadAndClearUpper) {
 2546       movsd (dst, Address(rscratch, 0));
 2547     } else {
 2548       movlpd(dst, Address(rscratch, 0));
 2549     }
 2550   }
 2551 }
 2552 
 2553 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2554   assert(rscratch != noreg || always_reachable(src), "missing");
 2555 
 2556   if (reachable(src)) {
 2557     movss(dst, as_Address(src));
 2558   } else {
 2559     lea(rscratch, src);
 2560     movss(dst, Address(rscratch, 0));
 2561   }
 2562 }
 2563 
 2564 void MacroAssembler::movptr(Register dst, Register src) {
 2565   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2566 }
 2567 
 2568 void MacroAssembler::movptr(Register dst, Address src) {
 2569   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2570 }
 2571 
 2572 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2573 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2574   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
 2575 }
 2576 
 2577 void MacroAssembler::movptr(Address dst, Register src) {
 2578   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2579 }
 2580 
 2581 void MacroAssembler::movptr(Address dst, int32_t src) {
 2582   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2583 }
 2584 
 2585 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2586   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2587   Assembler::movdqu(dst, src);
 2588 }
 2589 
 2590 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2591   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2592   Assembler::movdqu(dst, src);
 2593 }
 2594 
 2595 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2596   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2597   Assembler::movdqu(dst, src);
 2598 }
 2599 
 2600 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2601   assert(rscratch != noreg || always_reachable(src), "missing");
 2602 
 2603   if (reachable(src)) {
 2604     movdqu(dst, as_Address(src));
 2605   } else {
 2606     lea(rscratch, src);
 2607     movdqu(dst, Address(rscratch, 0));
 2608   }
 2609 }
 2610 
 2611 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2612   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2613   Assembler::vmovdqu(dst, src);
 2614 }
 2615 
 2616 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2617   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2618   Assembler::vmovdqu(dst, src);
 2619 }
 2620 
 2621 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2622   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2623   Assembler::vmovdqu(dst, src);
 2624 }
 2625 
 2626 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2627   assert(rscratch != noreg || always_reachable(src), "missing");
 2628 
 2629   if (reachable(src)) {
 2630     vmovdqu(dst, as_Address(src));
 2631   }
 2632   else {
 2633     lea(rscratch, src);
 2634     vmovdqu(dst, Address(rscratch, 0));
 2635   }
 2636 }
 2637 
 2638 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2639   assert(rscratch != noreg || always_reachable(src), "missing");
 2640 
 2641   if (vector_len == AVX_512bit) {
 2642     evmovdquq(dst, src, AVX_512bit, rscratch);
 2643   } else if (vector_len == AVX_256bit) {
 2644     vmovdqu(dst, src, rscratch);
 2645   } else {
 2646     movdqu(dst, src, rscratch);
 2647   }
 2648 }
 2649 
 2650 void MacroAssembler::kmov(KRegister dst, Address src) {
 2651   if (VM_Version::supports_avx512bw()) {
 2652     kmovql(dst, src);
 2653   } else {
 2654     assert(VM_Version::supports_evex(), "");
 2655     kmovwl(dst, src);
 2656   }
 2657 }
 2658 
 2659 void MacroAssembler::kmov(Address dst, KRegister src) {
 2660   if (VM_Version::supports_avx512bw()) {
 2661     kmovql(dst, src);
 2662   } else {
 2663     assert(VM_Version::supports_evex(), "");
 2664     kmovwl(dst, src);
 2665   }
 2666 }
 2667 
 2668 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2669   if (VM_Version::supports_avx512bw()) {
 2670     kmovql(dst, src);
 2671   } else {
 2672     assert(VM_Version::supports_evex(), "");
 2673     kmovwl(dst, src);
 2674   }
 2675 }
 2676 
 2677 void MacroAssembler::kmov(Register dst, KRegister src) {
 2678   if (VM_Version::supports_avx512bw()) {
 2679     kmovql(dst, src);
 2680   } else {
 2681     assert(VM_Version::supports_evex(), "");
 2682     kmovwl(dst, src);
 2683   }
 2684 }
 2685 
 2686 void MacroAssembler::kmov(KRegister dst, Register src) {
 2687   if (VM_Version::supports_avx512bw()) {
 2688     kmovql(dst, src);
 2689   } else {
 2690     assert(VM_Version::supports_evex(), "");
 2691     kmovwl(dst, src);
 2692   }
 2693 }
 2694 
 2695 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2696   assert(rscratch != noreg || always_reachable(src), "missing");
 2697 
 2698   if (reachable(src)) {
 2699     kmovql(dst, as_Address(src));
 2700   } else {
 2701     lea(rscratch, src);
 2702     kmovql(dst, Address(rscratch, 0));
 2703   }
 2704 }
 2705 
 2706 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2707   assert(rscratch != noreg || always_reachable(src), "missing");
 2708 
 2709   if (reachable(src)) {
 2710     kmovwl(dst, as_Address(src));
 2711   } else {
 2712     lea(rscratch, src);
 2713     kmovwl(dst, Address(rscratch, 0));
 2714   }
 2715 }
 2716 
 2717 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2718                                int vector_len, Register rscratch) {
 2719   assert(rscratch != noreg || always_reachable(src), "missing");
 2720 
 2721   if (reachable(src)) {
 2722     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2723   } else {
 2724     lea(rscratch, src);
 2725     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2726   }
 2727 }
 2728 
 2729 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2730                                int vector_len, Register rscratch) {
 2731   assert(rscratch != noreg || always_reachable(src), "missing");
 2732 
 2733   if (reachable(src)) {
 2734     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2735   } else {
 2736     lea(rscratch, src);
 2737     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2738   }
 2739 }
 2740 
 2741 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2742   assert(rscratch != noreg || always_reachable(src), "missing");
 2743 
 2744   if (reachable(src)) {
 2745     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2746   } else {
 2747     lea(rscratch, src);
 2748     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2749   }
 2750 }
 2751 
 2752 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2753   assert(rscratch != noreg || always_reachable(src), "missing");
 2754 
 2755   if (reachable(src)) {
 2756     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2757   } else {
 2758     lea(rscratch, src);
 2759     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2760   }
 2761 }
 2762 
 2763 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2764   assert(rscratch != noreg || always_reachable(src), "missing");
 2765 
 2766   if (reachable(src)) {
 2767     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2768   } else {
 2769     lea(rscratch, src);
 2770     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2771   }
 2772 }
 2773 
 2774 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2775   assert(rscratch != noreg || always_reachable(src), "missing");
 2776 
 2777   if (reachable(src)) {
 2778     Assembler::movdqa(dst, as_Address(src));
 2779   } else {
 2780     lea(rscratch, src);
 2781     Assembler::movdqa(dst, Address(rscratch, 0));
 2782   }
 2783 }
 2784 
 2785 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2786   assert(rscratch != noreg || always_reachable(src), "missing");
 2787 
 2788   if (reachable(src)) {
 2789     Assembler::movsd(dst, as_Address(src));
 2790   } else {
 2791     lea(rscratch, src);
 2792     Assembler::movsd(dst, Address(rscratch, 0));
 2793   }
 2794 }
 2795 
 2796 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2797   assert(rscratch != noreg || always_reachable(src), "missing");
 2798 
 2799   if (reachable(src)) {
 2800     Assembler::movss(dst, as_Address(src));
 2801   } else {
 2802     lea(rscratch, src);
 2803     Assembler::movss(dst, Address(rscratch, 0));
 2804   }
 2805 }
 2806 
 2807 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2808   assert(rscratch != noreg || always_reachable(src), "missing");
 2809 
 2810   if (reachable(src)) {
 2811     Assembler::movddup(dst, as_Address(src));
 2812   } else {
 2813     lea(rscratch, src);
 2814     Assembler::movddup(dst, Address(rscratch, 0));
 2815   }
 2816 }
 2817 
 2818 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2819   assert(rscratch != noreg || always_reachable(src), "missing");
 2820 
 2821   if (reachable(src)) {
 2822     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2823   } else {
 2824     lea(rscratch, src);
 2825     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2826   }
 2827 }
 2828 
 2829 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2830   assert(rscratch != noreg || always_reachable(src), "missing");
 2831 
 2832   if (reachable(src)) {
 2833     Assembler::mulsd(dst, as_Address(src));
 2834   } else {
 2835     lea(rscratch, src);
 2836     Assembler::mulsd(dst, Address(rscratch, 0));
 2837   }
 2838 }
 2839 
 2840 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2841   assert(rscratch != noreg || always_reachable(src), "missing");
 2842 
 2843   if (reachable(src)) {
 2844     Assembler::mulss(dst, as_Address(src));
 2845   } else {
 2846     lea(rscratch, src);
 2847     Assembler::mulss(dst, Address(rscratch, 0));
 2848   }
 2849 }
 2850 
 2851 void MacroAssembler::null_check(Register reg, int offset) {
 2852   if (needs_explicit_null_check(offset)) {
 2853     // provoke OS null exception if reg is null by
 2854     // accessing M[reg] w/o changing any (non-CC) registers
 2855     // NOTE: cmpl is plenty here to provoke a segv
 2856     cmpptr(rax, Address(reg, 0));
 2857     // Note: should probably use testl(rax, Address(reg, 0));
 2858     //       may be shorter code (however, this version of
 2859     //       testl needs to be implemented first)
 2860   } else {
 2861     // nothing to do, (later) access of M[reg + offset]
 2862     // will provoke OS null exception if reg is null
 2863   }
 2864 }
 2865 
 2866 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
 2867   andptr(markword, markWord::inline_type_mask_in_place);
 2868   cmpptr(markword, markWord::inline_type_pattern);
 2869   jcc(Assembler::equal, is_inline_type);
 2870 }
 2871 
 2872 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
 2873   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
 2874   testl(temp_reg, JVM_ACC_VALUE);
 2875   jcc(Assembler::notZero, is_inline_type);
 2876 }
 2877 
 2878 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
 2879   testptr(object, object);
 2880   jcc(Assembler::zero, not_inline_type);
 2881   const int is_inline_type_mask = markWord::inline_type_pattern;
 2882   movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
 2883   andptr(tmp, is_inline_type_mask);
 2884   cmpptr(tmp, is_inline_type_mask);
 2885   jcc(Assembler::notEqual, not_inline_type);
 2886 }
 2887 
 2888 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
 2889 #ifdef ASSERT
 2890   {
 2891     Label done_check;
 2892     test_klass_is_inline_type(klass, temp_reg, done_check);
 2893     stop("test_klass_is_empty_inline_type with non inline type klass");
 2894     bind(done_check);
 2895   }
 2896 #endif
 2897   movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
 2898   testl(temp_reg, InstanceKlassFlags::is_empty_inline_type_value());
 2899   jcc(Assembler::notZero, is_empty_inline_type);
 2900 }
 2901 
 2902 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
 2903   movl(temp_reg, flags);
 2904   shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
 2905   andl(temp_reg, 0x1);
 2906   testl(temp_reg, temp_reg);
 2907   jcc(Assembler::notZero, is_null_free_inline_type);
 2908 }
 2909 
 2910 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
 2911   movl(temp_reg, flags);
 2912   shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
 2913   andl(temp_reg, 0x1);
 2914   testl(temp_reg, temp_reg);
 2915   jcc(Assembler::zero, not_null_free_inline_type);
 2916 }
 2917 
 2918 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
 2919   movl(temp_reg, flags);
 2920   shrl(temp_reg, ConstantPoolCacheEntry::is_flat_shift);
 2921   andl(temp_reg, 0x1);
 2922   testl(temp_reg, temp_reg);
 2923   jcc(Assembler::notZero, is_flat);
 2924 }
 2925 
 2926 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
 2927   Label test_mark_word;
 2928   // load mark word
 2929   movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
 2930   // check displaced
 2931   testl(temp_reg, markWord::unlocked_value);
 2932   jccb(Assembler::notZero, test_mark_word);
 2933   // slow path use klass prototype
 2934   push(rscratch1);
 2935   load_prototype_header(temp_reg, oop, rscratch1);
 2936   pop(rscratch1);
 2937 
 2938   bind(test_mark_word);
 2939   testl(temp_reg, test_bit);
 2940   jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
 2941 }
 2942 
 2943 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
 2944                                          Label& is_flat_array) {
 2945 #ifdef _LP64
 2946   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
 2947 #else
 2948   load_klass(temp_reg, oop, noreg);
 2949   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2950   test_flat_array_layout(temp_reg, is_flat_array);
 2951 #endif
 2952 }
 2953 
 2954 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg,
 2955                                              Label& is_non_flat_array) {
 2956 #ifdef _LP64
 2957   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
 2958 #else
 2959   load_klass(temp_reg, oop, noreg);
 2960   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2961   test_non_flat_array_layout(temp_reg, is_non_flat_array);
 2962 #endif
 2963 }
 2964 
 2965 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
 2966 #ifdef _LP64
 2967   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
 2968 #else
 2969   load_klass(temp_reg, oop, noreg);
 2970   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2971   test_null_free_array_layout(temp_reg, is_null_free_array);
 2972 #endif
 2973 }
 2974 
 2975 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
 2976 #ifdef _LP64
 2977   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
 2978 #else
 2979   load_klass(temp_reg, oop, noreg);
 2980   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2981   test_non_null_free_array_layout(temp_reg, is_non_null_free_array);
 2982 #endif
 2983 }
 2984 
 2985 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
 2986   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 2987   jcc(Assembler::notZero, is_flat_array);
 2988 }
 2989 
 2990 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) {
 2991   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 2992   jcc(Assembler::zero, is_non_flat_array);
 2993 }
 2994 
 2995 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
 2996   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 2997   jcc(Assembler::notZero, is_null_free_array);
 2998 }
 2999 
 3000 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
 3001   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 3002   jcc(Assembler::zero, is_non_null_free_array);
 3003 }
 3004 
 3005 
 3006 void MacroAssembler::os_breakpoint() {
 3007   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 3008   // (e.g., MSVC can't call ps() otherwise)
 3009   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 3010 }
 3011 
 3012 void MacroAssembler::unimplemented(const char* what) {
 3013   const char* buf = nullptr;
 3014   {
 3015     ResourceMark rm;
 3016     stringStream ss;
 3017     ss.print("unimplemented: %s", what);
 3018     buf = code_string(ss.as_string());
 3019   }
 3020   stop(buf);
 3021 }
 3022 
 3023 #ifdef _LP64
 3024 #define XSTATE_BV 0x200
 3025 #endif
 3026 
 3027 void MacroAssembler::pop_CPU_state() {
 3028   pop_FPU_state();
 3029   pop_IU_state();
 3030 }
 3031 
 3032 void MacroAssembler::pop_FPU_state() {
 3033 #ifndef _LP64
 3034   frstor(Address(rsp, 0));
 3035 #else
 3036   fxrstor(Address(rsp, 0));
 3037 #endif
 3038   addptr(rsp, FPUStateSizeInWords * wordSize);
 3039 }
 3040 
 3041 void MacroAssembler::pop_IU_state() {
 3042   popa();
 3043   LP64_ONLY(addq(rsp, 8));
 3044   popf();
 3045 }
 3046 
 3047 // Save Integer and Float state
 3048 // Warning: Stack must be 16 byte aligned (64bit)
 3049 void MacroAssembler::push_CPU_state() {
 3050   push_IU_state();
 3051   push_FPU_state();
 3052 }
 3053 
 3054 void MacroAssembler::push_FPU_state() {
 3055   subptr(rsp, FPUStateSizeInWords * wordSize);
 3056 #ifndef _LP64
 3057   fnsave(Address(rsp, 0));
 3058   fwait();
 3059 #else
 3060   fxsave(Address(rsp, 0));
 3061 #endif // LP64
 3062 }
 3063 
 3064 void MacroAssembler::push_IU_state() {
 3065   // Push flags first because pusha kills them
 3066   pushf();
 3067   // Make sure rsp stays 16-byte aligned
 3068   LP64_ONLY(subq(rsp, 8));
 3069   pusha();
 3070 }
 3071 
 3072 void MacroAssembler::push_cont_fastpath() {
 3073   if (!Continuations::enabled()) return;
 3074 
 3075 #ifndef _LP64
 3076   Register rthread = rax;
 3077   Register rrealsp = rbx;
 3078   push(rthread);
 3079   push(rrealsp);
 3080 
 3081   get_thread(rthread);
 3082 
 3083   // The code below wants the original RSP.
 3084   // Move it back after the pushes above.
 3085   movptr(rrealsp, rsp);
 3086   addptr(rrealsp, 2*wordSize);
 3087 #else
 3088   Register rthread = r15_thread;
 3089   Register rrealsp = rsp;
 3090 #endif
 3091 
 3092   Label done;
 3093   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3094   jccb(Assembler::belowEqual, done);
 3095   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3096   bind(done);
 3097 
 3098 #ifndef _LP64
 3099   pop(rrealsp);
 3100   pop(rthread);
 3101 #endif
 3102 }
 3103 
 3104 void MacroAssembler::pop_cont_fastpath() {
 3105   if (!Continuations::enabled()) return;
 3106 
 3107 #ifndef _LP64
 3108   Register rthread = rax;
 3109   Register rrealsp = rbx;
 3110   push(rthread);
 3111   push(rrealsp);
 3112 
 3113   get_thread(rthread);
 3114 
 3115   // The code below wants the original RSP.
 3116   // Move it back after the pushes above.
 3117   movptr(rrealsp, rsp);
 3118   addptr(rrealsp, 2*wordSize);
 3119 #else
 3120   Register rthread = r15_thread;
 3121   Register rrealsp = rsp;
 3122 #endif
 3123 
 3124   Label done;
 3125   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3126   jccb(Assembler::below, done);
 3127   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3128   bind(done);
 3129 
 3130 #ifndef _LP64
 3131   pop(rrealsp);
 3132   pop(rthread);
 3133 #endif
 3134 }
 3135 
 3136 void MacroAssembler::inc_held_monitor_count() {
 3137 #ifndef _LP64
 3138   Register thread = rax;
 3139   push(thread);
 3140   get_thread(thread);
 3141   incrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3142   pop(thread);
 3143 #else // LP64
 3144   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3145 #endif
 3146 }
 3147 
 3148 void MacroAssembler::dec_held_monitor_count() {
 3149 #ifndef _LP64
 3150   Register thread = rax;
 3151   push(thread);
 3152   get_thread(thread);
 3153   decrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3154   pop(thread);
 3155 #else // LP64
 3156   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3157 #endif
 3158 }
 3159 
 3160 #ifdef ASSERT
 3161 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3162 #ifdef _LP64
 3163   Label no_cont;
 3164   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3165   testl(cont, cont);
 3166   jcc(Assembler::zero, no_cont);
 3167   stop(name);
 3168   bind(no_cont);
 3169 #else
 3170   Unimplemented();
 3171 #endif
 3172 }
 3173 #endif
 3174 
 3175 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3176   if (!java_thread->is_valid()) {
 3177     java_thread = rdi;
 3178     get_thread(java_thread);
 3179   }
 3180   // we must set sp to zero to clear frame
 3181   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3182   // must clear fp, so that compiled frames are not confused; it is
 3183   // possible that we need it only for debugging
 3184   if (clear_fp) {
 3185     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3186   }
 3187   // Always clear the pc because it could have been set by make_walkable()
 3188   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3189   vzeroupper();
 3190 }
 3191 
 3192 void MacroAssembler::restore_rax(Register tmp) {
 3193   if (tmp == noreg) pop(rax);
 3194   else if (tmp != rax) mov(rax, tmp);
 3195 }
 3196 
 3197 void MacroAssembler::round_to(Register reg, int modulus) {
 3198   addptr(reg, modulus - 1);
 3199   andptr(reg, -modulus);
 3200 }
 3201 
 3202 void MacroAssembler::save_rax(Register tmp) {
 3203   if (tmp == noreg) push(rax);
 3204   else if (tmp != rax) mov(tmp, rax);
 3205 }
 3206 
 3207 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3208   if (at_return) {
 3209     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3210     // we may safely use rsp instead to perform the stack watermark check.
 3211     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3212     jcc(Assembler::above, slow_path);
 3213     return;
 3214   }
 3215   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3216   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3217 }
 3218 
 3219 // Calls to C land
 3220 //
 3221 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3222 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3223 // has to be reset to 0. This is required to allow proper stack traversal.
 3224 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3225                                          Register last_java_sp,
 3226                                          Register last_java_fp,
 3227                                          address  last_java_pc,
 3228                                          Register rscratch) {
 3229   vzeroupper();
 3230   // determine java_thread register
 3231   if (!java_thread->is_valid()) {
 3232     java_thread = rdi;
 3233     get_thread(java_thread);
 3234   }
 3235   // determine last_java_sp register
 3236   if (!last_java_sp->is_valid()) {
 3237     last_java_sp = rsp;
 3238   }
 3239   // last_java_fp is optional
 3240   if (last_java_fp->is_valid()) {
 3241     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3242   }
 3243   // last_java_pc is optional
 3244   if (last_java_pc != nullptr) {
 3245     Address java_pc(java_thread,
 3246                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3247     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3248   }
 3249   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3250 }
 3251 
 3252 void MacroAssembler::shlptr(Register dst, int imm8) {
 3253   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3254 }
 3255 
 3256 void MacroAssembler::shrptr(Register dst, int imm8) {
 3257   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3258 }
 3259 
 3260 void MacroAssembler::sign_extend_byte(Register reg) {
 3261   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3262     movsbl(reg, reg); // movsxb
 3263   } else {
 3264     shll(reg, 24);
 3265     sarl(reg, 24);
 3266   }
 3267 }
 3268 
 3269 void MacroAssembler::sign_extend_short(Register reg) {
 3270   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3271     movswl(reg, reg); // movsxw
 3272   } else {
 3273     shll(reg, 16);
 3274     sarl(reg, 16);
 3275   }
 3276 }
 3277 
 3278 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3279   if (imm32 >= 0 && is8bit(imm32)) {
 3280     testb(dst, imm32);
 3281   } else {
 3282     Assembler::testl(dst, imm32);
 3283   }
 3284 }
 3285 
 3286 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3287   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3288     testb(dst, imm32);
 3289   } else {
 3290     Assembler::testl(dst, imm32);
 3291   }
 3292 }
 3293 
 3294 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3295   assert(always_reachable(src), "Address should be reachable");
 3296   testl(dst, as_Address(src));
 3297 }
 3298 
 3299 #ifdef _LP64
 3300 
 3301 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3302   if (imm32 >= 0) {
 3303     testl(dst, imm32);
 3304   } else {
 3305     Assembler::testq(dst, imm32);
 3306   }
 3307 }
 3308 
 3309 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3310   if (imm32 >= 0) {
 3311     testl(dst, imm32);
 3312   } else {
 3313     Assembler::testq(dst, imm32);
 3314   }
 3315 }
 3316 
 3317 #endif
 3318 
 3319 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3320   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3321   Assembler::pcmpeqb(dst, src);
 3322 }
 3323 
 3324 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3325   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3326   Assembler::pcmpeqw(dst, src);
 3327 }
 3328 
 3329 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3330   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3331   Assembler::pcmpestri(dst, src, imm8);
 3332 }
 3333 
 3334 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3335   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3336   Assembler::pcmpestri(dst, src, imm8);
 3337 }
 3338 
 3339 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3340   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3341   Assembler::pmovzxbw(dst, src);
 3342 }
 3343 
 3344 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3345   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3346   Assembler::pmovzxbw(dst, src);
 3347 }
 3348 
 3349 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3350   assert((src->encoding() < 16),"XMM register should be 0-15");
 3351   Assembler::pmovmskb(dst, src);
 3352 }
 3353 
 3354 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3355   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3356   Assembler::ptest(dst, src);
 3357 }
 3358 
 3359 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3360   assert(rscratch != noreg || always_reachable(src), "missing");
 3361 
 3362   if (reachable(src)) {
 3363     Assembler::sqrtss(dst, as_Address(src));
 3364   } else {
 3365     lea(rscratch, src);
 3366     Assembler::sqrtss(dst, Address(rscratch, 0));
 3367   }
 3368 }
 3369 
 3370 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3371   assert(rscratch != noreg || always_reachable(src), "missing");
 3372 
 3373   if (reachable(src)) {
 3374     Assembler::subsd(dst, as_Address(src));
 3375   } else {
 3376     lea(rscratch, src);
 3377     Assembler::subsd(dst, Address(rscratch, 0));
 3378   }
 3379 }
 3380 
 3381 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3382   assert(rscratch != noreg || always_reachable(src), "missing");
 3383 
 3384   if (reachable(src)) {
 3385     Assembler::roundsd(dst, as_Address(src), rmode);
 3386   } else {
 3387     lea(rscratch, src);
 3388     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3389   }
 3390 }
 3391 
 3392 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3393   assert(rscratch != noreg || always_reachable(src), "missing");
 3394 
 3395   if (reachable(src)) {
 3396     Assembler::subss(dst, as_Address(src));
 3397   } else {
 3398     lea(rscratch, src);
 3399     Assembler::subss(dst, Address(rscratch, 0));
 3400   }
 3401 }
 3402 
 3403 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3404   assert(rscratch != noreg || always_reachable(src), "missing");
 3405 
 3406   if (reachable(src)) {
 3407     Assembler::ucomisd(dst, as_Address(src));
 3408   } else {
 3409     lea(rscratch, src);
 3410     Assembler::ucomisd(dst, Address(rscratch, 0));
 3411   }
 3412 }
 3413 
 3414 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3415   assert(rscratch != noreg || always_reachable(src), "missing");
 3416 
 3417   if (reachable(src)) {
 3418     Assembler::ucomiss(dst, as_Address(src));
 3419   } else {
 3420     lea(rscratch, src);
 3421     Assembler::ucomiss(dst, Address(rscratch, 0));
 3422   }
 3423 }
 3424 
 3425 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3426   assert(rscratch != noreg || always_reachable(src), "missing");
 3427 
 3428   // Used in sign-bit flipping with aligned address.
 3429   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3430   if (reachable(src)) {
 3431     Assembler::xorpd(dst, as_Address(src));
 3432   } else {
 3433     lea(rscratch, src);
 3434     Assembler::xorpd(dst, Address(rscratch, 0));
 3435   }
 3436 }
 3437 
 3438 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3439   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3440     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3441   }
 3442   else {
 3443     Assembler::xorpd(dst, src);
 3444   }
 3445 }
 3446 
 3447 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3448   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3449     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3450   } else {
 3451     Assembler::xorps(dst, src);
 3452   }
 3453 }
 3454 
 3455 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3456   assert(rscratch != noreg || always_reachable(src), "missing");
 3457 
 3458   // Used in sign-bit flipping with aligned address.
 3459   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3460   if (reachable(src)) {
 3461     Assembler::xorps(dst, as_Address(src));
 3462   } else {
 3463     lea(rscratch, src);
 3464     Assembler::xorps(dst, Address(rscratch, 0));
 3465   }
 3466 }
 3467 
 3468 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3469   assert(rscratch != noreg || always_reachable(src), "missing");
 3470 
 3471   // Used in sign-bit flipping with aligned address.
 3472   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3473   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3474   if (reachable(src)) {
 3475     Assembler::pshufb(dst, as_Address(src));
 3476   } else {
 3477     lea(rscratch, src);
 3478     Assembler::pshufb(dst, Address(rscratch, 0));
 3479   }
 3480 }
 3481 
 3482 // AVX 3-operands instructions
 3483 
 3484 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3485   assert(rscratch != noreg || always_reachable(src), "missing");
 3486 
 3487   if (reachable(src)) {
 3488     vaddsd(dst, nds, as_Address(src));
 3489   } else {
 3490     lea(rscratch, src);
 3491     vaddsd(dst, nds, Address(rscratch, 0));
 3492   }
 3493 }
 3494 
 3495 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3496   assert(rscratch != noreg || always_reachable(src), "missing");
 3497 
 3498   if (reachable(src)) {
 3499     vaddss(dst, nds, as_Address(src));
 3500   } else {
 3501     lea(rscratch, src);
 3502     vaddss(dst, nds, Address(rscratch, 0));
 3503   }
 3504 }
 3505 
 3506 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3507   assert(UseAVX > 0, "requires some form of AVX");
 3508   assert(rscratch != noreg || always_reachable(src), "missing");
 3509 
 3510   if (reachable(src)) {
 3511     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3512   } else {
 3513     lea(rscratch, src);
 3514     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3515   }
 3516 }
 3517 
 3518 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3519   assert(UseAVX > 0, "requires some form of AVX");
 3520   assert(rscratch != noreg || always_reachable(src), "missing");
 3521 
 3522   if (reachable(src)) {
 3523     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3524   } else {
 3525     lea(rscratch, src);
 3526     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3527   }
 3528 }
 3529 
 3530 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3531   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3532   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3533 
 3534   vandps(dst, nds, negate_field, vector_len, rscratch);
 3535 }
 3536 
 3537 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3538   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3539   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3540 
 3541   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3542 }
 3543 
 3544 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3545   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3546   Assembler::vpaddb(dst, nds, src, vector_len);
 3547 }
 3548 
 3549 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3550   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3551   Assembler::vpaddb(dst, nds, src, vector_len);
 3552 }
 3553 
 3554 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3555   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3556   Assembler::vpaddw(dst, nds, src, vector_len);
 3557 }
 3558 
 3559 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3560   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3561   Assembler::vpaddw(dst, nds, src, vector_len);
 3562 }
 3563 
 3564 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3565   assert(rscratch != noreg || always_reachable(src), "missing");
 3566 
 3567   if (reachable(src)) {
 3568     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3569   } else {
 3570     lea(rscratch, src);
 3571     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3572   }
 3573 }
 3574 
 3575 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3576   assert(rscratch != noreg || always_reachable(src), "missing");
 3577 
 3578   if (reachable(src)) {
 3579     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3580   } else {
 3581     lea(rscratch, src);
 3582     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3583   }
 3584 }
 3585 
 3586 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3587   assert(rscratch != noreg || always_reachable(src), "missing");
 3588 
 3589   if (reachable(src)) {
 3590     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3591   } else {
 3592     lea(rscratch, src);
 3593     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3594   }
 3595 }
 3596 
 3597 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3598   assert(rscratch != noreg || always_reachable(src), "missing");
 3599 
 3600   if (reachable(src)) {
 3601     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3602   } else {
 3603     lea(rscratch, src);
 3604     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3605   }
 3606 }
 3607 
 3608 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3609   assert(rscratch != noreg || always_reachable(src), "missing");
 3610 
 3611   if (reachable(src)) {
 3612     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3613   } else {
 3614     lea(rscratch, src);
 3615     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3616   }
 3617 }
 3618 
 3619 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3620   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3621   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3622 }
 3623 
 3624 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3625   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3626   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3627 }
 3628 
 3629 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3630   assert(rscratch != noreg || always_reachable(src), "missing");
 3631 
 3632   if (reachable(src)) {
 3633     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3634   } else {
 3635     lea(rscratch, src);
 3636     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3637   }
 3638 }
 3639 
 3640 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3641                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3642   assert(rscratch != noreg || always_reachable(src), "missing");
 3643 
 3644   if (reachable(src)) {
 3645     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3646   } else {
 3647     lea(rscratch, src);
 3648     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3649   }
 3650 }
 3651 
 3652 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3653                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3654   assert(rscratch != noreg || always_reachable(src), "missing");
 3655 
 3656   if (reachable(src)) {
 3657     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3658   } else {
 3659     lea(rscratch, src);
 3660     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3661   }
 3662 }
 3663 
 3664 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3665                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3666   assert(rscratch != noreg || always_reachable(src), "missing");
 3667 
 3668   if (reachable(src)) {
 3669     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3670   } else {
 3671     lea(rscratch, src);
 3672     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3673   }
 3674 }
 3675 
 3676 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3677                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3678   assert(rscratch != noreg || always_reachable(src), "missing");
 3679 
 3680   if (reachable(src)) {
 3681     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3682   } else {
 3683     lea(rscratch, src);
 3684     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3685   }
 3686 }
 3687 
 3688 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3689   if (width == Assembler::Q) {
 3690     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3691   } else {
 3692     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3693   }
 3694 }
 3695 
 3696 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3697   int eq_cond_enc = 0x29;
 3698   int gt_cond_enc = 0x37;
 3699   if (width != Assembler::Q) {
 3700     eq_cond_enc = 0x74 + width;
 3701     gt_cond_enc = 0x64 + width;
 3702   }
 3703   switch (cond) {
 3704   case eq:
 3705     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3706     break;
 3707   case neq:
 3708     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3709     vallones(xtmp, vector_len);
 3710     vpxor(dst, xtmp, dst, vector_len);
 3711     break;
 3712   case le:
 3713     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3714     vallones(xtmp, vector_len);
 3715     vpxor(dst, xtmp, dst, vector_len);
 3716     break;
 3717   case nlt:
 3718     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3719     vallones(xtmp, vector_len);
 3720     vpxor(dst, xtmp, dst, vector_len);
 3721     break;
 3722   case lt:
 3723     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3724     break;
 3725   case nle:
 3726     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3727     break;
 3728   default:
 3729     assert(false, "Should not reach here");
 3730   }
 3731 }
 3732 
 3733 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3734   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3735   Assembler::vpmovzxbw(dst, src, vector_len);
 3736 }
 3737 
 3738 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3739   assert((src->encoding() < 16),"XMM register should be 0-15");
 3740   Assembler::vpmovmskb(dst, src, vector_len);
 3741 }
 3742 
 3743 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3744   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3745   Assembler::vpmullw(dst, nds, src, vector_len);
 3746 }
 3747 
 3748 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3749   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3750   Assembler::vpmullw(dst, nds, src, vector_len);
 3751 }
 3752 
 3753 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3754   assert((UseAVX > 0), "AVX support is needed");
 3755   assert(rscratch != noreg || always_reachable(src), "missing");
 3756 
 3757   if (reachable(src)) {
 3758     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3759   } else {
 3760     lea(rscratch, src);
 3761     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3762   }
 3763 }
 3764 
 3765 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3766   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3767   Assembler::vpsubb(dst, nds, src, vector_len);
 3768 }
 3769 
 3770 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3771   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3772   Assembler::vpsubb(dst, nds, src, vector_len);
 3773 }
 3774 
 3775 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3776   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3777   Assembler::vpsubw(dst, nds, src, vector_len);
 3778 }
 3779 
 3780 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3781   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3782   Assembler::vpsubw(dst, nds, src, vector_len);
 3783 }
 3784 
 3785 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3786   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3787   Assembler::vpsraw(dst, nds, shift, vector_len);
 3788 }
 3789 
 3790 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3791   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3792   Assembler::vpsraw(dst, nds, shift, vector_len);
 3793 }
 3794 
 3795 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3796   assert(UseAVX > 2,"");
 3797   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3798      vector_len = 2;
 3799   }
 3800   Assembler::evpsraq(dst, nds, shift, vector_len);
 3801 }
 3802 
 3803 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3804   assert(UseAVX > 2,"");
 3805   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3806      vector_len = 2;
 3807   }
 3808   Assembler::evpsraq(dst, nds, shift, vector_len);
 3809 }
 3810 
 3811 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3812   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3813   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3814 }
 3815 
 3816 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3817   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3818   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3819 }
 3820 
 3821 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3822   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3823   Assembler::vpsllw(dst, nds, shift, vector_len);
 3824 }
 3825 
 3826 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3827   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3828   Assembler::vpsllw(dst, nds, shift, vector_len);
 3829 }
 3830 
 3831 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3832   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3833   Assembler::vptest(dst, src);
 3834 }
 3835 
 3836 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3837   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3838   Assembler::punpcklbw(dst, src);
 3839 }
 3840 
 3841 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3842   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3843   Assembler::pshufd(dst, src, mode);
 3844 }
 3845 
 3846 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3847   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3848   Assembler::pshuflw(dst, src, mode);
 3849 }
 3850 
 3851 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3852   assert(rscratch != noreg || always_reachable(src), "missing");
 3853 
 3854   if (reachable(src)) {
 3855     vandpd(dst, nds, as_Address(src), vector_len);
 3856   } else {
 3857     lea(rscratch, src);
 3858     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3859   }
 3860 }
 3861 
 3862 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3863   assert(rscratch != noreg || always_reachable(src), "missing");
 3864 
 3865   if (reachable(src)) {
 3866     vandps(dst, nds, as_Address(src), vector_len);
 3867   } else {
 3868     lea(rscratch, src);
 3869     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3870   }
 3871 }
 3872 
 3873 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3874                             bool merge, int vector_len, Register rscratch) {
 3875   assert(rscratch != noreg || always_reachable(src), "missing");
 3876 
 3877   if (reachable(src)) {
 3878     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3879   } else {
 3880     lea(rscratch, src);
 3881     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3882   }
 3883 }
 3884 
 3885 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3886   assert(rscratch != noreg || always_reachable(src), "missing");
 3887 
 3888   if (reachable(src)) {
 3889     vdivsd(dst, nds, as_Address(src));
 3890   } else {
 3891     lea(rscratch, src);
 3892     vdivsd(dst, nds, Address(rscratch, 0));
 3893   }
 3894 }
 3895 
 3896 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3897   assert(rscratch != noreg || always_reachable(src), "missing");
 3898 
 3899   if (reachable(src)) {
 3900     vdivss(dst, nds, as_Address(src));
 3901   } else {
 3902     lea(rscratch, src);
 3903     vdivss(dst, nds, Address(rscratch, 0));
 3904   }
 3905 }
 3906 
 3907 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3908   assert(rscratch != noreg || always_reachable(src), "missing");
 3909 
 3910   if (reachable(src)) {
 3911     vmulsd(dst, nds, as_Address(src));
 3912   } else {
 3913     lea(rscratch, src);
 3914     vmulsd(dst, nds, Address(rscratch, 0));
 3915   }
 3916 }
 3917 
 3918 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3919   assert(rscratch != noreg || always_reachable(src), "missing");
 3920 
 3921   if (reachable(src)) {
 3922     vmulss(dst, nds, as_Address(src));
 3923   } else {
 3924     lea(rscratch, src);
 3925     vmulss(dst, nds, Address(rscratch, 0));
 3926   }
 3927 }
 3928 
 3929 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3930   assert(rscratch != noreg || always_reachable(src), "missing");
 3931 
 3932   if (reachable(src)) {
 3933     vsubsd(dst, nds, as_Address(src));
 3934   } else {
 3935     lea(rscratch, src);
 3936     vsubsd(dst, nds, Address(rscratch, 0));
 3937   }
 3938 }
 3939 
 3940 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3941   assert(rscratch != noreg || always_reachable(src), "missing");
 3942 
 3943   if (reachable(src)) {
 3944     vsubss(dst, nds, as_Address(src));
 3945   } else {
 3946     lea(rscratch, src);
 3947     vsubss(dst, nds, Address(rscratch, 0));
 3948   }
 3949 }
 3950 
 3951 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3952   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3953   assert(rscratch != noreg || always_reachable(src), "missing");
 3954 
 3955   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3956 }
 3957 
 3958 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3959   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3960   assert(rscratch != noreg || always_reachable(src), "missing");
 3961 
 3962   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3963 }
 3964 
 3965 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3966   assert(rscratch != noreg || always_reachable(src), "missing");
 3967 
 3968   if (reachable(src)) {
 3969     vxorpd(dst, nds, as_Address(src), vector_len);
 3970   } else {
 3971     lea(rscratch, src);
 3972     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 3973   }
 3974 }
 3975 
 3976 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3977   assert(rscratch != noreg || always_reachable(src), "missing");
 3978 
 3979   if (reachable(src)) {
 3980     vxorps(dst, nds, as_Address(src), vector_len);
 3981   } else {
 3982     lea(rscratch, src);
 3983     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 3984   }
 3985 }
 3986 
 3987 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3988   assert(rscratch != noreg || always_reachable(src), "missing");
 3989 
 3990   if (UseAVX > 1 || (vector_len < 1)) {
 3991     if (reachable(src)) {
 3992       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 3993     } else {
 3994       lea(rscratch, src);
 3995       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 3996     }
 3997   } else {
 3998     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 3999   }
 4000 }
 4001 
 4002 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4003   assert(rscratch != noreg || always_reachable(src), "missing");
 4004 
 4005   if (reachable(src)) {
 4006     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 4007   } else {
 4008     lea(rscratch, src);
 4009     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 4010   }
 4011 }
 4012 
 4013 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 4014   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 4015   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 4016   // The inverted mask is sign-extended
 4017   andptr(possibly_non_local, inverted_mask);
 4018 }
 4019 
 4020 void MacroAssembler::resolve_jobject(Register value,
 4021                                      Register thread,
 4022                                      Register tmp) {
 4023   assert_different_registers(value, thread, tmp);
 4024   Label done, tagged, weak_tagged;
 4025   testptr(value, value);
 4026   jcc(Assembler::zero, done);           // Use null as-is.
 4027   testptr(value, JNIHandles::tag_mask); // Test for tag.
 4028   jcc(Assembler::notZero, tagged);
 4029 
 4030   // Resolve local handle
 4031   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4032   verify_oop(value);
 4033   jmp(done);
 4034 
 4035   bind(tagged);
 4036   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4037   jcc(Assembler::notZero, weak_tagged);
 4038 
 4039   // Resolve global handle
 4040   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4041   verify_oop(value);
 4042   jmp(done);
 4043 
 4044   bind(weak_tagged);
 4045   // Resolve jweak.
 4046   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4047                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4048   verify_oop(value);
 4049 
 4050   bind(done);
 4051 }
 4052 
 4053 void MacroAssembler::resolve_global_jobject(Register value,
 4054                                             Register thread,
 4055                                             Register tmp) {
 4056   assert_different_registers(value, thread, tmp);
 4057   Label done;
 4058 
 4059   testptr(value, value);
 4060   jcc(Assembler::zero, done);           // Use null as-is.
 4061 
 4062 #ifdef ASSERT
 4063   {
 4064     Label valid_global_tag;
 4065     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4066     jcc(Assembler::notZero, valid_global_tag);
 4067     stop("non global jobject using resolve_global_jobject");
 4068     bind(valid_global_tag);
 4069   }
 4070 #endif
 4071 
 4072   // Resolve global handle
 4073   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4074   verify_oop(value);
 4075 
 4076   bind(done);
 4077 }
 4078 
 4079 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4080   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4081 }
 4082 
 4083 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4084 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4085   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4086 }
 4087 
 4088 void MacroAssembler::subptr(Register dst, Register src) {
 4089   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4090 }
 4091 
 4092 // C++ bool manipulation
 4093 void MacroAssembler::testbool(Register dst) {
 4094   if(sizeof(bool) == 1)
 4095     testb(dst, 0xff);
 4096   else if(sizeof(bool) == 2) {
 4097     // testw implementation needed for two byte bools
 4098     ShouldNotReachHere();
 4099   } else if(sizeof(bool) == 4)
 4100     testl(dst, dst);
 4101   else
 4102     // unsupported
 4103     ShouldNotReachHere();
 4104 }
 4105 
 4106 void MacroAssembler::testptr(Register dst, Register src) {
 4107   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4108 }
 4109 
 4110 // Object / value buffer allocation...
 4111 //
 4112 // Kills klass and rsi on LP64
 4113 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
 4114                                        Register t1, Register t2,
 4115                                        bool clear_fields, Label& alloc_failed)
 4116 {
 4117   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
 4118   Register layout_size = t1;
 4119   assert(new_obj == rax, "needs to be rax");
 4120   assert_different_registers(klass, new_obj, t1, t2);
 4121 
 4122   // get instance_size in InstanceKlass (scaled to a count of bytes)
 4123   movl(layout_size, Address(klass, Klass::layout_helper_offset()));
 4124   // test to see if it has a finalizer or is malformed in some way
 4125   testl(layout_size, Klass::_lh_instance_slow_path_bit);
 4126   jcc(Assembler::notZero, slow_case_no_pop);
 4127 
 4128   // Allocate the instance:
 4129   //  If TLAB is enabled:
 4130   //    Try to allocate in the TLAB.
 4131   //    If fails, go to the slow path.
 4132   //  Else If inline contiguous allocations are enabled:
 4133   //    Try to allocate in eden.
 4134   //    If fails due to heap end, go to slow path.
 4135   //
 4136   //  If TLAB is enabled OR inline contiguous is enabled:
 4137   //    Initialize the allocation.
 4138   //    Exit.
 4139   //
 4140   //  Go to slow path.
 4141 
 4142   push(klass);
 4143   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
 4144 #ifndef _LP64
 4145   if (UseTLAB) {
 4146     get_thread(thread);
 4147   }
 4148 #endif // _LP64
 4149 
 4150   if (UseTLAB) {
 4151     tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
 4152     if (ZeroTLAB || (!clear_fields)) {
 4153       // the fields have been already cleared
 4154       jmp(initialize_header);
 4155     } else {
 4156       // initialize both the header and fields
 4157       jmp(initialize_object);
 4158     }
 4159   } else {
 4160     jmp(slow_case);
 4161   }
 4162 
 4163   // If UseTLAB is true, the object is created above and there is an initialize need.
 4164   // Otherwise, skip and go to the slow path.
 4165   if (UseTLAB) {
 4166     if (clear_fields) {
 4167       // The object is initialized before the header.  If the object size is
 4168       // zero, go directly to the header initialization.
 4169       bind(initialize_object);
 4170       decrement(layout_size, sizeof(oopDesc));
 4171       jcc(Assembler::zero, initialize_header);
 4172 
 4173       // Initialize topmost object field, divide size by 8, check if odd and
 4174       // test if zero.
 4175       Register zero = klass;
 4176       xorl(zero, zero);    // use zero reg to clear memory (shorter code)
 4177       shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
 4178 
 4179   #ifdef ASSERT
 4180       // make sure instance_size was multiple of 8
 4181       Label L;
 4182       // Ignore partial flag stall after shrl() since it is debug VM
 4183       jcc(Assembler::carryClear, L);
 4184       stop("object size is not multiple of 2 - adjust this code");
 4185       bind(L);
 4186       // must be > 0, no extra check needed here
 4187   #endif
 4188 
 4189       // initialize remaining object fields: instance_size was a multiple of 8
 4190       {
 4191         Label loop;
 4192         bind(loop);
 4193         movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero);
 4194         NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero));
 4195         decrement(layout_size);
 4196         jcc(Assembler::notZero, loop);
 4197       }
 4198     } // clear_fields
 4199 
 4200     // initialize object header only.
 4201     bind(initialize_header);
 4202     pop(klass);
 4203     Register mark_word = t2;
 4204     movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
 4205     movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 4206 #ifdef _LP64
 4207     xorl(rsi, rsi);                 // use zero reg to clear memory (shorter code)
 4208     store_klass_gap(new_obj, rsi);  // zero klass gap for compressed oops
 4209 #endif
 4210     movptr(t2, klass);         // preserve klass
 4211     store_klass(new_obj, t2, rscratch1);  // src klass reg is potentially compressed
 4212 
 4213     jmp(done);
 4214   }
 4215 
 4216   bind(slow_case);
 4217   pop(klass);
 4218   bind(slow_case_no_pop);
 4219   jmp(alloc_failed);
 4220 
 4221   bind(done);
 4222 }
 4223 
 4224 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4225 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4226                                    Register var_size_in_bytes,
 4227                                    int con_size_in_bytes,
 4228                                    Register t1,
 4229                                    Register t2,
 4230                                    Label& slow_case) {
 4231   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4232   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4233 }
 4234 
 4235 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4236   RegSet regs;
 4237 #ifdef _LP64
 4238   regs += RegSet::of(rax, rcx, rdx);
 4239 #ifndef WINDOWS
 4240   regs += RegSet::of(rsi, rdi);
 4241 #endif
 4242   regs += RegSet::range(r8, r11);
 4243 #else
 4244   regs += RegSet::of(rax, rcx, rdx);
 4245 #endif
 4246   return regs;
 4247 }
 4248 
 4249 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4250   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4251 #if defined(WINDOWS) && defined(_LP64)
 4252   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4253   if (num_xmm_registers > 16) {
 4254      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4255   }
 4256   return result;
 4257 #else
 4258   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4259 #endif
 4260 }
 4261 
 4262 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4263 
 4264 #ifndef _LP64
 4265 static bool use_x87_registers() { return UseSSE < 2; }
 4266 #endif
 4267 static bool use_xmm_registers() { return UseSSE >= 1; }
 4268 
 4269 // C1 only ever uses the first double/float of the XMM register.
 4270 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4271 
 4272 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4273   if (UseSSE == 1) {
 4274     masm->movflt(Address(rsp, offset), reg);
 4275   } else {
 4276     masm->movdbl(Address(rsp, offset), reg);
 4277   }
 4278 }
 4279 
 4280 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4281   if (UseSSE == 1) {
 4282     masm->movflt(reg, Address(rsp, offset));
 4283   } else {
 4284     masm->movdbl(reg, Address(rsp, offset));
 4285   }
 4286 }
 4287 
 4288 int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu,
 4289                            int& gp_area_size, int& fp_area_size, int& xmm_area_size) {
 4290 
 4291   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4292                          StackAlignmentInBytes);
 4293 #ifdef _LP64
 4294   fp_area_size = 0;
 4295 #else
 4296   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4297 #endif
 4298   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4299 
 4300   return gp_area_size + fp_area_size + xmm_area_size;
 4301 }
 4302 
 4303 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4304   block_comment("push_call_clobbered_registers start");
 4305   // Regular registers
 4306   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4307 
 4308   int gp_area_size;
 4309   int fp_area_size;
 4310   int xmm_area_size;
 4311   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4312                                                gp_area_size, fp_area_size, xmm_area_size);
 4313   subptr(rsp, total_save_size);
 4314 
 4315   push_set(gp_registers_to_push, 0);
 4316 
 4317 #ifndef _LP64
 4318   if (save_fpu && use_x87_registers()) {
 4319     fnsave(Address(rsp, gp_area_size));
 4320     fwait();
 4321   }
 4322 #endif
 4323   if (save_fpu && use_xmm_registers()) {
 4324     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4325   }
 4326 
 4327   block_comment("push_call_clobbered_registers end");
 4328 }
 4329 
 4330 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4331   block_comment("pop_call_clobbered_registers start");
 4332 
 4333   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4334 
 4335   int gp_area_size;
 4336   int fp_area_size;
 4337   int xmm_area_size;
 4338   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4339                                                gp_area_size, fp_area_size, xmm_area_size);
 4340 
 4341   if (restore_fpu && use_xmm_registers()) {
 4342     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4343   }
 4344 #ifndef _LP64
 4345   if (restore_fpu && use_x87_registers()) {
 4346     frstor(Address(rsp, gp_area_size));
 4347   }
 4348 #endif
 4349 
 4350   pop_set(gp_registers_to_pop, 0);
 4351 
 4352   addptr(rsp, total_save_size);
 4353 
 4354   vzeroupper();
 4355 
 4356   block_comment("pop_call_clobbered_registers end");
 4357 }
 4358 
 4359 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4360   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4361   int spill_offset = offset;
 4362 
 4363   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4364     save_xmm_register(this, spill_offset, *it);
 4365     spill_offset += xmm_save_size();
 4366   }
 4367 }
 4368 
 4369 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4370   int restore_size = set.size() * xmm_save_size();
 4371   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4372 
 4373   int restore_offset = offset + restore_size - xmm_save_size();
 4374 
 4375   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4376     restore_xmm_register(this, restore_offset, *it);
 4377     restore_offset -= xmm_save_size();
 4378   }
 4379 }
 4380 
 4381 void MacroAssembler::push_set(RegSet set, int offset) {
 4382   int spill_offset;
 4383   if (offset == -1) {
 4384     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4385     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4386     subptr(rsp, aligned_size);
 4387     spill_offset = 0;
 4388   } else {
 4389     spill_offset = offset;
 4390   }
 4391 
 4392   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4393     movptr(Address(rsp, spill_offset), *it);
 4394     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4395   }
 4396 }
 4397 
 4398 void MacroAssembler::pop_set(RegSet set, int offset) {
 4399 
 4400   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4401   int restore_size = set.size() * gp_reg_size;
 4402   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4403 
 4404   int restore_offset;
 4405   if (offset == -1) {
 4406     restore_offset = restore_size - gp_reg_size;
 4407   } else {
 4408     restore_offset = offset + restore_size - gp_reg_size;
 4409   }
 4410   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4411     movptr(*it, Address(rsp, restore_offset));
 4412     restore_offset -= gp_reg_size;
 4413   }
 4414 
 4415   if (offset == -1) {
 4416     addptr(rsp, aligned_size);
 4417   }
 4418 }
 4419 
 4420 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4421 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4422   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4423   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4424   Label done;
 4425 
 4426   testptr(length_in_bytes, length_in_bytes);
 4427   jcc(Assembler::zero, done);
 4428 
 4429   // initialize topmost word, divide index by 2, check if odd and test if zero
 4430   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4431 #ifdef ASSERT
 4432   {
 4433     Label L;
 4434     testptr(length_in_bytes, BytesPerWord - 1);
 4435     jcc(Assembler::zero, L);
 4436     stop("length must be a multiple of BytesPerWord");
 4437     bind(L);
 4438   }
 4439 #endif
 4440   Register index = length_in_bytes;
 4441   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4442   if (UseIncDec) {
 4443     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4444   } else {
 4445     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4446     shrptr(index, 1);
 4447   }
 4448 #ifndef _LP64
 4449   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4450   {
 4451     Label even;
 4452     // note: if index was a multiple of 8, then it cannot
 4453     //       be 0 now otherwise it must have been 0 before
 4454     //       => if it is even, we don't need to check for 0 again
 4455     jcc(Assembler::carryClear, even);
 4456     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4457     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4458     // index could be 0 now, must check again
 4459     jcc(Assembler::zero, done);
 4460     bind(even);
 4461   }
 4462 #endif // !_LP64
 4463   // initialize remaining object fields: index is a multiple of 2 now
 4464   {
 4465     Label loop;
 4466     bind(loop);
 4467     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4468     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4469     decrement(index);
 4470     jcc(Assembler::notZero, loop);
 4471   }
 4472 
 4473   bind(done);
 4474 }
 4475 
 4476 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
 4477   movptr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
 4478 #ifdef ASSERT
 4479   {
 4480     Label done;
 4481     cmpptr(inline_klass, 0);
 4482     jcc(Assembler::notEqual, done);
 4483     stop("get_inline_type_field_klass contains no inline klass");
 4484     bind(done);
 4485   }
 4486 #endif
 4487   movptr(inline_klass, Address(inline_klass, index, Address::times_ptr));
 4488 }
 4489 
 4490 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 4491 #ifdef ASSERT
 4492   {
 4493     Label done_check;
 4494     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 4495     stop("get_default_value_oop from non inline type klass");
 4496     bind(done_check);
 4497   }
 4498 #endif
 4499   Register offset = temp_reg;
 4500   // Getting the offset of the pre-allocated default value
 4501   movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 4502   movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 4503 
 4504   // Getting the mirror
 4505   movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 4506   resolve_oop_handle(obj, inline_klass);
 4507 
 4508   // Getting the pre-allocated default value from the mirror
 4509   Address field(obj, offset, Address::times_1);
 4510   load_heap_oop(obj, field);
 4511 }
 4512 
 4513 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 4514 #ifdef ASSERT
 4515   {
 4516     Label done_check;
 4517     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 4518     stop("get_empty_value from non-empty inline klass");
 4519     bind(done_check);
 4520   }
 4521 #endif
 4522   get_default_value_oop(inline_klass, temp_reg, obj);
 4523 }
 4524 
 4525 
 4526 // Look up the method for a megamorphic invokeinterface call.
 4527 // The target method is determined by <intf_klass, itable_index>.
 4528 // The receiver klass is in recv_klass.
 4529 // On success, the result will be in method_result, and execution falls through.
 4530 // On failure, execution transfers to the given label.
 4531 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4532                                              Register intf_klass,
 4533                                              RegisterOrConstant itable_index,
 4534                                              Register method_result,
 4535                                              Register scan_temp,
 4536                                              Label& L_no_such_interface,
 4537                                              bool return_method) {
 4538   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4539   assert_different_registers(method_result, intf_klass, scan_temp);
 4540   assert(recv_klass != method_result || !return_method,
 4541          "recv_klass can be destroyed when method isn't needed");
 4542 
 4543   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4544          "caller must use same register for non-constant itable index as for method");
 4545 
 4546   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4547   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4548   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4549   int scan_step   = itableOffsetEntry::size() * wordSize;
 4550   int vte_size    = vtableEntry::size_in_bytes();
 4551   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4552   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4553 
 4554   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4555 
 4556   // %%% Could store the aligned, prescaled offset in the klassoop.
 4557   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4558 
 4559   if (return_method) {
 4560     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4561     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4562     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4563   }
 4564 
 4565   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4566   //   if (scan->interface() == intf) {
 4567   //     result = (klass + scan->offset() + itable_index);
 4568   //   }
 4569   // }
 4570   Label search, found_method;
 4571 
 4572   for (int peel = 1; peel >= 0; peel--) {
 4573     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4574     cmpptr(intf_klass, method_result);
 4575 
 4576     if (peel) {
 4577       jccb(Assembler::equal, found_method);
 4578     } else {
 4579       jccb(Assembler::notEqual, search);
 4580       // (invert the test to fall through to found_method...)
 4581     }
 4582 
 4583     if (!peel)  break;
 4584 
 4585     bind(search);
 4586 
 4587     // Check that the previous entry is non-null.  A null entry means that
 4588     // the receiver class doesn't implement the interface, and wasn't the
 4589     // same as when the caller was compiled.
 4590     testptr(method_result, method_result);
 4591     jcc(Assembler::zero, L_no_such_interface);
 4592     addptr(scan_temp, scan_step);
 4593   }
 4594 
 4595   bind(found_method);
 4596 
 4597   if (return_method) {
 4598     // Got a hit.
 4599     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4600     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4601   }
 4602 }
 4603 
 4604 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4605 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
 4606 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4607 // The target method is determined by <holder_klass, itable_index>.
 4608 // The receiver klass is in recv_klass.
 4609 // On success, the result will be in method_result, and execution falls through.
 4610 // On failure, execution transfers to the given label.
 4611 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4612                                                   Register holder_klass,
 4613                                                   Register resolved_klass,
 4614                                                   Register method_result,
 4615                                                   Register scan_temp,
 4616                                                   Register temp_reg2,
 4617                                                   Register receiver,
 4618                                                   int itable_index,
 4619                                                   Label& L_no_such_interface) {
 4620   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4621   Register temp_itbl_klass = method_result;
 4622   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4623 
 4624   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4625   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4626   int scan_step = itableOffsetEntry::size() * wordSize;
 4627   int vte_size = vtableEntry::size_in_bytes();
 4628   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4629   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4630   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4631   assert(vte_size == wordSize, "adjust times_vte_scale");
 4632 
 4633   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4634 
 4635   // temp_itbl_klass = recv_klass.itable[0]
 4636   // scan_temp = &recv_klass.itable[0] + step
 4637   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4638   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4639   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4640   xorptr(temp_reg, temp_reg);
 4641 
 4642   // Initial checks:
 4643   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4644   //   - if (itable[0] == 0), no such interface
 4645   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4646   cmpptr(holder_klass, resolved_klass);
 4647   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4648   testptr(temp_itbl_klass, temp_itbl_klass);
 4649   jccb(Assembler::zero, L_no_such_interface);
 4650   cmpptr(holder_klass, temp_itbl_klass);
 4651   jccb(Assembler::equal, L_holder_found);
 4652 
 4653   // Loop: Look for holder_klass record in itable
 4654   //   do {
 4655   //     tmp = itable[index];
 4656   //     index += step;
 4657   //     if (tmp == holder_klass) {
 4658   //       goto L_holder_found; // Found!
 4659   //     }
 4660   //   } while (tmp != 0);
 4661   //   goto L_no_such_interface // Not found.
 4662   Label L_scan_holder;
 4663   bind(L_scan_holder);
 4664     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4665     addptr(scan_temp, scan_step);
 4666     cmpptr(holder_klass, temp_itbl_klass);
 4667     jccb(Assembler::equal, L_holder_found);
 4668     testptr(temp_itbl_klass, temp_itbl_klass);
 4669     jccb(Assembler::notZero, L_scan_holder);
 4670 
 4671   jmpb(L_no_such_interface);
 4672 
 4673   // Loop: Look for resolved_class record in itable
 4674   //   do {
 4675   //     tmp = itable[index];
 4676   //     index += step;
 4677   //     if (tmp == holder_klass) {
 4678   //        // Also check if we have met a holder klass
 4679   //        holder_tmp = itable[index-step-ioffset];
 4680   //     }
 4681   //     if (tmp == resolved_klass) {
 4682   //        goto L_resolved_found;  // Found!
 4683   //     }
 4684   //   } while (tmp != 0);
 4685   //   goto L_no_such_interface // Not found.
 4686   //
 4687   Label L_loop_scan_resolved;
 4688   bind(L_loop_scan_resolved);
 4689     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4690     addptr(scan_temp, scan_step);
 4691     bind(L_loop_scan_resolved_entry);
 4692     cmpptr(holder_klass, temp_itbl_klass);
 4693     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4694     cmpptr(resolved_klass, temp_itbl_klass);
 4695     jccb(Assembler::equal, L_resolved_found);
 4696     testptr(temp_itbl_klass, temp_itbl_klass);
 4697     jccb(Assembler::notZero, L_loop_scan_resolved);
 4698 
 4699   jmpb(L_no_such_interface);
 4700 
 4701   Label L_ready;
 4702 
 4703   // See if we already have a holder klass. If not, go and scan for it.
 4704   bind(L_resolved_found);
 4705   testptr(temp_reg, temp_reg);
 4706   jccb(Assembler::zero, L_scan_holder);
 4707   jmpb(L_ready);
 4708 
 4709   bind(L_holder_found);
 4710   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4711 
 4712   // Finally, temp_reg contains holder_klass vtable offset
 4713   bind(L_ready);
 4714   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4715   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4716     load_klass(scan_temp, receiver, noreg);
 4717     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4718   } else {
 4719     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4720   }
 4721 }
 4722 
 4723 
 4724 // virtual method calling
 4725 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4726                                            RegisterOrConstant vtable_index,
 4727                                            Register method_result) {
 4728   const ByteSize base = Klass::vtable_start_offset();
 4729   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4730   Address vtable_entry_addr(recv_klass,
 4731                             vtable_index, Address::times_ptr,
 4732                             base + vtableEntry::method_offset());
 4733   movptr(method_result, vtable_entry_addr);
 4734 }
 4735 
 4736 
 4737 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4738                            Register super_klass,
 4739                            Register temp_reg,
 4740                            Label& L_success) {
 4741   Label L_failure;
 4742   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4743   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4744   bind(L_failure);
 4745 }
 4746 
 4747 
 4748 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4749                                                    Register super_klass,
 4750                                                    Register temp_reg,
 4751                                                    Label* L_success,
 4752                                                    Label* L_failure,
 4753                                                    Label* L_slow_path,
 4754                                         RegisterOrConstant super_check_offset) {
 4755   assert_different_registers(sub_klass, super_klass, temp_reg);
 4756   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4757   if (super_check_offset.is_register()) {
 4758     assert_different_registers(sub_klass, super_klass,
 4759                                super_check_offset.as_register());
 4760   } else if (must_load_sco) {
 4761     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4762   }
 4763 
 4764   Label L_fallthrough;
 4765   int label_nulls = 0;
 4766   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4767   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4768   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4769   assert(label_nulls <= 1, "at most one null in the batch");
 4770 
 4771   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4772   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4773   Address super_check_offset_addr(super_klass, sco_offset);
 4774 
 4775   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4776   // range of a jccb.  If this routine grows larger, reconsider at
 4777   // least some of these.
 4778 #define local_jcc(assembler_cond, label)                                \
 4779   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4780   else                             jcc( assembler_cond, label) /*omit semi*/
 4781 
 4782   // Hacked jmp, which may only be used just before L_fallthrough.
 4783 #define final_jmp(label)                                                \
 4784   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4785   else                            jmp(label)                /*omit semi*/
 4786 
 4787   // If the pointers are equal, we are done (e.g., String[] elements).
 4788   // This self-check enables sharing of secondary supertype arrays among
 4789   // non-primary types such as array-of-interface.  Otherwise, each such
 4790   // type would need its own customized SSA.
 4791   // We move this check to the front of the fast path because many
 4792   // type checks are in fact trivially successful in this manner,
 4793   // so we get a nicely predicted branch right at the start of the check.
 4794   cmpptr(sub_klass, super_klass);
 4795   local_jcc(Assembler::equal, *L_success);
 4796 
 4797   // Check the supertype display:
 4798   if (must_load_sco) {
 4799     // Positive movl does right thing on LP64.
 4800     movl(temp_reg, super_check_offset_addr);
 4801     super_check_offset = RegisterOrConstant(temp_reg);
 4802   }
 4803   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4804   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4805 
 4806   // This check has worked decisively for primary supers.
 4807   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4808   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4809   // This works in the same check above because of a tricky aliasing
 4810   // between the super_cache and the primary super display elements.
 4811   // (The 'super_check_addr' can address either, as the case requires.)
 4812   // Note that the cache is updated below if it does not help us find
 4813   // what we need immediately.
 4814   // So if it was a primary super, we can just fail immediately.
 4815   // Otherwise, it's the slow path for us (no success at this point).
 4816 
 4817   if (super_check_offset.is_register()) {
 4818     local_jcc(Assembler::equal, *L_success);
 4819     cmpl(super_check_offset.as_register(), sc_offset);
 4820     if (L_failure == &L_fallthrough) {
 4821       local_jcc(Assembler::equal, *L_slow_path);
 4822     } else {
 4823       local_jcc(Assembler::notEqual, *L_failure);
 4824       final_jmp(*L_slow_path);
 4825     }
 4826   } else if (super_check_offset.as_constant() == sc_offset) {
 4827     // Need a slow path; fast failure is impossible.
 4828     if (L_slow_path == &L_fallthrough) {
 4829       local_jcc(Assembler::equal, *L_success);
 4830     } else {
 4831       local_jcc(Assembler::notEqual, *L_slow_path);
 4832       final_jmp(*L_success);
 4833     }
 4834   } else {
 4835     // No slow path; it's a fast decision.
 4836     if (L_failure == &L_fallthrough) {
 4837       local_jcc(Assembler::equal, *L_success);
 4838     } else {
 4839       local_jcc(Assembler::notEqual, *L_failure);
 4840       final_jmp(*L_success);
 4841     }
 4842   }
 4843 
 4844   bind(L_fallthrough);
 4845 
 4846 #undef local_jcc
 4847 #undef final_jmp
 4848 }
 4849 
 4850 
 4851 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4852                                                    Register super_klass,
 4853                                                    Register temp_reg,
 4854                                                    Register temp2_reg,
 4855                                                    Label* L_success,
 4856                                                    Label* L_failure,
 4857                                                    bool set_cond_codes) {
 4858   assert_different_registers(sub_klass, super_klass, temp_reg);
 4859   if (temp2_reg != noreg)
 4860     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4861 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4862 
 4863   Label L_fallthrough;
 4864   int label_nulls = 0;
 4865   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4866   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4867   assert(label_nulls <= 1, "at most one null in the batch");
 4868 
 4869   // a couple of useful fields in sub_klass:
 4870   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4871   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4872   Address secondary_supers_addr(sub_klass, ss_offset);
 4873   Address super_cache_addr(     sub_klass, sc_offset);
 4874 
 4875   // Do a linear scan of the secondary super-klass chain.
 4876   // This code is rarely used, so simplicity is a virtue here.
 4877   // The repne_scan instruction uses fixed registers, which we must spill.
 4878   // Don't worry too much about pre-existing connections with the input regs.
 4879 
 4880   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4881   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4882 
 4883   // Get super_klass value into rax (even if it was in rdi or rcx).
 4884   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4885   if (super_klass != rax) {
 4886     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4887     mov(rax, super_klass);
 4888   }
 4889   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4890   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4891 
 4892 #ifndef PRODUCT
 4893   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4894   ExternalAddress pst_counter_addr((address) pst_counter);
 4895   NOT_LP64(  incrementl(pst_counter_addr) );
 4896   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4897   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4898 #endif //PRODUCT
 4899 
 4900   // We will consult the secondary-super array.
 4901   movptr(rdi, secondary_supers_addr);
 4902   // Load the array length.  (Positive movl does right thing on LP64.)
 4903   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4904   // Skip to start of data.
 4905   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4906 
 4907   // Scan RCX words at [RDI] for an occurrence of RAX.
 4908   // Set NZ/Z based on last compare.
 4909   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4910   // not change flags (only scas instruction which is repeated sets flags).
 4911   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4912 
 4913     testptr(rax,rax); // Set Z = 0
 4914     repne_scan();
 4915 
 4916   // Unspill the temp. registers:
 4917   if (pushed_rdi)  pop(rdi);
 4918   if (pushed_rcx)  pop(rcx);
 4919   if (pushed_rax)  pop(rax);
 4920 
 4921   if (set_cond_codes) {
 4922     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4923     assert(!pushed_rdi, "rdi must be left non-null");
 4924     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4925   }
 4926 
 4927   if (L_failure == &L_fallthrough)
 4928         jccb(Assembler::notEqual, *L_failure);
 4929   else  jcc(Assembler::notEqual, *L_failure);
 4930 
 4931   // Success.  Cache the super we found and proceed in triumph.
 4932   movptr(super_cache_addr, super_klass);
 4933 
 4934   if (L_success != &L_fallthrough) {
 4935     jmp(*L_success);
 4936   }
 4937 
 4938 #undef IS_A_TEMP
 4939 
 4940   bind(L_fallthrough);
 4941 }
 4942 
 4943 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 4944   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 4945 
 4946   Label L_fallthrough;
 4947   if (L_fast_path == nullptr) {
 4948     L_fast_path = &L_fallthrough;
 4949   } else if (L_slow_path == nullptr) {
 4950     L_slow_path = &L_fallthrough;
 4951   }
 4952 
 4953   // Fast path check: class is fully initialized
 4954   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 4955   jcc(Assembler::equal, *L_fast_path);
 4956 
 4957   // Fast path check: current thread is initializer thread
 4958   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 4959   if (L_slow_path == &L_fallthrough) {
 4960     jcc(Assembler::equal, *L_fast_path);
 4961     bind(*L_slow_path);
 4962   } else if (L_fast_path == &L_fallthrough) {
 4963     jcc(Assembler::notEqual, *L_slow_path);
 4964     bind(*L_fast_path);
 4965   } else {
 4966     Unimplemented();
 4967   }
 4968 }
 4969 
 4970 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 4971   if (VM_Version::supports_cmov()) {
 4972     cmovl(cc, dst, src);
 4973   } else {
 4974     Label L;
 4975     jccb(negate_condition(cc), L);
 4976     movl(dst, src);
 4977     bind(L);
 4978   }
 4979 }
 4980 
 4981 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 4982   if (VM_Version::supports_cmov()) {
 4983     cmovl(cc, dst, src);
 4984   } else {
 4985     Label L;
 4986     jccb(negate_condition(cc), L);
 4987     movl(dst, src);
 4988     bind(L);
 4989   }
 4990 }
 4991 
 4992 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 4993   if (!VerifyOops || VerifyAdapterSharing) {
 4994     // Below address of the code string confuses VerifyAdapterSharing
 4995     // because it may differ between otherwise equivalent adapters.
 4996     return;
 4997   }
 4998 
 4999   BLOCK_COMMENT("verify_oop {");
 5000 #ifdef _LP64
 5001   push(rscratch1);
 5002 #endif
 5003   push(rax);                          // save rax
 5004   push(reg);                          // pass register argument
 5005 
 5006   // Pass register number to verify_oop_subroutine
 5007   const char* b = nullptr;
 5008   {
 5009     ResourceMark rm;
 5010     stringStream ss;
 5011     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5012     b = code_string(ss.as_string());
 5013   }
 5014   ExternalAddress buffer((address) b);
 5015   pushptr(buffer.addr(), rscratch1);
 5016 
 5017   // call indirectly to solve generation ordering problem
 5018   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5019   call(rax);
 5020   // Caller pops the arguments (oop, message) and restores rax, r10
 5021   BLOCK_COMMENT("} verify_oop");
 5022 }
 5023 
 5024 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5025   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5026     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5027     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5028     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5029   } else if (VM_Version::supports_avx()) {
 5030     vpcmpeqd(dst, dst, dst, vector_len);
 5031   } else {
 5032     assert(VM_Version::supports_sse2(), "");
 5033     pcmpeqd(dst, dst);
 5034   }
 5035 }
 5036 
 5037 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5038                                          int extra_slot_offset) {
 5039   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5040   int stackElementSize = Interpreter::stackElementSize;
 5041   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5042 #ifdef ASSERT
 5043   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5044   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5045 #endif
 5046   Register             scale_reg    = noreg;
 5047   Address::ScaleFactor scale_factor = Address::no_scale;
 5048   if (arg_slot.is_constant()) {
 5049     offset += arg_slot.as_constant() * stackElementSize;
 5050   } else {
 5051     scale_reg    = arg_slot.as_register();
 5052     scale_factor = Address::times(stackElementSize);
 5053   }
 5054   offset += wordSize;           // return PC is on stack
 5055   return Address(rsp, scale_reg, scale_factor, offset);
 5056 }
 5057 
 5058 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5059   if (!VerifyOops || VerifyAdapterSharing) {
 5060     // Below address of the code string confuses VerifyAdapterSharing
 5061     // because it may differ between otherwise equivalent adapters.
 5062     return;
 5063   }
 5064 
 5065 #ifdef _LP64
 5066   push(rscratch1);
 5067 #endif
 5068   push(rax); // save rax,
 5069   // addr may contain rsp so we will have to adjust it based on the push
 5070   // we just did (and on 64 bit we do two pushes)
 5071   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5072   // stores rax into addr which is backwards of what was intended.
 5073   if (addr.uses(rsp)) {
 5074     lea(rax, addr);
 5075     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5076   } else {
 5077     pushptr(addr);
 5078   }
 5079 
 5080   // Pass register number to verify_oop_subroutine
 5081   const char* b = nullptr;
 5082   {
 5083     ResourceMark rm;
 5084     stringStream ss;
 5085     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5086     b = code_string(ss.as_string());
 5087   }
 5088   ExternalAddress buffer((address) b);
 5089   pushptr(buffer.addr(), rscratch1);
 5090 
 5091   // call indirectly to solve generation ordering problem
 5092   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5093   call(rax);
 5094   // Caller pops the arguments (addr, message) and restores rax, r10.
 5095 }
 5096 
 5097 void MacroAssembler::verify_tlab() {
 5098 #ifdef ASSERT
 5099   if (UseTLAB && VerifyOops) {
 5100     Label next, ok;
 5101     Register t1 = rsi;
 5102     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5103 
 5104     push(t1);
 5105     NOT_LP64(push(thread_reg));
 5106     NOT_LP64(get_thread(thread_reg));
 5107 
 5108     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5109     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5110     jcc(Assembler::aboveEqual, next);
 5111     STOP("assert(top >= start)");
 5112     should_not_reach_here();
 5113 
 5114     bind(next);
 5115     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5116     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5117     jcc(Assembler::aboveEqual, ok);
 5118     STOP("assert(top <= end)");
 5119     should_not_reach_here();
 5120 
 5121     bind(ok);
 5122     NOT_LP64(pop(thread_reg));
 5123     pop(t1);
 5124   }
 5125 #endif
 5126 }
 5127 
 5128 class ControlWord {
 5129  public:
 5130   int32_t _value;
 5131 
 5132   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5133   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5134   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5135   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5136   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5137   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5138   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5139   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5140 
 5141   void print() const {
 5142     // rounding control
 5143     const char* rc;
 5144     switch (rounding_control()) {
 5145       case 0: rc = "round near"; break;
 5146       case 1: rc = "round down"; break;
 5147       case 2: rc = "round up  "; break;
 5148       case 3: rc = "chop      "; break;
 5149       default:
 5150         rc = nullptr; // silence compiler warnings
 5151         fatal("Unknown rounding control: %d", rounding_control());
 5152     };
 5153     // precision control
 5154     const char* pc;
 5155     switch (precision_control()) {
 5156       case 0: pc = "24 bits "; break;
 5157       case 1: pc = "reserved"; break;
 5158       case 2: pc = "53 bits "; break;
 5159       case 3: pc = "64 bits "; break;
 5160       default:
 5161         pc = nullptr; // silence compiler warnings
 5162         fatal("Unknown precision control: %d", precision_control());
 5163     };
 5164     // flags
 5165     char f[9];
 5166     f[0] = ' ';
 5167     f[1] = ' ';
 5168     f[2] = (precision   ()) ? 'P' : 'p';
 5169     f[3] = (underflow   ()) ? 'U' : 'u';
 5170     f[4] = (overflow    ()) ? 'O' : 'o';
 5171     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5172     f[6] = (denormalized()) ? 'D' : 'd';
 5173     f[7] = (invalid     ()) ? 'I' : 'i';
 5174     f[8] = '\x0';
 5175     // output
 5176     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5177   }
 5178 
 5179 };
 5180 
 5181 class StatusWord {
 5182  public:
 5183   int32_t _value;
 5184 
 5185   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5186   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5187   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5188   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5189   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5190   int  top() const                     { return  (_value >> 11) & 7      ; }
 5191   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5192   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5193   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5194   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5195   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5196   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5197   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5198   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5199 
 5200   void print() const {
 5201     // condition codes
 5202     char c[5];
 5203     c[0] = (C3()) ? '3' : '-';
 5204     c[1] = (C2()) ? '2' : '-';
 5205     c[2] = (C1()) ? '1' : '-';
 5206     c[3] = (C0()) ? '0' : '-';
 5207     c[4] = '\x0';
 5208     // flags
 5209     char f[9];
 5210     f[0] = (error_status()) ? 'E' : '-';
 5211     f[1] = (stack_fault ()) ? 'S' : '-';
 5212     f[2] = (precision   ()) ? 'P' : '-';
 5213     f[3] = (underflow   ()) ? 'U' : '-';
 5214     f[4] = (overflow    ()) ? 'O' : '-';
 5215     f[5] = (zero_divide ()) ? 'Z' : '-';
 5216     f[6] = (denormalized()) ? 'D' : '-';
 5217     f[7] = (invalid     ()) ? 'I' : '-';
 5218     f[8] = '\x0';
 5219     // output
 5220     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5221   }
 5222 
 5223 };
 5224 
 5225 class TagWord {
 5226  public:
 5227   int32_t _value;
 5228 
 5229   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5230 
 5231   void print() const {
 5232     printf("%04x", _value & 0xFFFF);
 5233   }
 5234 
 5235 };
 5236 
 5237 class FPU_Register {
 5238  public:
 5239   int32_t _m0;
 5240   int32_t _m1;
 5241   int16_t _ex;
 5242 
 5243   bool is_indefinite() const           {
 5244     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5245   }
 5246 
 5247   void print() const {
 5248     char  sign = (_ex < 0) ? '-' : '+';
 5249     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5250     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5251   };
 5252 
 5253 };
 5254 
 5255 class FPU_State {
 5256  public:
 5257   enum {
 5258     register_size       = 10,
 5259     number_of_registers =  8,
 5260     register_mask       =  7
 5261   };
 5262 
 5263   ControlWord  _control_word;
 5264   StatusWord   _status_word;
 5265   TagWord      _tag_word;
 5266   int32_t      _error_offset;
 5267   int32_t      _error_selector;
 5268   int32_t      _data_offset;
 5269   int32_t      _data_selector;
 5270   int8_t       _register[register_size * number_of_registers];
 5271 
 5272   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5273   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5274 
 5275   const char* tag_as_string(int tag) const {
 5276     switch (tag) {
 5277       case 0: return "valid";
 5278       case 1: return "zero";
 5279       case 2: return "special";
 5280       case 3: return "empty";
 5281     }
 5282     ShouldNotReachHere();
 5283     return nullptr;
 5284   }
 5285 
 5286   void print() const {
 5287     // print computation registers
 5288     { int t = _status_word.top();
 5289       for (int i = 0; i < number_of_registers; i++) {
 5290         int j = (i - t) & register_mask;
 5291         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5292         st(j)->print();
 5293         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5294       }
 5295     }
 5296     printf("\n");
 5297     // print control registers
 5298     printf("ctrl = "); _control_word.print(); printf("\n");
 5299     printf("stat = "); _status_word .print(); printf("\n");
 5300     printf("tags = "); _tag_word    .print(); printf("\n");
 5301   }
 5302 
 5303 };
 5304 
 5305 class Flag_Register {
 5306  public:
 5307   int32_t _value;
 5308 
 5309   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5310   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5311   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5312   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5313   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5314   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5315   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5316 
 5317   void print() const {
 5318     // flags
 5319     char f[8];
 5320     f[0] = (overflow       ()) ? 'O' : '-';
 5321     f[1] = (direction      ()) ? 'D' : '-';
 5322     f[2] = (sign           ()) ? 'S' : '-';
 5323     f[3] = (zero           ()) ? 'Z' : '-';
 5324     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5325     f[5] = (parity         ()) ? 'P' : '-';
 5326     f[6] = (carry          ()) ? 'C' : '-';
 5327     f[7] = '\x0';
 5328     // output
 5329     printf("%08x  flags = %s", _value, f);
 5330   }
 5331 
 5332 };
 5333 
 5334 class IU_Register {
 5335  public:
 5336   int32_t _value;
 5337 
 5338   void print() const {
 5339     printf("%08x  %11d", _value, _value);
 5340   }
 5341 
 5342 };
 5343 
 5344 class IU_State {
 5345  public:
 5346   Flag_Register _eflags;
 5347   IU_Register   _rdi;
 5348   IU_Register   _rsi;
 5349   IU_Register   _rbp;
 5350   IU_Register   _rsp;
 5351   IU_Register   _rbx;
 5352   IU_Register   _rdx;
 5353   IU_Register   _rcx;
 5354   IU_Register   _rax;
 5355 
 5356   void print() const {
 5357     // computation registers
 5358     printf("rax,  = "); _rax.print(); printf("\n");
 5359     printf("rbx,  = "); _rbx.print(); printf("\n");
 5360     printf("rcx  = "); _rcx.print(); printf("\n");
 5361     printf("rdx  = "); _rdx.print(); printf("\n");
 5362     printf("rdi  = "); _rdi.print(); printf("\n");
 5363     printf("rsi  = "); _rsi.print(); printf("\n");
 5364     printf("rbp,  = "); _rbp.print(); printf("\n");
 5365     printf("rsp  = "); _rsp.print(); printf("\n");
 5366     printf("\n");
 5367     // control registers
 5368     printf("flgs = "); _eflags.print(); printf("\n");
 5369   }
 5370 };
 5371 
 5372 
 5373 class CPU_State {
 5374  public:
 5375   FPU_State _fpu_state;
 5376   IU_State  _iu_state;
 5377 
 5378   void print() const {
 5379     printf("--------------------------------------------------\n");
 5380     _iu_state .print();
 5381     printf("\n");
 5382     _fpu_state.print();
 5383     printf("--------------------------------------------------\n");
 5384   }
 5385 
 5386 };
 5387 
 5388 
 5389 static void _print_CPU_state(CPU_State* state) {
 5390   state->print();
 5391 };
 5392 
 5393 
 5394 void MacroAssembler::print_CPU_state() {
 5395   push_CPU_state();
 5396   push(rsp);                // pass CPU state
 5397   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5398   addptr(rsp, wordSize);       // discard argument
 5399   pop_CPU_state();
 5400 }
 5401 
 5402 
 5403 #ifndef _LP64
 5404 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5405   static int counter = 0;
 5406   FPU_State* fs = &state->_fpu_state;
 5407   counter++;
 5408   // For leaf calls, only verify that the top few elements remain empty.
 5409   // We only need 1 empty at the top for C2 code.
 5410   if( stack_depth < 0 ) {
 5411     if( fs->tag_for_st(7) != 3 ) {
 5412       printf("FPR7 not empty\n");
 5413       state->print();
 5414       assert(false, "error");
 5415       return false;
 5416     }
 5417     return true;                // All other stack states do not matter
 5418   }
 5419 
 5420   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5421          "bad FPU control word");
 5422 
 5423   // compute stack depth
 5424   int i = 0;
 5425   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5426   int d = i;
 5427   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5428   // verify findings
 5429   if (i != FPU_State::number_of_registers) {
 5430     // stack not contiguous
 5431     printf("%s: stack not contiguous at ST%d\n", s, i);
 5432     state->print();
 5433     assert(false, "error");
 5434     return false;
 5435   }
 5436   // check if computed stack depth corresponds to expected stack depth
 5437   if (stack_depth < 0) {
 5438     // expected stack depth is -stack_depth or less
 5439     if (d > -stack_depth) {
 5440       // too many elements on the stack
 5441       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5442       state->print();
 5443       assert(false, "error");
 5444       return false;
 5445     }
 5446   } else {
 5447     // expected stack depth is stack_depth
 5448     if (d != stack_depth) {
 5449       // wrong stack depth
 5450       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5451       state->print();
 5452       assert(false, "error");
 5453       return false;
 5454     }
 5455   }
 5456   // everything is cool
 5457   return true;
 5458 }
 5459 
 5460 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5461   if (!VerifyFPU) return;
 5462   push_CPU_state();
 5463   push(rsp);                // pass CPU state
 5464   ExternalAddress msg((address) s);
 5465   // pass message string s
 5466   pushptr(msg.addr(), noreg);
 5467   push(stack_depth);        // pass stack depth
 5468   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5469   addptr(rsp, 3 * wordSize);   // discard arguments
 5470   // check for error
 5471   { Label L;
 5472     testl(rax, rax);
 5473     jcc(Assembler::notZero, L);
 5474     int3();                  // break if error condition
 5475     bind(L);
 5476   }
 5477   pop_CPU_state();
 5478 }
 5479 #endif // _LP64
 5480 
 5481 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5482   // Either restore the MXCSR register after returning from the JNI Call
 5483   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5484   if (VM_Version::supports_sse()) {
 5485     if (RestoreMXCSROnJNICalls) {
 5486       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5487     } else if (CheckJNICalls) {
 5488       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5489     }
 5490   }
 5491   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5492   vzeroupper();
 5493 
 5494 #ifndef _LP64
 5495   // Either restore the x87 floating pointer control word after returning
 5496   // from the JNI call or verify that it wasn't changed.
 5497   if (CheckJNICalls) {
 5498     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5499   }
 5500 #endif // _LP64
 5501 }
 5502 
 5503 // ((OopHandle)result).resolve();
 5504 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5505   assert_different_registers(result, tmp);
 5506 
 5507   // Only 64 bit platforms support GCs that require a tmp register
 5508   // Only IN_HEAP loads require a thread_tmp register
 5509   // OopHandle::resolve is an indirection like jobject.
 5510   access_load_at(T_OBJECT, IN_NATIVE,
 5511                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5512 }
 5513 
 5514 // ((WeakHandle)result).resolve();
 5515 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5516   assert_different_registers(rresult, rtmp);
 5517   Label resolved;
 5518 
 5519   // A null weak handle resolves to null.
 5520   cmpptr(rresult, 0);
 5521   jcc(Assembler::equal, resolved);
 5522 
 5523   // Only 64 bit platforms support GCs that require a tmp register
 5524   // Only IN_HEAP loads require a thread_tmp register
 5525   // WeakHandle::resolve is an indirection like jweak.
 5526   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5527                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5528   bind(resolved);
 5529 }
 5530 
 5531 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5532   // get mirror
 5533   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5534   load_method_holder(mirror, method);
 5535   movptr(mirror, Address(mirror, mirror_offset));
 5536   resolve_oop_handle(mirror, tmp);
 5537 }
 5538 
 5539 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5540   load_method_holder(rresult, rmethod);
 5541   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5542 }
 5543 
 5544 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5545   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5546   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5547   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5548 }
 5549 
 5550 void MacroAssembler::load_metadata(Register dst, Register src) {
 5551   if (UseCompressedClassPointers) {
 5552     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5553   } else {
 5554     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5555   }
 5556 }
 5557 
 5558 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5559   assert_different_registers(src, tmp);
 5560   assert_different_registers(dst, tmp);
 5561 #ifdef _LP64
 5562   if (UseCompressedClassPointers) {
 5563     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5564     decode_klass_not_null(dst, tmp);
 5565   } else
 5566 #endif
 5567   movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5568 }
 5569 
 5570 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
 5571   load_klass(dst, src, tmp);
 5572   movptr(dst, Address(dst, Klass::prototype_header_offset()));
 5573 }
 5574 
 5575 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5576   assert_different_registers(src, tmp);
 5577   assert_different_registers(dst, tmp);
 5578 #ifdef _LP64
 5579   if (UseCompressedClassPointers) {
 5580     encode_klass_not_null(src, tmp);
 5581     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5582   } else
 5583 #endif
 5584     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5585 }
 5586 
 5587 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5588                                     Register tmp1, Register thread_tmp) {
 5589   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5590   decorators = AccessInternal::decorator_fixup(decorators, type);
 5591   bool as_raw = (decorators & AS_RAW) != 0;
 5592   if (as_raw) {
 5593     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5594   } else {
 5595     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5596   }
 5597 }
 5598 
 5599 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5600                                      Register tmp1, Register tmp2, Register tmp3) {
 5601   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5602   decorators = AccessInternal::decorator_fixup(decorators, type);
 5603   bool as_raw = (decorators & AS_RAW) != 0;
 5604   if (as_raw) {
 5605     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5606   } else {
 5607     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5608   }
 5609 }
 5610 
 5611 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
 5612                                        Register inline_klass) {
 5613   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5614   bs->value_copy(this, decorators, src, dst, inline_klass);
 5615 }
 5616 
 5617 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
 5618   movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 5619   movl(offset, Address(offset, InlineKlass::first_field_offset_offset()));
 5620 }
 5621 
 5622 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
 5623   // ((address) (void*) o) + vk->first_field_offset();
 5624   Register offset = (data == oop) ? rscratch1 : data;
 5625   first_field_offset(inline_klass, offset);
 5626   if (data == oop) {
 5627     addptr(data, offset);
 5628   } else {
 5629     lea(data, Address(oop, offset));
 5630   }
 5631 }
 5632 
 5633 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
 5634                                                 Register index, Register data) {
 5635   assert(index != rcx, "index needs to shift by rcx");
 5636   assert_different_registers(array, array_klass, index);
 5637   assert_different_registers(rcx, array, index);
 5638 
 5639   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
 5640   movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
 5641 
 5642   // Klass::layout_helper_log2_element_size(lh)
 5643   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
 5644   shrl(rcx, Klass::_lh_log2_element_size_shift);
 5645   andl(rcx, Klass::_lh_log2_element_size_mask);
 5646   shlptr(index); // index << rcx
 5647 
 5648   lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
 5649 }
 5650 
 5651 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5652                                    Register thread_tmp, DecoratorSet decorators) {
 5653   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5654 }
 5655 
 5656 // Doesn't do verification, generates fixed size code
 5657 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5658                                             Register thread_tmp, DecoratorSet decorators) {
 5659   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5660 }
 5661 
 5662 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5663                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5664   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5665 }
 5666 
 5667 // Used for storing nulls.
 5668 void MacroAssembler::store_heap_oop_null(Address dst) {
 5669   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5670 }
 5671 
 5672 #ifdef _LP64
 5673 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 5674   if (UseCompressedClassPointers) {
 5675     // Store to klass gap in destination
 5676     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 5677   }
 5678 }
 5679 
 5680 #ifdef ASSERT
 5681 void MacroAssembler::verify_heapbase(const char* msg) {
 5682   assert (UseCompressedOops, "should be compressed");
 5683   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5684   if (CheckCompressedOops) {
 5685     Label ok;
 5686     ExternalAddress src2(CompressedOops::ptrs_base_addr());
 5687     const bool is_src2_reachable = reachable(src2);
 5688     if (!is_src2_reachable) {
 5689       push(rscratch1);  // cmpptr trashes rscratch1
 5690     }
 5691     cmpptr(r12_heapbase, src2, rscratch1);
 5692     jcc(Assembler::equal, ok);
 5693     STOP(msg);
 5694     bind(ok);
 5695     if (!is_src2_reachable) {
 5696       pop(rscratch1);
 5697     }
 5698   }
 5699 }
 5700 #endif
 5701 
 5702 // Algorithm must match oop.inline.hpp encode_heap_oop.
 5703 void MacroAssembler::encode_heap_oop(Register r) {
 5704 #ifdef ASSERT
 5705   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 5706 #endif
 5707   verify_oop_msg(r, "broken oop in encode_heap_oop");
 5708   if (CompressedOops::base() == nullptr) {
 5709     if (CompressedOops::shift() != 0) {
 5710       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5711       shrq(r, LogMinObjAlignmentInBytes);
 5712     }
 5713     return;
 5714   }
 5715   testq(r, r);
 5716   cmovq(Assembler::equal, r, r12_heapbase);
 5717   subq(r, r12_heapbase);
 5718   shrq(r, LogMinObjAlignmentInBytes);
 5719 }
 5720 
 5721 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 5722 #ifdef ASSERT
 5723   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 5724   if (CheckCompressedOops) {
 5725     Label ok;
 5726     testq(r, r);
 5727     jcc(Assembler::notEqual, ok);
 5728     STOP("null oop passed to encode_heap_oop_not_null");
 5729     bind(ok);
 5730   }
 5731 #endif
 5732   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 5733   if (CompressedOops::base() != nullptr) {
 5734     subq(r, r12_heapbase);
 5735   }
 5736   if (CompressedOops::shift() != 0) {
 5737     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5738     shrq(r, LogMinObjAlignmentInBytes);
 5739   }
 5740 }
 5741 
 5742 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 5743 #ifdef ASSERT
 5744   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 5745   if (CheckCompressedOops) {
 5746     Label ok;
 5747     testq(src, src);
 5748     jcc(Assembler::notEqual, ok);
 5749     STOP("null oop passed to encode_heap_oop_not_null2");
 5750     bind(ok);
 5751   }
 5752 #endif
 5753   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 5754   if (dst != src) {
 5755     movq(dst, src);
 5756   }
 5757   if (CompressedOops::base() != nullptr) {
 5758     subq(dst, r12_heapbase);
 5759   }
 5760   if (CompressedOops::shift() != 0) {
 5761     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5762     shrq(dst, LogMinObjAlignmentInBytes);
 5763   }
 5764 }
 5765 
 5766 void  MacroAssembler::decode_heap_oop(Register r) {
 5767 #ifdef ASSERT
 5768   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 5769 #endif
 5770   if (CompressedOops::base() == nullptr) {
 5771     if (CompressedOops::shift() != 0) {
 5772       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5773       shlq(r, LogMinObjAlignmentInBytes);
 5774     }
 5775   } else {
 5776     Label done;
 5777     shlq(r, LogMinObjAlignmentInBytes);
 5778     jccb(Assembler::equal, done);
 5779     addq(r, r12_heapbase);
 5780     bind(done);
 5781   }
 5782   verify_oop_msg(r, "broken oop in decode_heap_oop");
 5783 }
 5784 
 5785 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 5786   // Note: it will change flags
 5787   assert (UseCompressedOops, "should only be used for compressed headers");
 5788   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5789   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5790   // vtableStubs also counts instructions in pd_code_size_limit.
 5791   // Also do not verify_oop as this is called by verify_oop.
 5792   if (CompressedOops::shift() != 0) {
 5793     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5794     shlq(r, LogMinObjAlignmentInBytes);
 5795     if (CompressedOops::base() != nullptr) {
 5796       addq(r, r12_heapbase);
 5797     }
 5798   } else {
 5799     assert (CompressedOops::base() == nullptr, "sanity");
 5800   }
 5801 }
 5802 
 5803 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 5804   // Note: it will change flags
 5805   assert (UseCompressedOops, "should only be used for compressed headers");
 5806   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5807   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5808   // vtableStubs also counts instructions in pd_code_size_limit.
 5809   // Also do not verify_oop as this is called by verify_oop.
 5810   if (CompressedOops::shift() != 0) {
 5811     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5812     if (LogMinObjAlignmentInBytes == Address::times_8) {
 5813       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 5814     } else {
 5815       if (dst != src) {
 5816         movq(dst, src);
 5817       }
 5818       shlq(dst, LogMinObjAlignmentInBytes);
 5819       if (CompressedOops::base() != nullptr) {
 5820         addq(dst, r12_heapbase);
 5821       }
 5822     }
 5823   } else {
 5824     assert (CompressedOops::base() == nullptr, "sanity");
 5825     if (dst != src) {
 5826       movq(dst, src);
 5827     }
 5828   }
 5829 }
 5830 
 5831 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 5832   assert_different_registers(r, tmp);
 5833   if (CompressedKlassPointers::base() != nullptr) {
 5834     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5835     subq(r, tmp);
 5836   }
 5837   if (CompressedKlassPointers::shift() != 0) {
 5838     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5839     shrq(r, LogKlassAlignmentInBytes);
 5840   }
 5841 }
 5842 
 5843 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 5844   assert_different_registers(src, dst);
 5845   if (CompressedKlassPointers::base() != nullptr) {
 5846     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 5847     addq(dst, src);
 5848   } else {
 5849     movptr(dst, src);
 5850   }
 5851   if (CompressedKlassPointers::shift() != 0) {
 5852     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5853     shrq(dst, LogKlassAlignmentInBytes);
 5854   }
 5855 }
 5856 
 5857 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 5858   assert_different_registers(r, tmp);
 5859   // Note: it will change flags
 5860   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 5861   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5862   // vtableStubs also counts instructions in pd_code_size_limit.
 5863   // Also do not verify_oop as this is called by verify_oop.
 5864   if (CompressedKlassPointers::shift() != 0) {
 5865     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5866     shlq(r, LogKlassAlignmentInBytes);
 5867   }
 5868   if (CompressedKlassPointers::base() != nullptr) {
 5869     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5870     addq(r, tmp);
 5871   }
 5872 }
 5873 
 5874 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 5875   assert_different_registers(src, dst);
 5876   // Note: it will change flags
 5877   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5878   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5879   // vtableStubs also counts instructions in pd_code_size_limit.
 5880   // Also do not verify_oop as this is called by verify_oop.
 5881 
 5882   if (CompressedKlassPointers::base() == nullptr &&
 5883       CompressedKlassPointers::shift() == 0) {
 5884     // The best case scenario is that there is no base or shift. Then it is already
 5885     // a pointer that needs nothing but a register rename.
 5886     movl(dst, src);
 5887   } else {
 5888     if (CompressedKlassPointers::base() != nullptr) {
 5889       mov64(dst, (int64_t)CompressedKlassPointers::base());
 5890     } else {
 5891       xorq(dst, dst);
 5892     }
 5893     if (CompressedKlassPointers::shift() != 0) {
 5894       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5895       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 5896       leaq(dst, Address(dst, src, Address::times_8, 0));
 5897     } else {
 5898       addq(dst, src);
 5899     }
 5900   }
 5901 }
 5902 
 5903 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 5904   assert (UseCompressedOops, "should only be used for compressed headers");
 5905   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5906   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5907   int oop_index = oop_recorder()->find_index(obj);
 5908   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5909   mov_narrow_oop(dst, oop_index, rspec);
 5910 }
 5911 
 5912 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 5913   assert (UseCompressedOops, "should only be used for compressed headers");
 5914   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5915   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5916   int oop_index = oop_recorder()->find_index(obj);
 5917   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5918   mov_narrow_oop(dst, oop_index, rspec);
 5919 }
 5920 
 5921 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 5922   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5923   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5924   int klass_index = oop_recorder()->find_index(k);
 5925   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5926   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5927 }
 5928 
 5929 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 5930   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5931   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5932   int klass_index = oop_recorder()->find_index(k);
 5933   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5934   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5935 }
 5936 
 5937 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 5938   assert (UseCompressedOops, "should only be used for compressed headers");
 5939   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5940   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5941   int oop_index = oop_recorder()->find_index(obj);
 5942   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5943   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 5944 }
 5945 
 5946 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 5947   assert (UseCompressedOops, "should only be used for compressed headers");
 5948   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5949   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5950   int oop_index = oop_recorder()->find_index(obj);
 5951   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5952   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 5953 }
 5954 
 5955 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 5956   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5957   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5958   int klass_index = oop_recorder()->find_index(k);
 5959   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5960   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5961 }
 5962 
 5963 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 5964   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5965   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5966   int klass_index = oop_recorder()->find_index(k);
 5967   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5968   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5969 }
 5970 
 5971 void MacroAssembler::reinit_heapbase() {
 5972   if (UseCompressedOops) {
 5973     if (Universe::heap() != nullptr) {
 5974       if (CompressedOops::base() == nullptr) {
 5975         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 5976       } else {
 5977         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 5978       }
 5979     } else {
 5980       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 5981     }
 5982   }
 5983 }
 5984 
 5985 #endif // _LP64
 5986 
 5987 #if COMPILER2_OR_JVMCI
 5988 
 5989 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 5990 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
 5991   // cnt - number of qwords (8-byte words).
 5992   // base - start address, qword aligned.
 5993   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 5994   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 5995   if (use64byteVector) {
 5996     evpbroadcastq(xtmp, val, AVX_512bit);
 5997   } else if (MaxVectorSize >= 32) {
 5998     movdq(xtmp, val);
 5999     punpcklqdq(xtmp, xtmp);
 6000     vinserti128_high(xtmp, xtmp);
 6001   } else {
 6002     movdq(xtmp, val);
 6003     punpcklqdq(xtmp, xtmp);
 6004   }
 6005   jmp(L_zero_64_bytes);
 6006 
 6007   BIND(L_loop);
 6008   if (MaxVectorSize >= 32) {
 6009     fill64(base, 0, xtmp, use64byteVector);
 6010   } else {
 6011     movdqu(Address(base,  0), xtmp);
 6012     movdqu(Address(base, 16), xtmp);
 6013     movdqu(Address(base, 32), xtmp);
 6014     movdqu(Address(base, 48), xtmp);
 6015   }
 6016   addptr(base, 64);
 6017 
 6018   BIND(L_zero_64_bytes);
 6019   subptr(cnt, 8);
 6020   jccb(Assembler::greaterEqual, L_loop);
 6021 
 6022   // Copy trailing 64 bytes
 6023   if (use64byteVector) {
 6024     addptr(cnt, 8);
 6025     jccb(Assembler::equal, L_end);
 6026     fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
 6027     jmp(L_end);
 6028   } else {
 6029     addptr(cnt, 4);
 6030     jccb(Assembler::less, L_tail);
 6031     if (MaxVectorSize >= 32) {
 6032       vmovdqu(Address(base, 0), xtmp);
 6033     } else {
 6034       movdqu(Address(base,  0), xtmp);
 6035       movdqu(Address(base, 16), xtmp);
 6036     }
 6037   }
 6038   addptr(base, 32);
 6039   subptr(cnt, 4);
 6040 
 6041   BIND(L_tail);
 6042   addptr(cnt, 4);
 6043   jccb(Assembler::lessEqual, L_end);
 6044   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6045     fill32_masked(3, base, 0, xtmp, mask, cnt, val);
 6046   } else {
 6047     decrement(cnt);
 6048 
 6049     BIND(L_sloop);
 6050     movq(Address(base, 0), xtmp);
 6051     addptr(base, 8);
 6052     decrement(cnt);
 6053     jccb(Assembler::greaterEqual, L_sloop);
 6054   }
 6055   BIND(L_end);
 6056 }
 6057 
 6058 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
 6059   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
 6060   // An inline type might be returned. If fields are in registers we
 6061   // need to allocate an inline type instance and initialize it with
 6062   // the value of the fields.
 6063   Label skip;
 6064   // We only need a new buffered inline type if a new one is not returned
 6065   testptr(rax, 1);
 6066   jcc(Assembler::zero, skip);
 6067   int call_offset = -1;
 6068 
 6069 #ifdef _LP64
 6070   // The following code is similar to allocate_instance but has some slight differences,
 6071   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
 6072   // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these.
 6073   Label slow_case;
 6074   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
 6075   mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
 6076   if (vk != nullptr) {
 6077     // Called from C1, where the return type is statically known.
 6078     movptr(rbx, (intptr_t)vk->get_InlineKlass());
 6079     jint obj_size = vk->layout_helper();
 6080     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
 6081     if (UseTLAB) {
 6082       tlab_allocate(r15_thread, rax, noreg, obj_size, r13, r14, slow_case);
 6083     } else {
 6084       jmp(slow_case);
 6085     }
 6086   } else {
 6087     // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
 6088     mov(rbx, rax);
 6089     andptr(rbx, -2);
 6090     movl(r14, Address(rbx, Klass::layout_helper_offset()));
 6091     if (UseTLAB) {
 6092       tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
 6093     } else {
 6094       jmp(slow_case);
 6095     }
 6096   }
 6097   if (UseTLAB) {
 6098     // 2. Initialize buffered inline instance header
 6099     Register buffer_obj = rax;
 6100     movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
 6101     xorl(r13, r13);
 6102     store_klass_gap(buffer_obj, r13);
 6103     if (vk == nullptr) {
 6104       // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
 6105       mov(r13, rbx);
 6106     }
 6107     store_klass(buffer_obj, rbx, rscratch1);
 6108     // 3. Initialize its fields with an inline class specific handler
 6109     if (vk != nullptr) {
 6110       call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
 6111     } else {
 6112       movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6113       movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
 6114       call(rbx);
 6115     }
 6116     jmp(skip);
 6117   }
 6118   bind(slow_case);
 6119   // We failed to allocate a new inline type, fall back to a runtime
 6120   // call. Some oop field may be live in some registers but we can't
 6121   // tell. That runtime call will take care of preserving them
 6122   // across a GC if there's one.
 6123   mov(rax, rscratch1);
 6124 #endif
 6125 
 6126   if (from_interpreter) {
 6127     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
 6128   } else {
 6129     call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
 6130     call_offset = offset();
 6131   }
 6132 
 6133   bind(skip);
 6134   return call_offset;
 6135 }
 6136 
 6137 // Move a value between registers/stack slots and update the reg_state
 6138 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
 6139   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
 6140   if (reg_state[to->value()] == reg_written) {
 6141     return true; // Already written
 6142   }
 6143   if (from != to && bt != T_VOID) {
 6144     if (reg_state[to->value()] == reg_readonly) {
 6145       return false; // Not yet writable
 6146     }
 6147     if (from->is_reg()) {
 6148       if (to->is_reg()) {
 6149         if (from->is_XMMRegister()) {
 6150           if (bt == T_DOUBLE) {
 6151             movdbl(to->as_XMMRegister(), from->as_XMMRegister());
 6152           } else {
 6153             assert(bt == T_FLOAT, "must be float");
 6154             movflt(to->as_XMMRegister(), from->as_XMMRegister());
 6155           }
 6156         } else {
 6157           movq(to->as_Register(), from->as_Register());
 6158         }
 6159       } else {
 6160         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6161         Address to_addr = Address(rsp, st_off);
 6162         if (from->is_XMMRegister()) {
 6163           if (bt == T_DOUBLE) {
 6164             movdbl(to_addr, from->as_XMMRegister());
 6165           } else {
 6166             assert(bt == T_FLOAT, "must be float");
 6167             movflt(to_addr, from->as_XMMRegister());
 6168           }
 6169         } else {
 6170           movq(to_addr, from->as_Register());
 6171         }
 6172       }
 6173     } else {
 6174       Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
 6175       if (to->is_reg()) {
 6176         if (to->is_XMMRegister()) {
 6177           if (bt == T_DOUBLE) {
 6178             movdbl(to->as_XMMRegister(), from_addr);
 6179           } else {
 6180             assert(bt == T_FLOAT, "must be float");
 6181             movflt(to->as_XMMRegister(), from_addr);
 6182           }
 6183         } else {
 6184           movq(to->as_Register(), from_addr);
 6185         }
 6186       } else {
 6187         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6188         movq(r13, from_addr);
 6189         movq(Address(rsp, st_off), r13);
 6190       }
 6191     }
 6192   }
 6193   // Update register states
 6194   reg_state[from->value()] = reg_writable;
 6195   reg_state[to->value()] = reg_written;
 6196   return true;
 6197 }
 6198 
 6199 // Calculate the extra stack space required for packing or unpacking inline
 6200 // args and adjust the stack pointer
 6201 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
 6202   // Two additional slots to account for return address
 6203   int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
 6204   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
 6205   // Save the return address, adjust the stack (make sure it is properly
 6206   // 16-byte aligned) and copy the return address to the new top of the stack.
 6207   // The stack will be repaired on return (see MacroAssembler::remove_frame).
 6208   assert(sp_inc > 0, "sanity");
 6209   pop(r13);
 6210   subptr(rsp, sp_inc);
 6211   push(r13);
 6212   return sp_inc;
 6213 }
 6214 
 6215 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
 6216 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
 6217                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
 6218                                           RegState reg_state[]) {
 6219   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
 6220   assert(from->is_valid(), "source must be valid");
 6221   bool progress = false;
 6222 #ifdef ASSERT
 6223   const int start_offset = offset();
 6224 #endif
 6225 
 6226   Label L_null, L_notNull;
 6227   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
 6228   Register tmp1 = r10;
 6229   Register tmp2 = r13;
 6230   Register fromReg = noreg;
 6231   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
 6232   bool done = true;
 6233   bool mark_done = true;
 6234   VMReg toReg;
 6235   BasicType bt;
 6236   // Check if argument requires a null check
 6237   bool null_check = false;
 6238   VMReg nullCheckReg;
 6239   while (stream.next(nullCheckReg, bt)) {
 6240     if (sig->at(stream.sig_index())._offset == -1) {
 6241       null_check = true;
 6242       break;
 6243     }
 6244   }
 6245   stream.reset(sig_index, to_index);
 6246   while (stream.next(toReg, bt)) {
 6247     assert(toReg->is_valid(), "destination must be valid");
 6248     int idx = (int)toReg->value();
 6249     if (reg_state[idx] == reg_readonly) {
 6250       if (idx != from->value()) {
 6251         mark_done = false;
 6252       }
 6253       done = false;
 6254       continue;
 6255     } else if (reg_state[idx] == reg_written) {
 6256       continue;
 6257     }
 6258     assert(reg_state[idx] == reg_writable, "must be writable");
 6259     reg_state[idx] = reg_written;
 6260     progress = true;
 6261 
 6262     if (fromReg == noreg) {
 6263       if (from->is_reg()) {
 6264         fromReg = from->as_Register();
 6265       } else {
 6266         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6267         movq(tmp1, Address(rsp, st_off));
 6268         fromReg = tmp1;
 6269       }
 6270       if (null_check) {
 6271         // Nullable inline type argument, emit null check
 6272         testptr(fromReg, fromReg);
 6273         jcc(Assembler::zero, L_null);
 6274       }
 6275     }
 6276     int off = sig->at(stream.sig_index())._offset;
 6277     if (off == -1) {
 6278       assert(null_check, "Missing null check at");
 6279       if (toReg->is_stack()) {
 6280         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6281         movq(Address(rsp, st_off), 1);
 6282       } else {
 6283         movq(toReg->as_Register(), 1);
 6284       }
 6285       continue;
 6286     }
 6287     assert(off > 0, "offset in object should be positive");
 6288     Address fromAddr = Address(fromReg, off);
 6289     if (!toReg->is_XMMRegister()) {
 6290       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
 6291       if (is_reference_type(bt)) {
 6292         load_heap_oop(dst, fromAddr);
 6293       } else {
 6294         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
 6295         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
 6296       }
 6297       if (toReg->is_stack()) {
 6298         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6299         movq(Address(rsp, st_off), dst);
 6300       }
 6301     } else if (bt == T_DOUBLE) {
 6302       movdbl(toReg->as_XMMRegister(), fromAddr);
 6303     } else {
 6304       assert(bt == T_FLOAT, "must be float");
 6305       movflt(toReg->as_XMMRegister(), fromAddr);
 6306     }
 6307   }
 6308   if (progress && null_check) {
 6309     if (done) {
 6310       jmp(L_notNull);
 6311       bind(L_null);
 6312       // Set IsInit field to zero to signal that the argument is null.
 6313       // Also set all oop fields to zero to make the GC happy.
 6314       stream.reset(sig_index, to_index);
 6315       while (stream.next(toReg, bt)) {
 6316         if (sig->at(stream.sig_index())._offset == -1 ||
 6317             bt == T_OBJECT || bt == T_ARRAY) {
 6318           if (toReg->is_stack()) {
 6319             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6320             movq(Address(rsp, st_off), 0);
 6321           } else {
 6322             xorq(toReg->as_Register(), toReg->as_Register());
 6323           }
 6324         }
 6325       }
 6326       bind(L_notNull);
 6327     } else {
 6328       bind(L_null);
 6329     }
 6330   }
 6331 
 6332   sig_index = stream.sig_index();
 6333   to_index = stream.regs_index();
 6334 
 6335   if (mark_done && reg_state[from->value()] != reg_written) {
 6336     // This is okay because no one else will write to that slot
 6337     reg_state[from->value()] = reg_writable;
 6338   }
 6339   from_index--;
 6340   assert(progress || (start_offset == offset()), "should not emit code");
 6341   return done;
 6342 }
 6343 
 6344 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
 6345                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
 6346                                         RegState reg_state[], Register val_array) {
 6347   assert(sig->at(sig_index)._bt == T_PRIMITIVE_OBJECT, "should be at end delimiter");
 6348   assert(to->is_valid(), "destination must be valid");
 6349 
 6350   if (reg_state[to->value()] == reg_written) {
 6351     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6352     return true; // Already written
 6353   }
 6354 
 6355   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
 6356   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
 6357   Register val_obj_tmp = r11;
 6358   Register from_reg_tmp = r14;
 6359   Register tmp1 = r10;
 6360   Register tmp2 = r13;
 6361   Register tmp3 = rbx;
 6362   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
 6363 
 6364   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
 6365 
 6366   if (reg_state[to->value()] == reg_readonly) {
 6367     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
 6368       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6369       return false; // Not yet writable
 6370     }
 6371     val_obj = val_obj_tmp;
 6372   }
 6373 
 6374   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_PRIMITIVE_OBJECT);
 6375   load_heap_oop(val_obj, Address(val_array, index));
 6376 
 6377   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
 6378   VMReg fromReg;
 6379   BasicType bt;
 6380   Label L_null;
 6381   while (stream.next(fromReg, bt)) {
 6382     assert(fromReg->is_valid(), "source must be valid");
 6383     reg_state[fromReg->value()] = reg_writable;
 6384 
 6385     int off = sig->at(stream.sig_index())._offset;
 6386     if (off == -1) {
 6387       // Nullable inline type argument, emit null check
 6388       Label L_notNull;
 6389       if (fromReg->is_stack()) {
 6390         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6391         testb(Address(rsp, ld_off), 1);
 6392       } else {
 6393         testb(fromReg->as_Register(), 1);
 6394       }
 6395       jcc(Assembler::notZero, L_notNull);
 6396       movptr(val_obj, 0);
 6397       jmp(L_null);
 6398       bind(L_notNull);
 6399       continue;
 6400     }
 6401 
 6402     assert(off > 0, "offset in object should be positive");
 6403     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 6404 
 6405     Address dst(val_obj, off);
 6406     if (!fromReg->is_XMMRegister()) {
 6407       Register src;
 6408       if (fromReg->is_stack()) {
 6409         src = from_reg_tmp;
 6410         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6411         load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 6412       } else {
 6413         src = fromReg->as_Register();
 6414       }
 6415       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
 6416       if (is_reference_type(bt)) {
 6417         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 6418       } else {
 6419         store_sized_value(dst, src, size_in_bytes);
 6420       }
 6421     } else if (bt == T_DOUBLE) {
 6422       movdbl(dst, fromReg->as_XMMRegister());
 6423     } else {
 6424       assert(bt == T_FLOAT, "must be float");
 6425       movflt(dst, fromReg->as_XMMRegister());
 6426     }
 6427   }
 6428   bind(L_null);
 6429   sig_index = stream.sig_index();
 6430   from_index = stream.regs_index();
 6431 
 6432   assert(reg_state[to->value()] == reg_writable, "must have already been read");
 6433   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
 6434   assert(success, "to register must be writeable");
 6435   return true;
 6436 }
 6437 
 6438 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
 6439   return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
 6440 }
 6441 
 6442 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
 6443   assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 6444   if (needs_stack_repair) {
 6445     movq(rbp, Address(rsp, initial_framesize));
 6446     // The stack increment resides just below the saved rbp
 6447     addq(rsp, Address(rsp, initial_framesize - wordSize));
 6448   } else {
 6449     if (initial_framesize > 0) {
 6450       addq(rsp, initial_framesize);
 6451     }
 6452     pop(rbp);
 6453   }
 6454 }
 6455 
 6456 // Clearing constant sized memory using YMM/ZMM registers.
 6457 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6458   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
 6459   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6460 
 6461   int vector64_count = (cnt & (~0x7)) >> 3;
 6462   cnt = cnt & 0x7;
 6463   const int fill64_per_loop = 4;
 6464   const int max_unrolled_fill64 = 8;
 6465 
 6466   // 64 byte initialization loop.
 6467   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6468   int start64 = 0;
 6469   if (vector64_count > max_unrolled_fill64) {
 6470     Label LOOP;
 6471     Register index = rtmp;
 6472 
 6473     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6474 
 6475     movl(index, 0);
 6476     BIND(LOOP);
 6477     for (int i = 0; i < fill64_per_loop; i++) {
 6478       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6479     }
 6480     addl(index, fill64_per_loop * 64);
 6481     cmpl(index, start64 * 64);
 6482     jccb(Assembler::less, LOOP);
 6483   }
 6484   for (int i = start64; i < vector64_count; i++) {
 6485     fill64(base, i * 64, xtmp, use64byteVector);
 6486   }
 6487 
 6488   // Clear remaining 64 byte tail.
 6489   int disp = vector64_count * 64;
 6490   if (cnt) {
 6491     switch (cnt) {
 6492       case 1:
 6493         movq(Address(base, disp), xtmp);
 6494         break;
 6495       case 2:
 6496         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6497         break;
 6498       case 3:
 6499         movl(rtmp, 0x7);
 6500         kmovwl(mask, rtmp);
 6501         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6502         break;
 6503       case 4:
 6504         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6505         break;
 6506       case 5:
 6507         if (use64byteVector) {
 6508           movl(rtmp, 0x1F);
 6509           kmovwl(mask, rtmp);
 6510           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6511         } else {
 6512           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6513           movq(Address(base, disp + 32), xtmp);
 6514         }
 6515         break;
 6516       case 6:
 6517         if (use64byteVector) {
 6518           movl(rtmp, 0x3F);
 6519           kmovwl(mask, rtmp);
 6520           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6521         } else {
 6522           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6523           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6524         }
 6525         break;
 6526       case 7:
 6527         if (use64byteVector) {
 6528           movl(rtmp, 0x7F);
 6529           kmovwl(mask, rtmp);
 6530           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6531         } else {
 6532           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6533           movl(rtmp, 0x7);
 6534           kmovwl(mask, rtmp);
 6535           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6536         }
 6537         break;
 6538       default:
 6539         fatal("Unexpected length : %d\n",cnt);
 6540         break;
 6541     }
 6542   }
 6543 }
 6544 
 6545 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
 6546                                bool is_large, bool word_copy_only, KRegister mask) {
 6547   // cnt      - number of qwords (8-byte words).
 6548   // base     - start address, qword aligned.
 6549   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6550   assert(base==rdi, "base register must be edi for rep stos");
 6551   assert(val==rax,   "val register must be eax for rep stos");
 6552   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6553   assert(InitArrayShortSize % BytesPerLong == 0,
 6554     "InitArrayShortSize should be the multiple of BytesPerLong");
 6555 
 6556   Label DONE;
 6557 
 6558   if (!is_large) {
 6559     Label LOOP, LONG;
 6560     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6561     jccb(Assembler::greater, LONG);
 6562 
 6563     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6564 
 6565     decrement(cnt);
 6566     jccb(Assembler::negative, DONE); // Zero length
 6567 
 6568     // Use individual pointer-sized stores for small counts:
 6569     BIND(LOOP);
 6570     movptr(Address(base, cnt, Address::times_ptr), val);
 6571     decrement(cnt);
 6572     jccb(Assembler::greaterEqual, LOOP);
 6573     jmpb(DONE);
 6574 
 6575     BIND(LONG);
 6576   }
 6577 
 6578   // Use longer rep-prefixed ops for non-small counts:
 6579   if (UseFastStosb && !word_copy_only) {
 6580     shlptr(cnt, 3); // convert to number of bytes
 6581     rep_stosb();
 6582   } else if (UseXMMForObjInit) {
 6583     xmm_clear_mem(base, cnt, val, xtmp, mask);
 6584   } else {
 6585     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6586     rep_stos();
 6587   }
 6588 
 6589   BIND(DONE);
 6590 }
 6591 
 6592 #endif //COMPILER2_OR_JVMCI
 6593 
 6594 
 6595 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6596                                    Register to, Register value, Register count,
 6597                                    Register rtmp, XMMRegister xtmp) {
 6598   ShortBranchVerifier sbv(this);
 6599   assert_different_registers(to, value, count, rtmp);
 6600   Label L_exit;
 6601   Label L_fill_2_bytes, L_fill_4_bytes;
 6602 
 6603 #if defined(COMPILER2) && defined(_LP64)
 6604   if(MaxVectorSize >=32 &&
 6605      VM_Version::supports_avx512vlbw() &&
 6606      VM_Version::supports_bmi2()) {
 6607     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6608     return;
 6609   }
 6610 #endif
 6611 
 6612   int shift = -1;
 6613   switch (t) {
 6614     case T_BYTE:
 6615       shift = 2;
 6616       break;
 6617     case T_SHORT:
 6618       shift = 1;
 6619       break;
 6620     case T_INT:
 6621       shift = 0;
 6622       break;
 6623     default: ShouldNotReachHere();
 6624   }
 6625 
 6626   if (t == T_BYTE) {
 6627     andl(value, 0xff);
 6628     movl(rtmp, value);
 6629     shll(rtmp, 8);
 6630     orl(value, rtmp);
 6631   }
 6632   if (t == T_SHORT) {
 6633     andl(value, 0xffff);
 6634   }
 6635   if (t == T_BYTE || t == T_SHORT) {
 6636     movl(rtmp, value);
 6637     shll(rtmp, 16);
 6638     orl(value, rtmp);
 6639   }
 6640 
 6641   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6642   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6643   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6644     Label L_skip_align2;
 6645     // align source address at 4 bytes address boundary
 6646     if (t == T_BYTE) {
 6647       Label L_skip_align1;
 6648       // One byte misalignment happens only for byte arrays
 6649       testptr(to, 1);
 6650       jccb(Assembler::zero, L_skip_align1);
 6651       movb(Address(to, 0), value);
 6652       increment(to);
 6653       decrement(count);
 6654       BIND(L_skip_align1);
 6655     }
 6656     // Two bytes misalignment happens only for byte and short (char) arrays
 6657     testptr(to, 2);
 6658     jccb(Assembler::zero, L_skip_align2);
 6659     movw(Address(to, 0), value);
 6660     addptr(to, 2);
 6661     subl(count, 1<<(shift-1));
 6662     BIND(L_skip_align2);
 6663   }
 6664   if (UseSSE < 2) {
 6665     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6666     // Fill 32-byte chunks
 6667     subl(count, 8 << shift);
 6668     jcc(Assembler::less, L_check_fill_8_bytes);
 6669     align(16);
 6670 
 6671     BIND(L_fill_32_bytes_loop);
 6672 
 6673     for (int i = 0; i < 32; i += 4) {
 6674       movl(Address(to, i), value);
 6675     }
 6676 
 6677     addptr(to, 32);
 6678     subl(count, 8 << shift);
 6679     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6680     BIND(L_check_fill_8_bytes);
 6681     addl(count, 8 << shift);
 6682     jccb(Assembler::zero, L_exit);
 6683     jmpb(L_fill_8_bytes);
 6684 
 6685     //
 6686     // length is too short, just fill qwords
 6687     //
 6688     BIND(L_fill_8_bytes_loop);
 6689     movl(Address(to, 0), value);
 6690     movl(Address(to, 4), value);
 6691     addptr(to, 8);
 6692     BIND(L_fill_8_bytes);
 6693     subl(count, 1 << (shift + 1));
 6694     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6695     // fall through to fill 4 bytes
 6696   } else {
 6697     Label L_fill_32_bytes;
 6698     if (!UseUnalignedLoadStores) {
 6699       // align to 8 bytes, we know we are 4 byte aligned to start
 6700       testptr(to, 4);
 6701       jccb(Assembler::zero, L_fill_32_bytes);
 6702       movl(Address(to, 0), value);
 6703       addptr(to, 4);
 6704       subl(count, 1<<shift);
 6705     }
 6706     BIND(L_fill_32_bytes);
 6707     {
 6708       assert( UseSSE >= 2, "supported cpu only" );
 6709       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6710       movdl(xtmp, value);
 6711       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6712         Label L_check_fill_32_bytes;
 6713         if (UseAVX > 2) {
 6714           // Fill 64-byte chunks
 6715           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6716 
 6717           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6718           cmpl(count, VM_Version::avx3_threshold());
 6719           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6720 
 6721           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6722 
 6723           subl(count, 16 << shift);
 6724           jccb(Assembler::less, L_check_fill_32_bytes);
 6725           align(16);
 6726 
 6727           BIND(L_fill_64_bytes_loop_avx3);
 6728           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6729           addptr(to, 64);
 6730           subl(count, 16 << shift);
 6731           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6732           jmpb(L_check_fill_32_bytes);
 6733 
 6734           BIND(L_check_fill_64_bytes_avx2);
 6735         }
 6736         // Fill 64-byte chunks
 6737         Label L_fill_64_bytes_loop;
 6738         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6739 
 6740         subl(count, 16 << shift);
 6741         jcc(Assembler::less, L_check_fill_32_bytes);
 6742         align(16);
 6743 
 6744         BIND(L_fill_64_bytes_loop);
 6745         vmovdqu(Address(to, 0), xtmp);
 6746         vmovdqu(Address(to, 32), xtmp);
 6747         addptr(to, 64);
 6748         subl(count, 16 << shift);
 6749         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6750 
 6751         BIND(L_check_fill_32_bytes);
 6752         addl(count, 8 << shift);
 6753         jccb(Assembler::less, L_check_fill_8_bytes);
 6754         vmovdqu(Address(to, 0), xtmp);
 6755         addptr(to, 32);
 6756         subl(count, 8 << shift);
 6757 
 6758         BIND(L_check_fill_8_bytes);
 6759         // clean upper bits of YMM registers
 6760         movdl(xtmp, value);
 6761         pshufd(xtmp, xtmp, 0);
 6762       } else {
 6763         // Fill 32-byte chunks
 6764         pshufd(xtmp, xtmp, 0);
 6765 
 6766         subl(count, 8 << shift);
 6767         jcc(Assembler::less, L_check_fill_8_bytes);
 6768         align(16);
 6769 
 6770         BIND(L_fill_32_bytes_loop);
 6771 
 6772         if (UseUnalignedLoadStores) {
 6773           movdqu(Address(to, 0), xtmp);
 6774           movdqu(Address(to, 16), xtmp);
 6775         } else {
 6776           movq(Address(to, 0), xtmp);
 6777           movq(Address(to, 8), xtmp);
 6778           movq(Address(to, 16), xtmp);
 6779           movq(Address(to, 24), xtmp);
 6780         }
 6781 
 6782         addptr(to, 32);
 6783         subl(count, 8 << shift);
 6784         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6785 
 6786         BIND(L_check_fill_8_bytes);
 6787       }
 6788       addl(count, 8 << shift);
 6789       jccb(Assembler::zero, L_exit);
 6790       jmpb(L_fill_8_bytes);
 6791 
 6792       //
 6793       // length is too short, just fill qwords
 6794       //
 6795       BIND(L_fill_8_bytes_loop);
 6796       movq(Address(to, 0), xtmp);
 6797       addptr(to, 8);
 6798       BIND(L_fill_8_bytes);
 6799       subl(count, 1 << (shift + 1));
 6800       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6801     }
 6802   }
 6803   // fill trailing 4 bytes
 6804   BIND(L_fill_4_bytes);
 6805   testl(count, 1<<shift);
 6806   jccb(Assembler::zero, L_fill_2_bytes);
 6807   movl(Address(to, 0), value);
 6808   if (t == T_BYTE || t == T_SHORT) {
 6809     Label L_fill_byte;
 6810     addptr(to, 4);
 6811     BIND(L_fill_2_bytes);
 6812     // fill trailing 2 bytes
 6813     testl(count, 1<<(shift-1));
 6814     jccb(Assembler::zero, L_fill_byte);
 6815     movw(Address(to, 0), value);
 6816     if (t == T_BYTE) {
 6817       addptr(to, 2);
 6818       BIND(L_fill_byte);
 6819       // fill trailing byte
 6820       testl(count, 1);
 6821       jccb(Assembler::zero, L_exit);
 6822       movb(Address(to, 0), value);
 6823     } else {
 6824       BIND(L_fill_byte);
 6825     }
 6826   } else {
 6827     BIND(L_fill_2_bytes);
 6828   }
 6829   BIND(L_exit);
 6830 }
 6831 
 6832 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6833   switch(type) {
 6834     case T_BYTE:
 6835     case T_BOOLEAN:
 6836       evpbroadcastb(dst, src, vector_len);
 6837       break;
 6838     case T_SHORT:
 6839     case T_CHAR:
 6840       evpbroadcastw(dst, src, vector_len);
 6841       break;
 6842     case T_INT:
 6843     case T_FLOAT:
 6844       evpbroadcastd(dst, src, vector_len);
 6845       break;
 6846     case T_LONG:
 6847     case T_DOUBLE:
 6848       evpbroadcastq(dst, src, vector_len);
 6849       break;
 6850     default:
 6851       fatal("Unhandled type : %s", type2name(type));
 6852       break;
 6853   }
 6854 }
 6855 
 6856 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6857    //@IntrinsicCandidate
 6858    //private static int implEncodeISOArray(byte[] sa, int sp,
 6859    //byte[] da, int dp, int len) {
 6860    //  int i = 0;
 6861    //  for (; i < len; i++) {
 6862    //    char c = StringUTF16.getChar(sa, sp++);
 6863    //    if (c > '\u00FF')
 6864    //      break;
 6865    //    da[dp++] = (byte)c;
 6866    //  }
 6867    //  return i;
 6868    //}
 6869    //
 6870    //@IntrinsicCandidate
 6871    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6872    //    byte[] da, int dp, int len) {
 6873    //  int i = 0;
 6874    //  for (; i < len; i++) {
 6875    //    char c = sa[sp++];
 6876    //    if (c >= '\u0080')
 6877    //      break;
 6878    //    da[dp++] = (byte)c;
 6879    //  }
 6880    //  return i;
 6881    //}
 6882 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6883   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6884   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6885   Register tmp5, Register result, bool ascii) {
 6886 
 6887   // rsi: src
 6888   // rdi: dst
 6889   // rdx: len
 6890   // rcx: tmp5
 6891   // rax: result
 6892   ShortBranchVerifier sbv(this);
 6893   assert_different_registers(src, dst, len, tmp5, result);
 6894   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6895 
 6896   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6897   int short_mask = ascii ? 0xff80 : 0xff00;
 6898 
 6899   // set result
 6900   xorl(result, result);
 6901   // check for zero length
 6902   testl(len, len);
 6903   jcc(Assembler::zero, L_done);
 6904 
 6905   movl(result, len);
 6906 
 6907   // Setup pointers
 6908   lea(src, Address(src, len, Address::times_2)); // char[]
 6909   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6910   negptr(len);
 6911 
 6912   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6913     Label L_copy_8_chars, L_copy_8_chars_exit;
 6914     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6915 
 6916     if (UseAVX >= 2) {
 6917       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6918       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6919       movdl(tmp1Reg, tmp5);
 6920       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 6921       jmp(L_chars_32_check);
 6922 
 6923       bind(L_copy_32_chars);
 6924       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 6925       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 6926       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6927       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6928       jccb(Assembler::notZero, L_copy_32_chars_exit);
 6929       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6930       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 6931       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 6932 
 6933       bind(L_chars_32_check);
 6934       addptr(len, 32);
 6935       jcc(Assembler::lessEqual, L_copy_32_chars);
 6936 
 6937       bind(L_copy_32_chars_exit);
 6938       subptr(len, 16);
 6939       jccb(Assembler::greater, L_copy_16_chars_exit);
 6940 
 6941     } else if (UseSSE42Intrinsics) {
 6942       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6943       movdl(tmp1Reg, tmp5);
 6944       pshufd(tmp1Reg, tmp1Reg, 0);
 6945       jmpb(L_chars_16_check);
 6946     }
 6947 
 6948     bind(L_copy_16_chars);
 6949     if (UseAVX >= 2) {
 6950       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 6951       vptest(tmp2Reg, tmp1Reg);
 6952       jcc(Assembler::notZero, L_copy_16_chars_exit);
 6953       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 6954       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 6955     } else {
 6956       if (UseAVX > 0) {
 6957         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6958         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6959         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 6960       } else {
 6961         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6962         por(tmp2Reg, tmp3Reg);
 6963         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6964         por(tmp2Reg, tmp4Reg);
 6965       }
 6966       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6967       jccb(Assembler::notZero, L_copy_16_chars_exit);
 6968       packuswb(tmp3Reg, tmp4Reg);
 6969     }
 6970     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 6971 
 6972     bind(L_chars_16_check);
 6973     addptr(len, 16);
 6974     jcc(Assembler::lessEqual, L_copy_16_chars);
 6975 
 6976     bind(L_copy_16_chars_exit);
 6977     if (UseAVX >= 2) {
 6978       // clean upper bits of YMM registers
 6979       vpxor(tmp2Reg, tmp2Reg);
 6980       vpxor(tmp3Reg, tmp3Reg);
 6981       vpxor(tmp4Reg, tmp4Reg);
 6982       movdl(tmp1Reg, tmp5);
 6983       pshufd(tmp1Reg, tmp1Reg, 0);
 6984     }
 6985     subptr(len, 8);
 6986     jccb(Assembler::greater, L_copy_8_chars_exit);
 6987 
 6988     bind(L_copy_8_chars);
 6989     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 6990     ptest(tmp3Reg, tmp1Reg);
 6991     jccb(Assembler::notZero, L_copy_8_chars_exit);
 6992     packuswb(tmp3Reg, tmp1Reg);
 6993     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 6994     addptr(len, 8);
 6995     jccb(Assembler::lessEqual, L_copy_8_chars);
 6996 
 6997     bind(L_copy_8_chars_exit);
 6998     subptr(len, 8);
 6999     jccb(Assembler::zero, L_done);
 7000   }
 7001 
 7002   bind(L_copy_1_char);
 7003   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 7004   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 7005   jccb(Assembler::notZero, L_copy_1_char_exit);
 7006   movb(Address(dst, len, Address::times_1, 0), tmp5);
 7007   addptr(len, 1);
 7008   jccb(Assembler::less, L_copy_1_char);
 7009 
 7010   bind(L_copy_1_char_exit);
 7011   addptr(result, len); // len is negative count of not processed elements
 7012 
 7013   bind(L_done);
 7014 }
 7015 
 7016 #ifdef _LP64
 7017 /**
 7018  * Helper for multiply_to_len().
 7019  */
 7020 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 7021   addq(dest_lo, src1);
 7022   adcq(dest_hi, 0);
 7023   addq(dest_lo, src2);
 7024   adcq(dest_hi, 0);
 7025 }
 7026 
 7027 /**
 7028  * Multiply 64 bit by 64 bit first loop.
 7029  */
 7030 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 7031                                            Register y, Register y_idx, Register z,
 7032                                            Register carry, Register product,
 7033                                            Register idx, Register kdx) {
 7034   //
 7035   //  jlong carry, x[], y[], z[];
 7036   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7037   //    huge_128 product = y[idx] * x[xstart] + carry;
 7038   //    z[kdx] = (jlong)product;
 7039   //    carry  = (jlong)(product >>> 64);
 7040   //  }
 7041   //  z[xstart] = carry;
 7042   //
 7043 
 7044   Label L_first_loop, L_first_loop_exit;
 7045   Label L_one_x, L_one_y, L_multiply;
 7046 
 7047   decrementl(xstart);
 7048   jcc(Assembler::negative, L_one_x);
 7049 
 7050   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7051   rorq(x_xstart, 32); // convert big-endian to little-endian
 7052 
 7053   bind(L_first_loop);
 7054   decrementl(idx);
 7055   jcc(Assembler::negative, L_first_loop_exit);
 7056   decrementl(idx);
 7057   jcc(Assembler::negative, L_one_y);
 7058   movq(y_idx, Address(y, idx, Address::times_4,  0));
 7059   rorq(y_idx, 32); // convert big-endian to little-endian
 7060   bind(L_multiply);
 7061   movq(product, x_xstart);
 7062   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 7063   addq(product, carry);
 7064   adcq(rdx, 0);
 7065   subl(kdx, 2);
 7066   movl(Address(z, kdx, Address::times_4,  4), product);
 7067   shrq(product, 32);
 7068   movl(Address(z, kdx, Address::times_4,  0), product);
 7069   movq(carry, rdx);
 7070   jmp(L_first_loop);
 7071 
 7072   bind(L_one_y);
 7073   movl(y_idx, Address(y,  0));
 7074   jmp(L_multiply);
 7075 
 7076   bind(L_one_x);
 7077   movl(x_xstart, Address(x,  0));
 7078   jmp(L_first_loop);
 7079 
 7080   bind(L_first_loop_exit);
 7081 }
 7082 
 7083 /**
 7084  * Multiply 64 bit by 64 bit and add 128 bit.
 7085  */
 7086 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 7087                                             Register yz_idx, Register idx,
 7088                                             Register carry, Register product, int offset) {
 7089   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 7090   //     z[kdx] = (jlong)product;
 7091 
 7092   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 7093   rorq(yz_idx, 32); // convert big-endian to little-endian
 7094   movq(product, x_xstart);
 7095   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 7096   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 7097   rorq(yz_idx, 32); // convert big-endian to little-endian
 7098 
 7099   add2_with_carry(rdx, product, carry, yz_idx);
 7100 
 7101   movl(Address(z, idx, Address::times_4,  offset+4), product);
 7102   shrq(product, 32);
 7103   movl(Address(z, idx, Address::times_4,  offset), product);
 7104 
 7105 }
 7106 
 7107 /**
 7108  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 7109  */
 7110 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 7111                                              Register yz_idx, Register idx, Register jdx,
 7112                                              Register carry, Register product,
 7113                                              Register carry2) {
 7114   //   jlong carry, x[], y[], z[];
 7115   //   int kdx = ystart+1;
 7116   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7117   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 7118   //     z[kdx+idx+1] = (jlong)product;
 7119   //     jlong carry2  = (jlong)(product >>> 64);
 7120   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 7121   //     z[kdx+idx] = (jlong)product;
 7122   //     carry  = (jlong)(product >>> 64);
 7123   //   }
 7124   //   idx += 2;
 7125   //   if (idx > 0) {
 7126   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 7127   //     z[kdx+idx] = (jlong)product;
 7128   //     carry  = (jlong)(product >>> 64);
 7129   //   }
 7130   //
 7131 
 7132   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7133 
 7134   movl(jdx, idx);
 7135   andl(jdx, 0xFFFFFFFC);
 7136   shrl(jdx, 2);
 7137 
 7138   bind(L_third_loop);
 7139   subl(jdx, 1);
 7140   jcc(Assembler::negative, L_third_loop_exit);
 7141   subl(idx, 4);
 7142 
 7143   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 7144   movq(carry2, rdx);
 7145 
 7146   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 7147   movq(carry, rdx);
 7148   jmp(L_third_loop);
 7149 
 7150   bind (L_third_loop_exit);
 7151 
 7152   andl (idx, 0x3);
 7153   jcc(Assembler::zero, L_post_third_loop_done);
 7154 
 7155   Label L_check_1;
 7156   subl(idx, 2);
 7157   jcc(Assembler::negative, L_check_1);
 7158 
 7159   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 7160   movq(carry, rdx);
 7161 
 7162   bind (L_check_1);
 7163   addl (idx, 0x2);
 7164   andl (idx, 0x1);
 7165   subl(idx, 1);
 7166   jcc(Assembler::negative, L_post_third_loop_done);
 7167 
 7168   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 7169   movq(product, x_xstart);
 7170   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 7171   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 7172 
 7173   add2_with_carry(rdx, product, yz_idx, carry);
 7174 
 7175   movl(Address(z, idx, Address::times_4,  0), product);
 7176   shrq(product, 32);
 7177 
 7178   shlq(rdx, 32);
 7179   orq(product, rdx);
 7180   movq(carry, product);
 7181 
 7182   bind(L_post_third_loop_done);
 7183 }
 7184 
 7185 /**
 7186  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 7187  *
 7188  */
 7189 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 7190                                                   Register carry, Register carry2,
 7191                                                   Register idx, Register jdx,
 7192                                                   Register yz_idx1, Register yz_idx2,
 7193                                                   Register tmp, Register tmp3, Register tmp4) {
 7194   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 7195 
 7196   //   jlong carry, x[], y[], z[];
 7197   //   int kdx = ystart+1;
 7198   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7199   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 7200   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 7201   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 7202   //     carry  = (jlong)(tmp4 >>> 64);
 7203   //     z[kdx+idx+1] = (jlong)tmp3;
 7204   //     z[kdx+idx] = (jlong)tmp4;
 7205   //   }
 7206   //   idx += 2;
 7207   //   if (idx > 0) {
 7208   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 7209   //     z[kdx+idx] = (jlong)yz_idx1;
 7210   //     carry  = (jlong)(yz_idx1 >>> 64);
 7211   //   }
 7212   //
 7213 
 7214   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7215 
 7216   movl(jdx, idx);
 7217   andl(jdx, 0xFFFFFFFC);
 7218   shrl(jdx, 2);
 7219 
 7220   bind(L_third_loop);
 7221   subl(jdx, 1);
 7222   jcc(Assembler::negative, L_third_loop_exit);
 7223   subl(idx, 4);
 7224 
 7225   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 7226   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 7227   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 7228   rorxq(yz_idx2, yz_idx2, 32);
 7229 
 7230   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 7231   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 7232 
 7233   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 7234   rorxq(yz_idx1, yz_idx1, 32);
 7235   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7236   rorxq(yz_idx2, yz_idx2, 32);
 7237 
 7238   if (VM_Version::supports_adx()) {
 7239     adcxq(tmp3, carry);
 7240     adoxq(tmp3, yz_idx1);
 7241 
 7242     adcxq(tmp4, tmp);
 7243     adoxq(tmp4, yz_idx2);
 7244 
 7245     movl(carry, 0); // does not affect flags
 7246     adcxq(carry2, carry);
 7247     adoxq(carry2, carry);
 7248   } else {
 7249     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 7250     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 7251   }
 7252   movq(carry, carry2);
 7253 
 7254   movl(Address(z, idx, Address::times_4, 12), tmp3);
 7255   shrq(tmp3, 32);
 7256   movl(Address(z, idx, Address::times_4,  8), tmp3);
 7257 
 7258   movl(Address(z, idx, Address::times_4,  4), tmp4);
 7259   shrq(tmp4, 32);
 7260   movl(Address(z, idx, Address::times_4,  0), tmp4);
 7261 
 7262   jmp(L_third_loop);
 7263 
 7264   bind (L_third_loop_exit);
 7265 
 7266   andl (idx, 0x3);
 7267   jcc(Assembler::zero, L_post_third_loop_done);
 7268 
 7269   Label L_check_1;
 7270   subl(idx, 2);
 7271   jcc(Assembler::negative, L_check_1);
 7272 
 7273   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 7274   rorxq(yz_idx1, yz_idx1, 32);
 7275   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 7276   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7277   rorxq(yz_idx2, yz_idx2, 32);
 7278 
 7279   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 7280 
 7281   movl(Address(z, idx, Address::times_4,  4), tmp3);
 7282   shrq(tmp3, 32);
 7283   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7284   movq(carry, tmp4);
 7285 
 7286   bind (L_check_1);
 7287   addl (idx, 0x2);
 7288   andl (idx, 0x1);
 7289   subl(idx, 1);
 7290   jcc(Assembler::negative, L_post_third_loop_done);
 7291   movl(tmp4, Address(y, idx, Address::times_4,  0));
 7292   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 7293   movl(tmp4, Address(z, idx, Address::times_4,  0));
 7294 
 7295   add2_with_carry(carry2, tmp3, tmp4, carry);
 7296 
 7297   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7298   shrq(tmp3, 32);
 7299 
 7300   shlq(carry2, 32);
 7301   orq(tmp3, carry2);
 7302   movq(carry, tmp3);
 7303 
 7304   bind(L_post_third_loop_done);
 7305 }
 7306 
 7307 /**
 7308  * Code for BigInteger::multiplyToLen() intrinsic.
 7309  *
 7310  * rdi: x
 7311  * rax: xlen
 7312  * rsi: y
 7313  * rcx: ylen
 7314  * r8:  z
 7315  * r11: zlen
 7316  * r12: tmp1
 7317  * r13: tmp2
 7318  * r14: tmp3
 7319  * r15: tmp4
 7320  * rbx: tmp5
 7321  *
 7322  */
 7323 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
 7324                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 7325   ShortBranchVerifier sbv(this);
 7326   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 7327 
 7328   push(tmp1);
 7329   push(tmp2);
 7330   push(tmp3);
 7331   push(tmp4);
 7332   push(tmp5);
 7333 
 7334   push(xlen);
 7335   push(zlen);
 7336 
 7337   const Register idx = tmp1;
 7338   const Register kdx = tmp2;
 7339   const Register xstart = tmp3;
 7340 
 7341   const Register y_idx = tmp4;
 7342   const Register carry = tmp5;
 7343   const Register product  = xlen;
 7344   const Register x_xstart = zlen;  // reuse register
 7345 
 7346   // First Loop.
 7347   //
 7348   //  final static long LONG_MASK = 0xffffffffL;
 7349   //  int xstart = xlen - 1;
 7350   //  int ystart = ylen - 1;
 7351   //  long carry = 0;
 7352   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7353   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7354   //    z[kdx] = (int)product;
 7355   //    carry = product >>> 32;
 7356   //  }
 7357   //  z[xstart] = (int)carry;
 7358   //
 7359 
 7360   movl(idx, ylen);      // idx = ylen;
 7361   movl(kdx, zlen);      // kdx = xlen+ylen;
 7362   xorq(carry, carry);   // carry = 0;
 7363 
 7364   Label L_done;
 7365 
 7366   movl(xstart, xlen);
 7367   decrementl(xstart);
 7368   jcc(Assembler::negative, L_done);
 7369 
 7370   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7371 
 7372   Label L_second_loop;
 7373   testl(kdx, kdx);
 7374   jcc(Assembler::zero, L_second_loop);
 7375 
 7376   Label L_carry;
 7377   subl(kdx, 1);
 7378   jcc(Assembler::zero, L_carry);
 7379 
 7380   movl(Address(z, kdx, Address::times_4,  0), carry);
 7381   shrq(carry, 32);
 7382   subl(kdx, 1);
 7383 
 7384   bind(L_carry);
 7385   movl(Address(z, kdx, Address::times_4,  0), carry);
 7386 
 7387   // Second and third (nested) loops.
 7388   //
 7389   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7390   //   carry = 0;
 7391   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7392   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7393   //                    (z[k] & LONG_MASK) + carry;
 7394   //     z[k] = (int)product;
 7395   //     carry = product >>> 32;
 7396   //   }
 7397   //   z[i] = (int)carry;
 7398   // }
 7399   //
 7400   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7401 
 7402   const Register jdx = tmp1;
 7403 
 7404   bind(L_second_loop);
 7405   xorl(carry, carry);    // carry = 0;
 7406   movl(jdx, ylen);       // j = ystart+1
 7407 
 7408   subl(xstart, 1);       // i = xstart-1;
 7409   jcc(Assembler::negative, L_done);
 7410 
 7411   push (z);
 7412 
 7413   Label L_last_x;
 7414   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7415   subl(xstart, 1);       // i = xstart-1;
 7416   jcc(Assembler::negative, L_last_x);
 7417 
 7418   if (UseBMI2Instructions) {
 7419     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7420     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7421   } else {
 7422     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7423     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7424   }
 7425 
 7426   Label L_third_loop_prologue;
 7427   bind(L_third_loop_prologue);
 7428 
 7429   push (x);
 7430   push (xstart);
 7431   push (ylen);
 7432 
 7433 
 7434   if (UseBMI2Instructions) {
 7435     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7436   } else { // !UseBMI2Instructions
 7437     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7438   }
 7439 
 7440   pop(ylen);
 7441   pop(xlen);
 7442   pop(x);
 7443   pop(z);
 7444 
 7445   movl(tmp3, xlen);
 7446   addl(tmp3, 1);
 7447   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7448   subl(tmp3, 1);
 7449   jccb(Assembler::negative, L_done);
 7450 
 7451   shrq(carry, 32);
 7452   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7453   jmp(L_second_loop);
 7454 
 7455   // Next infrequent code is moved outside loops.
 7456   bind(L_last_x);
 7457   if (UseBMI2Instructions) {
 7458     movl(rdx, Address(x,  0));
 7459   } else {
 7460     movl(x_xstart, Address(x,  0));
 7461   }
 7462   jmp(L_third_loop_prologue);
 7463 
 7464   bind(L_done);
 7465 
 7466   pop(zlen);
 7467   pop(xlen);
 7468 
 7469   pop(tmp5);
 7470   pop(tmp4);
 7471   pop(tmp3);
 7472   pop(tmp2);
 7473   pop(tmp1);
 7474 }
 7475 
 7476 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7477   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7478   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7479   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7480   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7481   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7482   Label SAME_TILL_END, DONE;
 7483   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7484 
 7485   //scale is in rcx in both Win64 and Unix
 7486   ShortBranchVerifier sbv(this);
 7487 
 7488   shlq(length);
 7489   xorq(result, result);
 7490 
 7491   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7492       VM_Version::supports_avx512vlbw()) {
 7493     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7494 
 7495     cmpq(length, 64);
 7496     jcc(Assembler::less, VECTOR32_TAIL);
 7497 
 7498     movq(tmp1, length);
 7499     andq(tmp1, 0x3F);      // tail count
 7500     andq(length, ~(0x3F)); //vector count
 7501 
 7502     bind(VECTOR64_LOOP);
 7503     // AVX512 code to compare 64 byte vectors.
 7504     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7505     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7506     kortestql(k7, k7);
 7507     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7508     addq(result, 64);
 7509     subq(length, 64);
 7510     jccb(Assembler::notZero, VECTOR64_LOOP);
 7511 
 7512     //bind(VECTOR64_TAIL);
 7513     testq(tmp1, tmp1);
 7514     jcc(Assembler::zero, SAME_TILL_END);
 7515 
 7516     //bind(VECTOR64_TAIL);
 7517     // AVX512 code to compare up to 63 byte vectors.
 7518     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7519     shlxq(tmp2, tmp2, tmp1);
 7520     notq(tmp2);
 7521     kmovql(k3, tmp2);
 7522 
 7523     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7524     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7525 
 7526     ktestql(k7, k3);
 7527     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7528 
 7529     bind(VECTOR64_NOT_EQUAL);
 7530     kmovql(tmp1, k7);
 7531     notq(tmp1);
 7532     tzcntq(tmp1, tmp1);
 7533     addq(result, tmp1);
 7534     shrq(result);
 7535     jmp(DONE);
 7536     bind(VECTOR32_TAIL);
 7537   }
 7538 
 7539   cmpq(length, 8);
 7540   jcc(Assembler::equal, VECTOR8_LOOP);
 7541   jcc(Assembler::less, VECTOR4_TAIL);
 7542 
 7543   if (UseAVX >= 2) {
 7544     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7545 
 7546     cmpq(length, 16);
 7547     jcc(Assembler::equal, VECTOR16_LOOP);
 7548     jcc(Assembler::less, VECTOR8_LOOP);
 7549 
 7550     cmpq(length, 32);
 7551     jccb(Assembler::less, VECTOR16_TAIL);
 7552 
 7553     subq(length, 32);
 7554     bind(VECTOR32_LOOP);
 7555     vmovdqu(rymm0, Address(obja, result));
 7556     vmovdqu(rymm1, Address(objb, result));
 7557     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7558     vptest(rymm2, rymm2);
 7559     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7560     addq(result, 32);
 7561     subq(length, 32);
 7562     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7563     addq(length, 32);
 7564     jcc(Assembler::equal, SAME_TILL_END);
 7565     //falling through if less than 32 bytes left //close the branch here.
 7566 
 7567     bind(VECTOR16_TAIL);
 7568     cmpq(length, 16);
 7569     jccb(Assembler::less, VECTOR8_TAIL);
 7570     bind(VECTOR16_LOOP);
 7571     movdqu(rymm0, Address(obja, result));
 7572     movdqu(rymm1, Address(objb, result));
 7573     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7574     ptest(rymm2, rymm2);
 7575     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7576     addq(result, 16);
 7577     subq(length, 16);
 7578     jcc(Assembler::equal, SAME_TILL_END);
 7579     //falling through if less than 16 bytes left
 7580   } else {//regular intrinsics
 7581 
 7582     cmpq(length, 16);
 7583     jccb(Assembler::less, VECTOR8_TAIL);
 7584 
 7585     subq(length, 16);
 7586     bind(VECTOR16_LOOP);
 7587     movdqu(rymm0, Address(obja, result));
 7588     movdqu(rymm1, Address(objb, result));
 7589     pxor(rymm0, rymm1);
 7590     ptest(rymm0, rymm0);
 7591     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7592     addq(result, 16);
 7593     subq(length, 16);
 7594     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7595     addq(length, 16);
 7596     jcc(Assembler::equal, SAME_TILL_END);
 7597     //falling through if less than 16 bytes left
 7598   }
 7599 
 7600   bind(VECTOR8_TAIL);
 7601   cmpq(length, 8);
 7602   jccb(Assembler::less, VECTOR4_TAIL);
 7603   bind(VECTOR8_LOOP);
 7604   movq(tmp1, Address(obja, result));
 7605   movq(tmp2, Address(objb, result));
 7606   xorq(tmp1, tmp2);
 7607   testq(tmp1, tmp1);
 7608   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7609   addq(result, 8);
 7610   subq(length, 8);
 7611   jcc(Assembler::equal, SAME_TILL_END);
 7612   //falling through if less than 8 bytes left
 7613 
 7614   bind(VECTOR4_TAIL);
 7615   cmpq(length, 4);
 7616   jccb(Assembler::less, BYTES_TAIL);
 7617   bind(VECTOR4_LOOP);
 7618   movl(tmp1, Address(obja, result));
 7619   xorl(tmp1, Address(objb, result));
 7620   testl(tmp1, tmp1);
 7621   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7622   addq(result, 4);
 7623   subq(length, 4);
 7624   jcc(Assembler::equal, SAME_TILL_END);
 7625   //falling through if less than 4 bytes left
 7626 
 7627   bind(BYTES_TAIL);
 7628   bind(BYTES_LOOP);
 7629   load_unsigned_byte(tmp1, Address(obja, result));
 7630   load_unsigned_byte(tmp2, Address(objb, result));
 7631   xorl(tmp1, tmp2);
 7632   testl(tmp1, tmp1);
 7633   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7634   decq(length);
 7635   jcc(Assembler::zero, SAME_TILL_END);
 7636   incq(result);
 7637   load_unsigned_byte(tmp1, Address(obja, result));
 7638   load_unsigned_byte(tmp2, Address(objb, result));
 7639   xorl(tmp1, tmp2);
 7640   testl(tmp1, tmp1);
 7641   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7642   decq(length);
 7643   jcc(Assembler::zero, SAME_TILL_END);
 7644   incq(result);
 7645   load_unsigned_byte(tmp1, Address(obja, result));
 7646   load_unsigned_byte(tmp2, Address(objb, result));
 7647   xorl(tmp1, tmp2);
 7648   testl(tmp1, tmp1);
 7649   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7650   jmp(SAME_TILL_END);
 7651 
 7652   if (UseAVX >= 2) {
 7653     bind(VECTOR32_NOT_EQUAL);
 7654     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7655     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7656     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7657     vpmovmskb(tmp1, rymm0);
 7658     bsfq(tmp1, tmp1);
 7659     addq(result, tmp1);
 7660     shrq(result);
 7661     jmp(DONE);
 7662   }
 7663 
 7664   bind(VECTOR16_NOT_EQUAL);
 7665   if (UseAVX >= 2) {
 7666     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7667     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7668     pxor(rymm0, rymm2);
 7669   } else {
 7670     pcmpeqb(rymm2, rymm2);
 7671     pxor(rymm0, rymm1);
 7672     pcmpeqb(rymm0, rymm1);
 7673     pxor(rymm0, rymm2);
 7674   }
 7675   pmovmskb(tmp1, rymm0);
 7676   bsfq(tmp1, tmp1);
 7677   addq(result, tmp1);
 7678   shrq(result);
 7679   jmpb(DONE);
 7680 
 7681   bind(VECTOR8_NOT_EQUAL);
 7682   bind(VECTOR4_NOT_EQUAL);
 7683   bsfq(tmp1, tmp1);
 7684   shrq(tmp1, 3);
 7685   addq(result, tmp1);
 7686   bind(BYTES_NOT_EQUAL);
 7687   shrq(result);
 7688   jmpb(DONE);
 7689 
 7690   bind(SAME_TILL_END);
 7691   mov64(result, -1);
 7692 
 7693   bind(DONE);
 7694 }
 7695 
 7696 //Helper functions for square_to_len()
 7697 
 7698 /**
 7699  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7700  * Preserves x and z and modifies rest of the registers.
 7701  */
 7702 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7703   // Perform square and right shift by 1
 7704   // Handle odd xlen case first, then for even xlen do the following
 7705   // jlong carry = 0;
 7706   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7707   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7708   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7709   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7710   //     carry = (jlong)product;
 7711   // }
 7712 
 7713   xorq(tmp5, tmp5);     // carry
 7714   xorq(rdxReg, rdxReg);
 7715   xorl(tmp1, tmp1);     // index for x
 7716   xorl(tmp4, tmp4);     // index for z
 7717 
 7718   Label L_first_loop, L_first_loop_exit;
 7719 
 7720   testl(xlen, 1);
 7721   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7722 
 7723   // Square and right shift by 1 the odd element using 32 bit multiply
 7724   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7725   imulq(raxReg, raxReg);
 7726   shrq(raxReg, 1);
 7727   adcq(tmp5, 0);
 7728   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7729   incrementl(tmp1);
 7730   addl(tmp4, 2);
 7731 
 7732   // Square and  right shift by 1 the rest using 64 bit multiply
 7733   bind(L_first_loop);
 7734   cmpptr(tmp1, xlen);
 7735   jccb(Assembler::equal, L_first_loop_exit);
 7736 
 7737   // Square
 7738   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7739   rorq(raxReg, 32);    // convert big-endian to little-endian
 7740   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7741 
 7742   // Right shift by 1 and save carry
 7743   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7744   rcrq(rdxReg, 1);
 7745   rcrq(raxReg, 1);
 7746   adcq(tmp5, 0);
 7747 
 7748   // Store result in z
 7749   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7750   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7751 
 7752   // Update indices for x and z
 7753   addl(tmp1, 2);
 7754   addl(tmp4, 4);
 7755   jmp(L_first_loop);
 7756 
 7757   bind(L_first_loop_exit);
 7758 }
 7759 
 7760 
 7761 /**
 7762  * Perform the following multiply add operation using BMI2 instructions
 7763  * carry:sum = sum + op1*op2 + carry
 7764  * op2 should be in rdx
 7765  * op2 is preserved, all other registers are modified
 7766  */
 7767 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7768   // assert op2 is rdx
 7769   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7770   addq(sum, carry);
 7771   adcq(tmp2, 0);
 7772   addq(sum, op1);
 7773   adcq(tmp2, 0);
 7774   movq(carry, tmp2);
 7775 }
 7776 
 7777 /**
 7778  * Perform the following multiply add operation:
 7779  * carry:sum = sum + op1*op2 + carry
 7780  * Preserves op1, op2 and modifies rest of registers
 7781  */
 7782 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7783   // rdx:rax = op1 * op2
 7784   movq(raxReg, op2);
 7785   mulq(op1);
 7786 
 7787   //  rdx:rax = sum + carry + rdx:rax
 7788   addq(sum, carry);
 7789   adcq(rdxReg, 0);
 7790   addq(sum, raxReg);
 7791   adcq(rdxReg, 0);
 7792 
 7793   // carry:sum = rdx:sum
 7794   movq(carry, rdxReg);
 7795 }
 7796 
 7797 /**
 7798  * Add 64 bit long carry into z[] with carry propagation.
 7799  * Preserves z and carry register values and modifies rest of registers.
 7800  *
 7801  */
 7802 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7803   Label L_fourth_loop, L_fourth_loop_exit;
 7804 
 7805   movl(tmp1, 1);
 7806   subl(zlen, 2);
 7807   addq(Address(z, zlen, Address::times_4, 0), carry);
 7808 
 7809   bind(L_fourth_loop);
 7810   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7811   subl(zlen, 2);
 7812   jccb(Assembler::negative, L_fourth_loop_exit);
 7813   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7814   jmp(L_fourth_loop);
 7815   bind(L_fourth_loop_exit);
 7816 }
 7817 
 7818 /**
 7819  * Shift z[] left by 1 bit.
 7820  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7821  *
 7822  */
 7823 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7824 
 7825   Label L_fifth_loop, L_fifth_loop_exit;
 7826 
 7827   // Fifth loop
 7828   // Perform primitiveLeftShift(z, zlen, 1)
 7829 
 7830   const Register prev_carry = tmp1;
 7831   const Register new_carry = tmp4;
 7832   const Register value = tmp2;
 7833   const Register zidx = tmp3;
 7834 
 7835   // int zidx, carry;
 7836   // long value;
 7837   // carry = 0;
 7838   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7839   //    (carry:value)  = (z[i] << 1) | carry ;
 7840   //    z[i] = value;
 7841   // }
 7842 
 7843   movl(zidx, zlen);
 7844   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7845 
 7846   bind(L_fifth_loop);
 7847   decl(zidx);  // Use decl to preserve carry flag
 7848   decl(zidx);
 7849   jccb(Assembler::negative, L_fifth_loop_exit);
 7850 
 7851   if (UseBMI2Instructions) {
 7852      movq(value, Address(z, zidx, Address::times_4, 0));
 7853      rclq(value, 1);
 7854      rorxq(value, value, 32);
 7855      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7856   }
 7857   else {
 7858     // clear new_carry
 7859     xorl(new_carry, new_carry);
 7860 
 7861     // Shift z[i] by 1, or in previous carry and save new carry
 7862     movq(value, Address(z, zidx, Address::times_4, 0));
 7863     shlq(value, 1);
 7864     adcl(new_carry, 0);
 7865 
 7866     orq(value, prev_carry);
 7867     rorq(value, 0x20);
 7868     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7869 
 7870     // Set previous carry = new carry
 7871     movl(prev_carry, new_carry);
 7872   }
 7873   jmp(L_fifth_loop);
 7874 
 7875   bind(L_fifth_loop_exit);
 7876 }
 7877 
 7878 
 7879 /**
 7880  * Code for BigInteger::squareToLen() intrinsic
 7881  *
 7882  * rdi: x
 7883  * rsi: len
 7884  * r8:  z
 7885  * rcx: zlen
 7886  * r12: tmp1
 7887  * r13: tmp2
 7888  * r14: tmp3
 7889  * r15: tmp4
 7890  * rbx: tmp5
 7891  *
 7892  */
 7893 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7894 
 7895   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7896   push(tmp1);
 7897   push(tmp2);
 7898   push(tmp3);
 7899   push(tmp4);
 7900   push(tmp5);
 7901 
 7902   // First loop
 7903   // Store the squares, right shifted one bit (i.e., divided by 2).
 7904   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7905 
 7906   // Add in off-diagonal sums.
 7907   //
 7908   // Second, third (nested) and fourth loops.
 7909   // zlen +=2;
 7910   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7911   //    carry = 0;
 7912   //    long op2 = x[xidx:xidx+1];
 7913   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7914   //       k -= 2;
 7915   //       long op1 = x[j:j+1];
 7916   //       long sum = z[k:k+1];
 7917   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7918   //       z[k:k+1] = sum;
 7919   //    }
 7920   //    add_one_64(z, k, carry, tmp_regs);
 7921   // }
 7922 
 7923   const Register carry = tmp5;
 7924   const Register sum = tmp3;
 7925   const Register op1 = tmp4;
 7926   Register op2 = tmp2;
 7927 
 7928   push(zlen);
 7929   push(len);
 7930   addl(zlen,2);
 7931   bind(L_second_loop);
 7932   xorq(carry, carry);
 7933   subl(zlen, 4);
 7934   subl(len, 2);
 7935   push(zlen);
 7936   push(len);
 7937   cmpl(len, 0);
 7938   jccb(Assembler::lessEqual, L_second_loop_exit);
 7939 
 7940   // Multiply an array by one 64 bit long.
 7941   if (UseBMI2Instructions) {
 7942     op2 = rdxReg;
 7943     movq(op2, Address(x, len, Address::times_4,  0));
 7944     rorxq(op2, op2, 32);
 7945   }
 7946   else {
 7947     movq(op2, Address(x, len, Address::times_4,  0));
 7948     rorq(op2, 32);
 7949   }
 7950 
 7951   bind(L_third_loop);
 7952   decrementl(len);
 7953   jccb(Assembler::negative, L_third_loop_exit);
 7954   decrementl(len);
 7955   jccb(Assembler::negative, L_last_x);
 7956 
 7957   movq(op1, Address(x, len, Address::times_4,  0));
 7958   rorq(op1, 32);
 7959 
 7960   bind(L_multiply);
 7961   subl(zlen, 2);
 7962   movq(sum, Address(z, zlen, Address::times_4,  0));
 7963 
 7964   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 7965   if (UseBMI2Instructions) {
 7966     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 7967   }
 7968   else {
 7969     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7970   }
 7971 
 7972   movq(Address(z, zlen, Address::times_4, 0), sum);
 7973 
 7974   jmp(L_third_loop);
 7975   bind(L_third_loop_exit);
 7976 
 7977   // Fourth loop
 7978   // Add 64 bit long carry into z with carry propagation.
 7979   // Uses offsetted zlen.
 7980   add_one_64(z, zlen, carry, tmp1);
 7981 
 7982   pop(len);
 7983   pop(zlen);
 7984   jmp(L_second_loop);
 7985 
 7986   // Next infrequent code is moved outside loops.
 7987   bind(L_last_x);
 7988   movl(op1, Address(x, 0));
 7989   jmp(L_multiply);
 7990 
 7991   bind(L_second_loop_exit);
 7992   pop(len);
 7993   pop(zlen);
 7994   pop(len);
 7995   pop(zlen);
 7996 
 7997   // Fifth loop
 7998   // Shift z left 1 bit.
 7999   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 8000 
 8001   // z[zlen-1] |= x[len-1] & 1;
 8002   movl(tmp3, Address(x, len, Address::times_4, -4));
 8003   andl(tmp3, 1);
 8004   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 8005 
 8006   pop(tmp5);
 8007   pop(tmp4);
 8008   pop(tmp3);
 8009   pop(tmp2);
 8010   pop(tmp1);
 8011 }
 8012 
 8013 /**
 8014  * Helper function for mul_add()
 8015  * Multiply the in[] by int k and add to out[] starting at offset offs using
 8016  * 128 bit by 32 bit multiply and return the carry in tmp5.
 8017  * Only quad int aligned length of in[] is operated on in this function.
 8018  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 8019  * This function preserves out, in and k registers.
 8020  * len and offset point to the appropriate index in "in" & "out" correspondingly
 8021  * tmp5 has the carry.
 8022  * other registers are temporary and are modified.
 8023  *
 8024  */
 8025 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 8026   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 8027   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8028 
 8029   Label L_first_loop, L_first_loop_exit;
 8030 
 8031   movl(tmp1, len);
 8032   shrl(tmp1, 2);
 8033 
 8034   bind(L_first_loop);
 8035   subl(tmp1, 1);
 8036   jccb(Assembler::negative, L_first_loop_exit);
 8037 
 8038   subl(len, 4);
 8039   subl(offset, 4);
 8040 
 8041   Register op2 = tmp2;
 8042   const Register sum = tmp3;
 8043   const Register op1 = tmp4;
 8044   const Register carry = tmp5;
 8045 
 8046   if (UseBMI2Instructions) {
 8047     op2 = rdxReg;
 8048   }
 8049 
 8050   movq(op1, Address(in, len, Address::times_4,  8));
 8051   rorq(op1, 32);
 8052   movq(sum, Address(out, offset, Address::times_4,  8));
 8053   rorq(sum, 32);
 8054   if (UseBMI2Instructions) {
 8055     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8056   }
 8057   else {
 8058     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8059   }
 8060   // Store back in big endian from little endian
 8061   rorq(sum, 0x20);
 8062   movq(Address(out, offset, Address::times_4,  8), sum);
 8063 
 8064   movq(op1, Address(in, len, Address::times_4,  0));
 8065   rorq(op1, 32);
 8066   movq(sum, Address(out, offset, Address::times_4,  0));
 8067   rorq(sum, 32);
 8068   if (UseBMI2Instructions) {
 8069     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8070   }
 8071   else {
 8072     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8073   }
 8074   // Store back in big endian from little endian
 8075   rorq(sum, 0x20);
 8076   movq(Address(out, offset, Address::times_4,  0), sum);
 8077 
 8078   jmp(L_first_loop);
 8079   bind(L_first_loop_exit);
 8080 }
 8081 
 8082 /**
 8083  * Code for BigInteger::mulAdd() intrinsic
 8084  *
 8085  * rdi: out
 8086  * rsi: in
 8087  * r11: offs (out.length - offset)
 8088  * rcx: len
 8089  * r8:  k
 8090  * r12: tmp1
 8091  * r13: tmp2
 8092  * r14: tmp3
 8093  * r15: tmp4
 8094  * rbx: tmp5
 8095  * Multiply the in[] by word k and add to out[], return the carry in rax
 8096  */
 8097 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 8098    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 8099    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8100 
 8101   Label L_carry, L_last_in, L_done;
 8102 
 8103 // carry = 0;
 8104 // for (int j=len-1; j >= 0; j--) {
 8105 //    long product = (in[j] & LONG_MASK) * kLong +
 8106 //                   (out[offs] & LONG_MASK) + carry;
 8107 //    out[offs--] = (int)product;
 8108 //    carry = product >>> 32;
 8109 // }
 8110 //
 8111   push(tmp1);
 8112   push(tmp2);
 8113   push(tmp3);
 8114   push(tmp4);
 8115   push(tmp5);
 8116 
 8117   Register op2 = tmp2;
 8118   const Register sum = tmp3;
 8119   const Register op1 = tmp4;
 8120   const Register carry =  tmp5;
 8121 
 8122   if (UseBMI2Instructions) {
 8123     op2 = rdxReg;
 8124     movl(op2, k);
 8125   }
 8126   else {
 8127     movl(op2, k);
 8128   }
 8129 
 8130   xorq(carry, carry);
 8131 
 8132   //First loop
 8133 
 8134   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 8135   //The carry is in tmp5
 8136   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8137 
 8138   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 8139   decrementl(len);
 8140   jccb(Assembler::negative, L_carry);
 8141   decrementl(len);
 8142   jccb(Assembler::negative, L_last_in);
 8143 
 8144   movq(op1, Address(in, len, Address::times_4,  0));
 8145   rorq(op1, 32);
 8146 
 8147   subl(offs, 2);
 8148   movq(sum, Address(out, offs, Address::times_4,  0));
 8149   rorq(sum, 32);
 8150 
 8151   if (UseBMI2Instructions) {
 8152     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8153   }
 8154   else {
 8155     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8156   }
 8157 
 8158   // Store back in big endian from little endian
 8159   rorq(sum, 0x20);
 8160   movq(Address(out, offs, Address::times_4,  0), sum);
 8161 
 8162   testl(len, len);
 8163   jccb(Assembler::zero, L_carry);
 8164 
 8165   //Multiply the last in[] entry, if any
 8166   bind(L_last_in);
 8167   movl(op1, Address(in, 0));
 8168   movl(sum, Address(out, offs, Address::times_4,  -4));
 8169 
 8170   movl(raxReg, k);
 8171   mull(op1); //tmp4 * eax -> edx:eax
 8172   addl(sum, carry);
 8173   adcl(rdxReg, 0);
 8174   addl(sum, raxReg);
 8175   adcl(rdxReg, 0);
 8176   movl(carry, rdxReg);
 8177 
 8178   movl(Address(out, offs, Address::times_4,  -4), sum);
 8179 
 8180   bind(L_carry);
 8181   //return tmp5/carry as carry in rax
 8182   movl(rax, carry);
 8183 
 8184   bind(L_done);
 8185   pop(tmp5);
 8186   pop(tmp4);
 8187   pop(tmp3);
 8188   pop(tmp2);
 8189   pop(tmp1);
 8190 }
 8191 #endif
 8192 
 8193 /**
 8194  * Emits code to update CRC-32 with a byte value according to constants in table
 8195  *
 8196  * @param [in,out]crc   Register containing the crc.
 8197  * @param [in]val       Register containing the byte to fold into the CRC.
 8198  * @param [in]table     Register containing the table of crc constants.
 8199  *
 8200  * uint32_t crc;
 8201  * val = crc_table[(val ^ crc) & 0xFF];
 8202  * crc = val ^ (crc >> 8);
 8203  *
 8204  */
 8205 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 8206   xorl(val, crc);
 8207   andl(val, 0xFF);
 8208   shrl(crc, 8); // unsigned shift
 8209   xorl(crc, Address(table, val, Address::times_4, 0));
 8210 }
 8211 
 8212 /**
 8213  * Fold 128-bit data chunk
 8214  */
 8215 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 8216   if (UseAVX > 0) {
 8217     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 8218     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 8219     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 8220     pxor(xcrc, xtmp);
 8221   } else {
 8222     movdqa(xtmp, xcrc);
 8223     pclmulhdq(xtmp, xK);   // [123:64]
 8224     pclmulldq(xcrc, xK);   // [63:0]
 8225     pxor(xcrc, xtmp);
 8226     movdqu(xtmp, Address(buf, offset));
 8227     pxor(xcrc, xtmp);
 8228   }
 8229 }
 8230 
 8231 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 8232   if (UseAVX > 0) {
 8233     vpclmulhdq(xtmp, xK, xcrc);
 8234     vpclmulldq(xcrc, xK, xcrc);
 8235     pxor(xcrc, xbuf);
 8236     pxor(xcrc, xtmp);
 8237   } else {
 8238     movdqa(xtmp, xcrc);
 8239     pclmulhdq(xtmp, xK);
 8240     pclmulldq(xcrc, xK);
 8241     pxor(xcrc, xbuf);
 8242     pxor(xcrc, xtmp);
 8243   }
 8244 }
 8245 
 8246 /**
 8247  * 8-bit folds to compute 32-bit CRC
 8248  *
 8249  * uint64_t xcrc;
 8250  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 8251  */
 8252 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 8253   movdl(tmp, xcrc);
 8254   andl(tmp, 0xFF);
 8255   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 8256   psrldq(xcrc, 1); // unsigned shift one byte
 8257   pxor(xcrc, xtmp);
 8258 }
 8259 
 8260 /**
 8261  * uint32_t crc;
 8262  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 8263  */
 8264 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 8265   movl(tmp, crc);
 8266   andl(tmp, 0xFF);
 8267   shrl(crc, 8);
 8268   xorl(crc, Address(table, tmp, Address::times_4, 0));
 8269 }
 8270 
 8271 /**
 8272  * @param crc   register containing existing CRC (32-bit)
 8273  * @param buf   register pointing to input byte buffer (byte*)
 8274  * @param len   register containing number of bytes
 8275  * @param table register that will contain address of CRC table
 8276  * @param tmp   scratch register
 8277  */
 8278 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 8279   assert_different_registers(crc, buf, len, table, tmp, rax);
 8280 
 8281   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8282   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8283 
 8284   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8285   // context for the registers used, where all instructions below are using 128-bit mode
 8286   // On EVEX without VL and BW, these instructions will all be AVX.
 8287   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 8288   notl(crc); // ~crc
 8289   cmpl(len, 16);
 8290   jcc(Assembler::less, L_tail);
 8291 
 8292   // Align buffer to 16 bytes
 8293   movl(tmp, buf);
 8294   andl(tmp, 0xF);
 8295   jccb(Assembler::zero, L_aligned);
 8296   subl(tmp,  16);
 8297   addl(len, tmp);
 8298 
 8299   align(4);
 8300   BIND(L_align_loop);
 8301   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8302   update_byte_crc32(crc, rax, table);
 8303   increment(buf);
 8304   incrementl(tmp);
 8305   jccb(Assembler::less, L_align_loop);
 8306 
 8307   BIND(L_aligned);
 8308   movl(tmp, len); // save
 8309   shrl(len, 4);
 8310   jcc(Assembler::zero, L_tail_restore);
 8311 
 8312   // Fold crc into first bytes of vector
 8313   movdqa(xmm1, Address(buf, 0));
 8314   movdl(rax, xmm1);
 8315   xorl(crc, rax);
 8316   if (VM_Version::supports_sse4_1()) {
 8317     pinsrd(xmm1, crc, 0);
 8318   } else {
 8319     pinsrw(xmm1, crc, 0);
 8320     shrl(crc, 16);
 8321     pinsrw(xmm1, crc, 1);
 8322   }
 8323   addptr(buf, 16);
 8324   subl(len, 4); // len > 0
 8325   jcc(Assembler::less, L_fold_tail);
 8326 
 8327   movdqa(xmm2, Address(buf,  0));
 8328   movdqa(xmm3, Address(buf, 16));
 8329   movdqa(xmm4, Address(buf, 32));
 8330   addptr(buf, 48);
 8331   subl(len, 3);
 8332   jcc(Assembler::lessEqual, L_fold_512b);
 8333 
 8334   // Fold total 512 bits of polynomial on each iteration,
 8335   // 128 bits per each of 4 parallel streams.
 8336   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8337 
 8338   align32();
 8339   BIND(L_fold_512b_loop);
 8340   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8341   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8342   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8343   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8344   addptr(buf, 64);
 8345   subl(len, 4);
 8346   jcc(Assembler::greater, L_fold_512b_loop);
 8347 
 8348   // Fold 512 bits to 128 bits.
 8349   BIND(L_fold_512b);
 8350   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8351   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8352   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8353   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8354 
 8355   // Fold the rest of 128 bits data chunks
 8356   BIND(L_fold_tail);
 8357   addl(len, 3);
 8358   jccb(Assembler::lessEqual, L_fold_128b);
 8359   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8360 
 8361   BIND(L_fold_tail_loop);
 8362   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8363   addptr(buf, 16);
 8364   decrementl(len);
 8365   jccb(Assembler::greater, L_fold_tail_loop);
 8366 
 8367   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8368   BIND(L_fold_128b);
 8369   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8370   if (UseAVX > 0) {
 8371     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8372     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8373     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8374   } else {
 8375     movdqa(xmm2, xmm0);
 8376     pclmulqdq(xmm2, xmm1, 0x1);
 8377     movdqa(xmm3, xmm0);
 8378     pand(xmm3, xmm2);
 8379     pclmulqdq(xmm0, xmm3, 0x1);
 8380   }
 8381   psrldq(xmm1, 8);
 8382   psrldq(xmm2, 4);
 8383   pxor(xmm0, xmm1);
 8384   pxor(xmm0, xmm2);
 8385 
 8386   // 8 8-bit folds to compute 32-bit CRC.
 8387   for (int j = 0; j < 4; j++) {
 8388     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8389   }
 8390   movdl(crc, xmm0); // mov 32 bits to general register
 8391   for (int j = 0; j < 4; j++) {
 8392     fold_8bit_crc32(crc, table, rax);
 8393   }
 8394 
 8395   BIND(L_tail_restore);
 8396   movl(len, tmp); // restore
 8397   BIND(L_tail);
 8398   andl(len, 0xf);
 8399   jccb(Assembler::zero, L_exit);
 8400 
 8401   // Fold the rest of bytes
 8402   align(4);
 8403   BIND(L_tail_loop);
 8404   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8405   update_byte_crc32(crc, rax, table);
 8406   increment(buf);
 8407   decrementl(len);
 8408   jccb(Assembler::greater, L_tail_loop);
 8409 
 8410   BIND(L_exit);
 8411   notl(crc); // ~c
 8412 }
 8413 
 8414 #ifdef _LP64
 8415 // Helper function for AVX 512 CRC32
 8416 // Fold 512-bit data chunks
 8417 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8418                                              Register pos, int offset) {
 8419   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8420   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8421   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8422   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8423   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8424 }
 8425 
 8426 // Helper function for AVX 512 CRC32
 8427 // Compute CRC32 for < 256B buffers
 8428 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8429                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8430                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8431 
 8432   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8433   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8434   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8435 
 8436   // check if there is enough buffer to be able to fold 16B at a time
 8437   cmpl(len, 32);
 8438   jcc(Assembler::less, L_less_than_32);
 8439 
 8440   // if there is, load the constants
 8441   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8442   movdl(xmm0, crc);                        // get the initial crc value
 8443   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8444   pxor(xmm7, xmm0);
 8445 
 8446   // update the buffer pointer
 8447   addl(pos, 16);
 8448   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8449   subl(len, 32);
 8450   jmp(L_16B_reduction_loop);
 8451 
 8452   bind(L_less_than_32);
 8453   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8454   movl(rax, crc);
 8455   testl(len, len);
 8456   jcc(Assembler::equal, L_cleanup);
 8457 
 8458   movdl(xmm0, crc);                        //get the initial crc value
 8459 
 8460   cmpl(len, 16);
 8461   jcc(Assembler::equal, L_exact_16_left);
 8462   jcc(Assembler::less, L_less_than_16_left);
 8463 
 8464   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8465   pxor(xmm7, xmm0);                       //xor the initial crc value
 8466   addl(pos, 16);
 8467   subl(len, 16);
 8468   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8469   jmp(L_get_last_two_xmms);
 8470 
 8471   bind(L_less_than_16_left);
 8472   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8473   pxor(xmm1, xmm1);
 8474   movptr(tmp1, rsp);
 8475   movdqu(Address(tmp1, 0 * 16), xmm1);
 8476 
 8477   cmpl(len, 4);
 8478   jcc(Assembler::less, L_only_less_than_4);
 8479 
 8480   //backup the counter value
 8481   movl(tmp2, len);
 8482   cmpl(len, 8);
 8483   jcc(Assembler::less, L_less_than_8_left);
 8484 
 8485   //load 8 Bytes
 8486   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8487   movq(Address(tmp1, 0 * 16), rax);
 8488   addptr(tmp1, 8);
 8489   subl(len, 8);
 8490   addl(pos, 8);
 8491 
 8492   bind(L_less_than_8_left);
 8493   cmpl(len, 4);
 8494   jcc(Assembler::less, L_less_than_4_left);
 8495 
 8496   //load 4 Bytes
 8497   movl(rax, Address(buf, pos, Address::times_1, 0));
 8498   movl(Address(tmp1, 0 * 16), rax);
 8499   addptr(tmp1, 4);
 8500   subl(len, 4);
 8501   addl(pos, 4);
 8502 
 8503   bind(L_less_than_4_left);
 8504   cmpl(len, 2);
 8505   jcc(Assembler::less, L_less_than_2_left);
 8506 
 8507   // load 2 Bytes
 8508   movw(rax, Address(buf, pos, Address::times_1, 0));
 8509   movl(Address(tmp1, 0 * 16), rax);
 8510   addptr(tmp1, 2);
 8511   subl(len, 2);
 8512   addl(pos, 2);
 8513 
 8514   bind(L_less_than_2_left);
 8515   cmpl(len, 1);
 8516   jcc(Assembler::less, L_zero_left);
 8517 
 8518   // load 1 Byte
 8519   movb(rax, Address(buf, pos, Address::times_1, 0));
 8520   movb(Address(tmp1, 0 * 16), rax);
 8521 
 8522   bind(L_zero_left);
 8523   movdqu(xmm7, Address(rsp, 0));
 8524   pxor(xmm7, xmm0);                       //xor the initial crc value
 8525 
 8526   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8527   movdqu(xmm0, Address(rax, tmp2));
 8528   pshufb(xmm7, xmm0);
 8529   jmp(L_128_done);
 8530 
 8531   bind(L_exact_16_left);
 8532   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8533   pxor(xmm7, xmm0);                       //xor the initial crc value
 8534   jmp(L_128_done);
 8535 
 8536   bind(L_only_less_than_4);
 8537   cmpl(len, 3);
 8538   jcc(Assembler::less, L_only_less_than_3);
 8539 
 8540   // load 3 Bytes
 8541   movb(rax, Address(buf, pos, Address::times_1, 0));
 8542   movb(Address(tmp1, 0), rax);
 8543 
 8544   movb(rax, Address(buf, pos, Address::times_1, 1));
 8545   movb(Address(tmp1, 1), rax);
 8546 
 8547   movb(rax, Address(buf, pos, Address::times_1, 2));
 8548   movb(Address(tmp1, 2), rax);
 8549 
 8550   movdqu(xmm7, Address(rsp, 0));
 8551   pxor(xmm7, xmm0);                     //xor the initial crc value
 8552 
 8553   pslldq(xmm7, 0x5);
 8554   jmp(L_barrett);
 8555   bind(L_only_less_than_3);
 8556   cmpl(len, 2);
 8557   jcc(Assembler::less, L_only_less_than_2);
 8558 
 8559   // load 2 Bytes
 8560   movb(rax, Address(buf, pos, Address::times_1, 0));
 8561   movb(Address(tmp1, 0), rax);
 8562 
 8563   movb(rax, Address(buf, pos, Address::times_1, 1));
 8564   movb(Address(tmp1, 1), rax);
 8565 
 8566   movdqu(xmm7, Address(rsp, 0));
 8567   pxor(xmm7, xmm0);                     //xor the initial crc value
 8568 
 8569   pslldq(xmm7, 0x6);
 8570   jmp(L_barrett);
 8571 
 8572   bind(L_only_less_than_2);
 8573   //load 1 Byte
 8574   movb(rax, Address(buf, pos, Address::times_1, 0));
 8575   movb(Address(tmp1, 0), rax);
 8576 
 8577   movdqu(xmm7, Address(rsp, 0));
 8578   pxor(xmm7, xmm0);                     //xor the initial crc value
 8579 
 8580   pslldq(xmm7, 0x7);
 8581 }
 8582 
 8583 /**
 8584 * Compute CRC32 using AVX512 instructions
 8585 * param crc   register containing existing CRC (32-bit)
 8586 * param buf   register pointing to input byte buffer (byte*)
 8587 * param len   register containing number of bytes
 8588 * param table address of crc or crc32c table
 8589 * param tmp1  scratch register
 8590 * param tmp2  scratch register
 8591 * return rax  result register
 8592 *
 8593 * This routine is identical for crc32c with the exception of the precomputed constant
 8594 * table which will be passed as the table argument.  The calculation steps are
 8595 * the same for both variants.
 8596 */
 8597 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8598   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8599 
 8600   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8601   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8602   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8603   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8604   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8605 
 8606   const Register pos = r12;
 8607   push(r12);
 8608   subptr(rsp, 16 * 2 + 8);
 8609 
 8610   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8611   // context for the registers used, where all instructions below are using 128-bit mode
 8612   // On EVEX without VL and BW, these instructions will all be AVX.
 8613   movl(pos, 0);
 8614 
 8615   // check if smaller than 256B
 8616   cmpl(len, 256);
 8617   jcc(Assembler::less, L_less_than_256);
 8618 
 8619   // load the initial crc value
 8620   movdl(xmm10, crc);
 8621 
 8622   // receive the initial 64B data, xor the initial crc value
 8623   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8624   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8625   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8626   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8627 
 8628   subl(len, 256);
 8629   cmpl(len, 256);
 8630   jcc(Assembler::less, L_fold_128_B_loop);
 8631 
 8632   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8633   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8634   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8635   subl(len, 256);
 8636 
 8637   bind(L_fold_256_B_loop);
 8638   addl(pos, 256);
 8639   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8640   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8641   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8642   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8643 
 8644   subl(len, 256);
 8645   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8646 
 8647   // Fold 256 into 128
 8648   addl(pos, 256);
 8649   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8650   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8651   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8652 
 8653   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8654   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8655   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8656 
 8657   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8658   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8659 
 8660   addl(len, 128);
 8661   jmp(L_fold_128_B_register);
 8662 
 8663   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8664   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8665 
 8666   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8667   bind(L_fold_128_B_loop);
 8668   addl(pos, 128);
 8669   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8670   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8671 
 8672   subl(len, 128);
 8673   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8674 
 8675   addl(pos, 128);
 8676 
 8677   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8678   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8679   bind(L_fold_128_B_register);
 8680   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8681   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8682   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8683   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8684   // save last that has no multiplicand
 8685   vextracti64x2(xmm7, xmm4, 3);
 8686 
 8687   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8688   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8689   // Needed later in reduction loop
 8690   movdqu(xmm10, Address(table, 1 * 16));
 8691   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8692   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8693 
 8694   // Swap 1,0,3,2 - 01 00 11 10
 8695   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8696   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8697   vextracti128(xmm5, xmm8, 1);
 8698   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8699 
 8700   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8701   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8702   addl(len, 128 - 16);
 8703   jcc(Assembler::less, L_final_reduction_for_128);
 8704 
 8705   bind(L_16B_reduction_loop);
 8706   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8707   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8708   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8709   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8710   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8711   addl(pos, 16);
 8712   subl(len, 16);
 8713   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8714 
 8715   bind(L_final_reduction_for_128);
 8716   addl(len, 16);
 8717   jcc(Assembler::equal, L_128_done);
 8718 
 8719   bind(L_get_last_two_xmms);
 8720   movdqu(xmm2, xmm7);
 8721   addl(pos, len);
 8722   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8723   subl(pos, len);
 8724 
 8725   // get rid of the extra data that was loaded before
 8726   // load the shift constant
 8727   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8728   movdqu(xmm0, Address(rax, len));
 8729   addl(rax, len);
 8730 
 8731   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8732   //Change mask to 512
 8733   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8734   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8735 
 8736   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8737   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8738   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8739   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8740   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8741 
 8742   bind(L_128_done);
 8743   // compute crc of a 128-bit value
 8744   movdqu(xmm10, Address(table, 3 * 16));
 8745   movdqu(xmm0, xmm7);
 8746 
 8747   // 64b fold
 8748   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8749   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8750   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8751 
 8752   // 32b fold
 8753   movdqu(xmm0, xmm7);
 8754   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8755   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8756   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8757   jmp(L_barrett);
 8758 
 8759   bind(L_less_than_256);
 8760   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8761 
 8762   //barrett reduction
 8763   bind(L_barrett);
 8764   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8765   movdqu(xmm1, xmm7);
 8766   movdqu(xmm2, xmm7);
 8767   movdqu(xmm10, Address(table, 4 * 16));
 8768 
 8769   pclmulqdq(xmm7, xmm10, 0x0);
 8770   pxor(xmm7, xmm2);
 8771   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8772   movdqu(xmm2, xmm7);
 8773   pclmulqdq(xmm7, xmm10, 0x10);
 8774   pxor(xmm7, xmm2);
 8775   pxor(xmm7, xmm1);
 8776   pextrd(crc, xmm7, 2);
 8777 
 8778   bind(L_cleanup);
 8779   addptr(rsp, 16 * 2 + 8);
 8780   pop(r12);
 8781 }
 8782 
 8783 // S. Gueron / Information Processing Letters 112 (2012) 184
 8784 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8785 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8786 // Output: the 64-bit carry-less product of B * CONST
 8787 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8788                                      Register tmp1, Register tmp2, Register tmp3) {
 8789   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8790   if (n > 0) {
 8791     addq(tmp3, n * 256 * 8);
 8792   }
 8793   //    Q1 = TABLEExt[n][B & 0xFF];
 8794   movl(tmp1, in);
 8795   andl(tmp1, 0x000000FF);
 8796   shll(tmp1, 3);
 8797   addq(tmp1, tmp3);
 8798   movq(tmp1, Address(tmp1, 0));
 8799 
 8800   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8801   movl(tmp2, in);
 8802   shrl(tmp2, 8);
 8803   andl(tmp2, 0x000000FF);
 8804   shll(tmp2, 3);
 8805   addq(tmp2, tmp3);
 8806   movq(tmp2, Address(tmp2, 0));
 8807 
 8808   shlq(tmp2, 8);
 8809   xorq(tmp1, tmp2);
 8810 
 8811   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8812   movl(tmp2, in);
 8813   shrl(tmp2, 16);
 8814   andl(tmp2, 0x000000FF);
 8815   shll(tmp2, 3);
 8816   addq(tmp2, tmp3);
 8817   movq(tmp2, Address(tmp2, 0));
 8818 
 8819   shlq(tmp2, 16);
 8820   xorq(tmp1, tmp2);
 8821 
 8822   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8823   shrl(in, 24);
 8824   andl(in, 0x000000FF);
 8825   shll(in, 3);
 8826   addq(in, tmp3);
 8827   movq(in, Address(in, 0));
 8828 
 8829   shlq(in, 24);
 8830   xorq(in, tmp1);
 8831   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8832 }
 8833 
 8834 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8835                                       Register in_out,
 8836                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8837                                       XMMRegister w_xtmp2,
 8838                                       Register tmp1,
 8839                                       Register n_tmp2, Register n_tmp3) {
 8840   if (is_pclmulqdq_supported) {
 8841     movdl(w_xtmp1, in_out); // modified blindly
 8842 
 8843     movl(tmp1, const_or_pre_comp_const_index);
 8844     movdl(w_xtmp2, tmp1);
 8845     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8846 
 8847     movdq(in_out, w_xtmp1);
 8848   } else {
 8849     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8850   }
 8851 }
 8852 
 8853 // Recombination Alternative 2: No bit-reflections
 8854 // T1 = (CRC_A * U1) << 1
 8855 // T2 = (CRC_B * U2) << 1
 8856 // C1 = T1 >> 32
 8857 // C2 = T2 >> 32
 8858 // T1 = T1 & 0xFFFFFFFF
 8859 // T2 = T2 & 0xFFFFFFFF
 8860 // T1 = CRC32(0, T1)
 8861 // T2 = CRC32(0, T2)
 8862 // C1 = C1 ^ T1
 8863 // C2 = C2 ^ T2
 8864 // CRC = C1 ^ C2 ^ CRC_C
 8865 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8866                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8867                                      Register tmp1, Register tmp2,
 8868                                      Register n_tmp3) {
 8869   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8870   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8871   shlq(in_out, 1);
 8872   movl(tmp1, in_out);
 8873   shrq(in_out, 32);
 8874   xorl(tmp2, tmp2);
 8875   crc32(tmp2, tmp1, 4);
 8876   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8877   shlq(in1, 1);
 8878   movl(tmp1, in1);
 8879   shrq(in1, 32);
 8880   xorl(tmp2, tmp2);
 8881   crc32(tmp2, tmp1, 4);
 8882   xorl(in1, tmp2);
 8883   xorl(in_out, in1);
 8884   xorl(in_out, in2);
 8885 }
 8886 
 8887 // Set N to predefined value
 8888 // Subtract from a length of a buffer
 8889 // execute in a loop:
 8890 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8891 // for i = 1 to N do
 8892 //  CRC_A = CRC32(CRC_A, A[i])
 8893 //  CRC_B = CRC32(CRC_B, B[i])
 8894 //  CRC_C = CRC32(CRC_C, C[i])
 8895 // end for
 8896 // Recombine
 8897 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8898                                        Register in_out1, Register in_out2, Register in_out3,
 8899                                        Register tmp1, Register tmp2, Register tmp3,
 8900                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8901                                        Register tmp4, Register tmp5,
 8902                                        Register n_tmp6) {
 8903   Label L_processPartitions;
 8904   Label L_processPartition;
 8905   Label L_exit;
 8906 
 8907   bind(L_processPartitions);
 8908   cmpl(in_out1, 3 * size);
 8909   jcc(Assembler::less, L_exit);
 8910     xorl(tmp1, tmp1);
 8911     xorl(tmp2, tmp2);
 8912     movq(tmp3, in_out2);
 8913     addq(tmp3, size);
 8914 
 8915     bind(L_processPartition);
 8916       crc32(in_out3, Address(in_out2, 0), 8);
 8917       crc32(tmp1, Address(in_out2, size), 8);
 8918       crc32(tmp2, Address(in_out2, size * 2), 8);
 8919       addq(in_out2, 8);
 8920       cmpq(in_out2, tmp3);
 8921       jcc(Assembler::less, L_processPartition);
 8922     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8923             w_xtmp1, w_xtmp2, w_xtmp3,
 8924             tmp4, tmp5,
 8925             n_tmp6);
 8926     addq(in_out2, 2 * size);
 8927     subl(in_out1, 3 * size);
 8928     jmp(L_processPartitions);
 8929 
 8930   bind(L_exit);
 8931 }
 8932 #else
 8933 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 8934                                      Register tmp1, Register tmp2, Register tmp3,
 8935                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 8936   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8937   if (n > 0) {
 8938     addl(tmp3, n * 256 * 8);
 8939   }
 8940   //    Q1 = TABLEExt[n][B & 0xFF];
 8941   movl(tmp1, in_out);
 8942   andl(tmp1, 0x000000FF);
 8943   shll(tmp1, 3);
 8944   addl(tmp1, tmp3);
 8945   movq(xtmp1, Address(tmp1, 0));
 8946 
 8947   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8948   movl(tmp2, in_out);
 8949   shrl(tmp2, 8);
 8950   andl(tmp2, 0x000000FF);
 8951   shll(tmp2, 3);
 8952   addl(tmp2, tmp3);
 8953   movq(xtmp2, Address(tmp2, 0));
 8954 
 8955   psllq(xtmp2, 8);
 8956   pxor(xtmp1, xtmp2);
 8957 
 8958   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8959   movl(tmp2, in_out);
 8960   shrl(tmp2, 16);
 8961   andl(tmp2, 0x000000FF);
 8962   shll(tmp2, 3);
 8963   addl(tmp2, tmp3);
 8964   movq(xtmp2, Address(tmp2, 0));
 8965 
 8966   psllq(xtmp2, 16);
 8967   pxor(xtmp1, xtmp2);
 8968 
 8969   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8970   shrl(in_out, 24);
 8971   andl(in_out, 0x000000FF);
 8972   shll(in_out, 3);
 8973   addl(in_out, tmp3);
 8974   movq(xtmp2, Address(in_out, 0));
 8975 
 8976   psllq(xtmp2, 24);
 8977   pxor(xtmp1, xtmp2); // Result in CXMM
 8978   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8979 }
 8980 
 8981 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8982                                       Register in_out,
 8983                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8984                                       XMMRegister w_xtmp2,
 8985                                       Register tmp1,
 8986                                       Register n_tmp2, Register n_tmp3) {
 8987   if (is_pclmulqdq_supported) {
 8988     movdl(w_xtmp1, in_out);
 8989 
 8990     movl(tmp1, const_or_pre_comp_const_index);
 8991     movdl(w_xtmp2, tmp1);
 8992     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8993     // Keep result in XMM since GPR is 32 bit in length
 8994   } else {
 8995     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 8996   }
 8997 }
 8998 
 8999 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9000                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9001                                      Register tmp1, Register tmp2,
 9002                                      Register n_tmp3) {
 9003   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9004   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9005 
 9006   psllq(w_xtmp1, 1);
 9007   movdl(tmp1, w_xtmp1);
 9008   psrlq(w_xtmp1, 32);
 9009   movdl(in_out, w_xtmp1);
 9010 
 9011   xorl(tmp2, tmp2);
 9012   crc32(tmp2, tmp1, 4);
 9013   xorl(in_out, tmp2);
 9014 
 9015   psllq(w_xtmp2, 1);
 9016   movdl(tmp1, w_xtmp2);
 9017   psrlq(w_xtmp2, 32);
 9018   movdl(in1, w_xtmp2);
 9019 
 9020   xorl(tmp2, tmp2);
 9021   crc32(tmp2, tmp1, 4);
 9022   xorl(in1, tmp2);
 9023   xorl(in_out, in1);
 9024   xorl(in_out, in2);
 9025 }
 9026 
 9027 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9028                                        Register in_out1, Register in_out2, Register in_out3,
 9029                                        Register tmp1, Register tmp2, Register tmp3,
 9030                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9031                                        Register tmp4, Register tmp5,
 9032                                        Register n_tmp6) {
 9033   Label L_processPartitions;
 9034   Label L_processPartition;
 9035   Label L_exit;
 9036 
 9037   bind(L_processPartitions);
 9038   cmpl(in_out1, 3 * size);
 9039   jcc(Assembler::less, L_exit);
 9040     xorl(tmp1, tmp1);
 9041     xorl(tmp2, tmp2);
 9042     movl(tmp3, in_out2);
 9043     addl(tmp3, size);
 9044 
 9045     bind(L_processPartition);
 9046       crc32(in_out3, Address(in_out2, 0), 4);
 9047       crc32(tmp1, Address(in_out2, size), 4);
 9048       crc32(tmp2, Address(in_out2, size*2), 4);
 9049       crc32(in_out3, Address(in_out2, 0+4), 4);
 9050       crc32(tmp1, Address(in_out2, size+4), 4);
 9051       crc32(tmp2, Address(in_out2, size*2+4), 4);
 9052       addl(in_out2, 8);
 9053       cmpl(in_out2, tmp3);
 9054       jcc(Assembler::less, L_processPartition);
 9055 
 9056         push(tmp3);
 9057         push(in_out1);
 9058         push(in_out2);
 9059         tmp4 = tmp3;
 9060         tmp5 = in_out1;
 9061         n_tmp6 = in_out2;
 9062 
 9063       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9064             w_xtmp1, w_xtmp2, w_xtmp3,
 9065             tmp4, tmp5,
 9066             n_tmp6);
 9067 
 9068         pop(in_out2);
 9069         pop(in_out1);
 9070         pop(tmp3);
 9071 
 9072     addl(in_out2, 2 * size);
 9073     subl(in_out1, 3 * size);
 9074     jmp(L_processPartitions);
 9075 
 9076   bind(L_exit);
 9077 }
 9078 #endif //LP64
 9079 
 9080 #ifdef _LP64
 9081 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 9082 // Input: A buffer I of L bytes.
 9083 // Output: the CRC32C value of the buffer.
 9084 // Notations:
 9085 // Write L = 24N + r, with N = floor (L/24).
 9086 // r = L mod 24 (0 <= r < 24).
 9087 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 9088 // N quadwords, and R consists of r bytes.
 9089 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 9090 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 9091 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 9092 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 9093 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9094                                           Register tmp1, Register tmp2, Register tmp3,
 9095                                           Register tmp4, Register tmp5, Register tmp6,
 9096                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9097                                           bool is_pclmulqdq_supported) {
 9098   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9099   Label L_wordByWord;
 9100   Label L_byteByByteProlog;
 9101   Label L_byteByByte;
 9102   Label L_exit;
 9103 
 9104   if (is_pclmulqdq_supported ) {
 9105     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 9106     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
 9107 
 9108     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 9109     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 9110 
 9111     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 9112     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 9113     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 9114   } else {
 9115     const_or_pre_comp_const_index[0] = 1;
 9116     const_or_pre_comp_const_index[1] = 0;
 9117 
 9118     const_or_pre_comp_const_index[2] = 3;
 9119     const_or_pre_comp_const_index[3] = 2;
 9120 
 9121     const_or_pre_comp_const_index[4] = 5;
 9122     const_or_pre_comp_const_index[5] = 4;
 9123    }
 9124   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9125                     in2, in1, in_out,
 9126                     tmp1, tmp2, tmp3,
 9127                     w_xtmp1, w_xtmp2, w_xtmp3,
 9128                     tmp4, tmp5,
 9129                     tmp6);
 9130   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9131                     in2, in1, in_out,
 9132                     tmp1, tmp2, tmp3,
 9133                     w_xtmp1, w_xtmp2, w_xtmp3,
 9134                     tmp4, tmp5,
 9135                     tmp6);
 9136   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9137                     in2, in1, in_out,
 9138                     tmp1, tmp2, tmp3,
 9139                     w_xtmp1, w_xtmp2, w_xtmp3,
 9140                     tmp4, tmp5,
 9141                     tmp6);
 9142   movl(tmp1, in2);
 9143   andl(tmp1, 0x00000007);
 9144   negl(tmp1);
 9145   addl(tmp1, in2);
 9146   addq(tmp1, in1);
 9147 
 9148   cmpq(in1, tmp1);
 9149   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 9150   align(16);
 9151   BIND(L_wordByWord);
 9152     crc32(in_out, Address(in1, 0), 8);
 9153     addq(in1, 8);
 9154     cmpq(in1, tmp1);
 9155     jcc(Assembler::less, L_wordByWord);
 9156 
 9157   BIND(L_byteByByteProlog);
 9158   andl(in2, 0x00000007);
 9159   movl(tmp2, 1);
 9160 
 9161   cmpl(tmp2, in2);
 9162   jccb(Assembler::greater, L_exit);
 9163   BIND(L_byteByByte);
 9164     crc32(in_out, Address(in1, 0), 1);
 9165     incq(in1);
 9166     incl(tmp2);
 9167     cmpl(tmp2, in2);
 9168     jcc(Assembler::lessEqual, L_byteByByte);
 9169 
 9170   BIND(L_exit);
 9171 }
 9172 #else
 9173 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9174                                           Register tmp1, Register  tmp2, Register tmp3,
 9175                                           Register tmp4, Register  tmp5, Register tmp6,
 9176                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9177                                           bool is_pclmulqdq_supported) {
 9178   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9179   Label L_wordByWord;
 9180   Label L_byteByByteProlog;
 9181   Label L_byteByByte;
 9182   Label L_exit;
 9183 
 9184   if (is_pclmulqdq_supported) {
 9185     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 9186     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
 9187 
 9188     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 9189     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 9190 
 9191     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 9192     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 9193   } else {
 9194     const_or_pre_comp_const_index[0] = 1;
 9195     const_or_pre_comp_const_index[1] = 0;
 9196 
 9197     const_or_pre_comp_const_index[2] = 3;
 9198     const_or_pre_comp_const_index[3] = 2;
 9199 
 9200     const_or_pre_comp_const_index[4] = 5;
 9201     const_or_pre_comp_const_index[5] = 4;
 9202   }
 9203   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9204                     in2, in1, in_out,
 9205                     tmp1, tmp2, tmp3,
 9206                     w_xtmp1, w_xtmp2, w_xtmp3,
 9207                     tmp4, tmp5,
 9208                     tmp6);
 9209   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9210                     in2, in1, in_out,
 9211                     tmp1, tmp2, tmp3,
 9212                     w_xtmp1, w_xtmp2, w_xtmp3,
 9213                     tmp4, tmp5,
 9214                     tmp6);
 9215   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9216                     in2, in1, in_out,
 9217                     tmp1, tmp2, tmp3,
 9218                     w_xtmp1, w_xtmp2, w_xtmp3,
 9219                     tmp4, tmp5,
 9220                     tmp6);
 9221   movl(tmp1, in2);
 9222   andl(tmp1, 0x00000007);
 9223   negl(tmp1);
 9224   addl(tmp1, in2);
 9225   addl(tmp1, in1);
 9226 
 9227   BIND(L_wordByWord);
 9228   cmpl(in1, tmp1);
 9229   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 9230     crc32(in_out, Address(in1,0), 4);
 9231     addl(in1, 4);
 9232     jmp(L_wordByWord);
 9233 
 9234   BIND(L_byteByByteProlog);
 9235   andl(in2, 0x00000007);
 9236   movl(tmp2, 1);
 9237 
 9238   BIND(L_byteByByte);
 9239   cmpl(tmp2, in2);
 9240   jccb(Assembler::greater, L_exit);
 9241     movb(tmp1, Address(in1, 0));
 9242     crc32(in_out, tmp1, 1);
 9243     incl(in1);
 9244     incl(tmp2);
 9245     jmp(L_byteByByte);
 9246 
 9247   BIND(L_exit);
 9248 }
 9249 #endif // LP64
 9250 #undef BIND
 9251 #undef BLOCK_COMMENT
 9252 
 9253 // Compress char[] array to byte[].
 9254 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
 9255 //   @IntrinsicCandidate
 9256 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 9257 //     for (int i = 0; i < len; i++) {
 9258 //       int c = src[srcOff++];
 9259 //       if (c >>> 8 != 0) {
 9260 //         return 0;
 9261 //       }
 9262 //       dst[dstOff++] = (byte)c;
 9263 //     }
 9264 //     return len;
 9265 //   }
 9266 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 9267   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 9268   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 9269   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 9270   Label copy_chars_loop, return_length, return_zero, done;
 9271 
 9272   // rsi: src
 9273   // rdi: dst
 9274   // rdx: len
 9275   // rcx: tmp5
 9276   // rax: result
 9277 
 9278   // rsi holds start addr of source char[] to be compressed
 9279   // rdi holds start addr of destination byte[]
 9280   // rdx holds length
 9281 
 9282   assert(len != result, "");
 9283 
 9284   // save length for return
 9285   push(len);
 9286 
 9287   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 9288     VM_Version::supports_avx512vlbw() &&
 9289     VM_Version::supports_bmi2()) {
 9290 
 9291     Label copy_32_loop, copy_loop_tail, below_threshold;
 9292 
 9293     // alignment
 9294     Label post_alignment;
 9295 
 9296     // if length of the string is less than 16, handle it in an old fashioned way
 9297     testl(len, -32);
 9298     jcc(Assembler::zero, below_threshold);
 9299 
 9300     // First check whether a character is compressible ( <= 0xFF).
 9301     // Create mask to test for Unicode chars inside zmm vector
 9302     movl(result, 0x00FF);
 9303     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
 9304 
 9305     testl(len, -64);
 9306     jcc(Assembler::zero, post_alignment);
 9307 
 9308     movl(tmp5, dst);
 9309     andl(tmp5, (32 - 1));
 9310     negl(tmp5);
 9311     andl(tmp5, (32 - 1));
 9312 
 9313     // bail out when there is nothing to be done
 9314     testl(tmp5, 0xFFFFFFFF);
 9315     jcc(Assembler::zero, post_alignment);
 9316 
 9317     // ~(~0 << len), where len is the # of remaining elements to process
 9318     movl(result, 0xFFFFFFFF);
 9319     shlxl(result, result, tmp5);
 9320     notl(result);
 9321     kmovdl(mask2, result);
 9322 
 9323     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9324     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9325     ktestd(mask1, mask2);
 9326     jcc(Assembler::carryClear, return_zero);
 9327 
 9328     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9329 
 9330     addptr(src, tmp5);
 9331     addptr(src, tmp5);
 9332     addptr(dst, tmp5);
 9333     subl(len, tmp5);
 9334 
 9335     bind(post_alignment);
 9336     // end of alignment
 9337 
 9338     movl(tmp5, len);
 9339     andl(tmp5, (32 - 1));    // tail count (in chars)
 9340     andl(len, ~(32 - 1));    // vector count (in chars)
 9341     jcc(Assembler::zero, copy_loop_tail);
 9342 
 9343     lea(src, Address(src, len, Address::times_2));
 9344     lea(dst, Address(dst, len, Address::times_1));
 9345     negptr(len);
 9346 
 9347     bind(copy_32_loop);
 9348     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9349     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9350     kortestdl(mask1, mask1);
 9351     jcc(Assembler::carryClear, return_zero);
 9352 
 9353     // All elements in current processed chunk are valid candidates for
 9354     // compression. Write a truncated byte elements to the memory.
 9355     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9356     addptr(len, 32);
 9357     jcc(Assembler::notZero, copy_32_loop);
 9358 
 9359     bind(copy_loop_tail);
 9360     // bail out when there is nothing to be done
 9361     testl(tmp5, 0xFFFFFFFF);
 9362     jcc(Assembler::zero, return_length);
 9363 
 9364     movl(len, tmp5);
 9365 
 9366     // ~(~0 << len), where len is the # of remaining elements to process
 9367     movl(result, 0xFFFFFFFF);
 9368     shlxl(result, result, len);
 9369     notl(result);
 9370 
 9371     kmovdl(mask2, result);
 9372 
 9373     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9374     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9375     ktestd(mask1, mask2);
 9376     jcc(Assembler::carryClear, return_zero);
 9377 
 9378     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9379     jmp(return_length);
 9380 
 9381     bind(below_threshold);
 9382   }
 9383 
 9384   if (UseSSE42Intrinsics) {
 9385     Label copy_32_loop, copy_16, copy_tail;
 9386 
 9387     movl(result, len);
 9388 
 9389     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9390 
 9391     // vectored compression
 9392     andl(len, 0xfffffff0);    // vector count (in chars)
 9393     andl(result, 0x0000000f);    // tail count (in chars)
 9394     testl(len, len);
 9395     jcc(Assembler::zero, copy_16);
 9396 
 9397     // compress 16 chars per iter
 9398     movdl(tmp1Reg, tmp5);
 9399     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9400     pxor(tmp4Reg, tmp4Reg);
 9401 
 9402     lea(src, Address(src, len, Address::times_2));
 9403     lea(dst, Address(dst, len, Address::times_1));
 9404     negptr(len);
 9405 
 9406     bind(copy_32_loop);
 9407     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9408     por(tmp4Reg, tmp2Reg);
 9409     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9410     por(tmp4Reg, tmp3Reg);
 9411     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9412     jcc(Assembler::notZero, return_zero);
 9413     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9414     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9415     addptr(len, 16);
 9416     jcc(Assembler::notZero, copy_32_loop);
 9417 
 9418     // compress next vector of 8 chars (if any)
 9419     bind(copy_16);
 9420     movl(len, result);
 9421     andl(len, 0xfffffff8);    // vector count (in chars)
 9422     andl(result, 0x00000007);    // tail count (in chars)
 9423     testl(len, len);
 9424     jccb(Assembler::zero, copy_tail);
 9425 
 9426     movdl(tmp1Reg, tmp5);
 9427     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9428     pxor(tmp3Reg, tmp3Reg);
 9429 
 9430     movdqu(tmp2Reg, Address(src, 0));
 9431     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9432     jccb(Assembler::notZero, return_zero);
 9433     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9434     movq(Address(dst, 0), tmp2Reg);
 9435     addptr(src, 16);
 9436     addptr(dst, 8);
 9437 
 9438     bind(copy_tail);
 9439     movl(len, result);
 9440   }
 9441   // compress 1 char per iter
 9442   testl(len, len);
 9443   jccb(Assembler::zero, return_length);
 9444   lea(src, Address(src, len, Address::times_2));
 9445   lea(dst, Address(dst, len, Address::times_1));
 9446   negptr(len);
 9447 
 9448   bind(copy_chars_loop);
 9449   load_unsigned_short(result, Address(src, len, Address::times_2));
 9450   testl(result, 0xff00);      // check if Unicode char
 9451   jccb(Assembler::notZero, return_zero);
 9452   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
 9453   increment(len);
 9454   jcc(Assembler::notZero, copy_chars_loop);
 9455 
 9456   // if compression succeeded, return length
 9457   bind(return_length);
 9458   pop(result);
 9459   jmpb(done);
 9460 
 9461   // if compression failed, return 0
 9462   bind(return_zero);
 9463   xorl(result, result);
 9464   addptr(rsp, wordSize);
 9465 
 9466   bind(done);
 9467 }
 9468 
 9469 // Inflate byte[] array to char[].
 9470 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9471 //   @IntrinsicCandidate
 9472 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9473 //     for (int i = 0; i < len; i++) {
 9474 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9475 //     }
 9476 //   }
 9477 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9478   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9479   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9480   // rsi: src
 9481   // rdi: dst
 9482   // rdx: len
 9483   // rcx: tmp2
 9484 
 9485   // rsi holds start addr of source byte[] to be inflated
 9486   // rdi holds start addr of destination char[]
 9487   // rdx holds length
 9488   assert_different_registers(src, dst, len, tmp2);
 9489   movl(tmp2, len);
 9490   if ((UseAVX > 2) && // AVX512
 9491     VM_Version::supports_avx512vlbw() &&
 9492     VM_Version::supports_bmi2()) {
 9493 
 9494     Label copy_32_loop, copy_tail;
 9495     Register tmp3_aliased = len;
 9496 
 9497     // if length of the string is less than 16, handle it in an old fashioned way
 9498     testl(len, -16);
 9499     jcc(Assembler::zero, below_threshold);
 9500 
 9501     testl(len, -1 * AVX3Threshold);
 9502     jcc(Assembler::zero, avx3_threshold);
 9503 
 9504     // In order to use only one arithmetic operation for the main loop we use
 9505     // this pre-calculation
 9506     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9507     andl(len, -32);     // vector count
 9508     jccb(Assembler::zero, copy_tail);
 9509 
 9510     lea(src, Address(src, len, Address::times_1));
 9511     lea(dst, Address(dst, len, Address::times_2));
 9512     negptr(len);
 9513 
 9514 
 9515     // inflate 32 chars per iter
 9516     bind(copy_32_loop);
 9517     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9518     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9519     addptr(len, 32);
 9520     jcc(Assembler::notZero, copy_32_loop);
 9521 
 9522     bind(copy_tail);
 9523     // bail out when there is nothing to be done
 9524     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9525     jcc(Assembler::zero, done);
 9526 
 9527     // ~(~0 << length), where length is the # of remaining elements to process
 9528     movl(tmp3_aliased, -1);
 9529     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9530     notl(tmp3_aliased);
 9531     kmovdl(mask, tmp3_aliased);
 9532     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9533     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9534 
 9535     jmp(done);
 9536     bind(avx3_threshold);
 9537   }
 9538   if (UseSSE42Intrinsics) {
 9539     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9540 
 9541     if (UseAVX > 1) {
 9542       andl(tmp2, (16 - 1));
 9543       andl(len, -16);
 9544       jccb(Assembler::zero, copy_new_tail);
 9545     } else {
 9546       andl(tmp2, 0x00000007);   // tail count (in chars)
 9547       andl(len, 0xfffffff8);    // vector count (in chars)
 9548       jccb(Assembler::zero, copy_tail);
 9549     }
 9550 
 9551     // vectored inflation
 9552     lea(src, Address(src, len, Address::times_1));
 9553     lea(dst, Address(dst, len, Address::times_2));
 9554     negptr(len);
 9555 
 9556     if (UseAVX > 1) {
 9557       bind(copy_16_loop);
 9558       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9559       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9560       addptr(len, 16);
 9561       jcc(Assembler::notZero, copy_16_loop);
 9562 
 9563       bind(below_threshold);
 9564       bind(copy_new_tail);
 9565       movl(len, tmp2);
 9566       andl(tmp2, 0x00000007);
 9567       andl(len, 0xFFFFFFF8);
 9568       jccb(Assembler::zero, copy_tail);
 9569 
 9570       pmovzxbw(tmp1, Address(src, 0));
 9571       movdqu(Address(dst, 0), tmp1);
 9572       addptr(src, 8);
 9573       addptr(dst, 2 * 8);
 9574 
 9575       jmp(copy_tail, true);
 9576     }
 9577 
 9578     // inflate 8 chars per iter
 9579     bind(copy_8_loop);
 9580     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9581     movdqu(Address(dst, len, Address::times_2), tmp1);
 9582     addptr(len, 8);
 9583     jcc(Assembler::notZero, copy_8_loop);
 9584 
 9585     bind(copy_tail);
 9586     movl(len, tmp2);
 9587 
 9588     cmpl(len, 4);
 9589     jccb(Assembler::less, copy_bytes);
 9590 
 9591     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9592     pmovzxbw(tmp1, tmp1);
 9593     movq(Address(dst, 0), tmp1);
 9594     subptr(len, 4);
 9595     addptr(src, 4);
 9596     addptr(dst, 8);
 9597 
 9598     bind(copy_bytes);
 9599   } else {
 9600     bind(below_threshold);
 9601   }
 9602 
 9603   testl(len, len);
 9604   jccb(Assembler::zero, done);
 9605   lea(src, Address(src, len, Address::times_1));
 9606   lea(dst, Address(dst, len, Address::times_2));
 9607   negptr(len);
 9608 
 9609   // inflate 1 char per iter
 9610   bind(copy_chars_loop);
 9611   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9612   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9613   increment(len);
 9614   jcc(Assembler::notZero, copy_chars_loop);
 9615 
 9616   bind(done);
 9617 }
 9618 
 9619 
 9620 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9621   switch(type) {
 9622     case T_BYTE:
 9623     case T_BOOLEAN:
 9624       evmovdqub(dst, kmask, src, merge, vector_len);
 9625       break;
 9626     case T_CHAR:
 9627     case T_SHORT:
 9628       evmovdquw(dst, kmask, src, merge, vector_len);
 9629       break;
 9630     case T_INT:
 9631     case T_FLOAT:
 9632       evmovdqul(dst, kmask, src, merge, vector_len);
 9633       break;
 9634     case T_LONG:
 9635     case T_DOUBLE:
 9636       evmovdquq(dst, kmask, src, merge, vector_len);
 9637       break;
 9638     default:
 9639       fatal("Unexpected type argument %s", type2name(type));
 9640       break;
 9641   }
 9642 }
 9643 
 9644 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9645   switch(type) {
 9646     case T_BYTE:
 9647     case T_BOOLEAN:
 9648       evmovdqub(dst, kmask, src, merge, vector_len);
 9649       break;
 9650     case T_CHAR:
 9651     case T_SHORT:
 9652       evmovdquw(dst, kmask, src, merge, vector_len);
 9653       break;
 9654     case T_INT:
 9655     case T_FLOAT:
 9656       evmovdqul(dst, kmask, src, merge, vector_len);
 9657       break;
 9658     case T_LONG:
 9659     case T_DOUBLE:
 9660       evmovdquq(dst, kmask, src, merge, vector_len);
 9661       break;
 9662     default:
 9663       fatal("Unexpected type argument %s", type2name(type));
 9664       break;
 9665   }
 9666 }
 9667 
 9668 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9669   switch(masklen) {
 9670     case 2:
 9671        knotbl(dst, src);
 9672        movl(rtmp, 3);
 9673        kmovbl(ktmp, rtmp);
 9674        kandbl(dst, ktmp, dst);
 9675        break;
 9676     case 4:
 9677        knotbl(dst, src);
 9678        movl(rtmp, 15);
 9679        kmovbl(ktmp, rtmp);
 9680        kandbl(dst, ktmp, dst);
 9681        break;
 9682     case 8:
 9683        knotbl(dst, src);
 9684        break;
 9685     case 16:
 9686        knotwl(dst, src);
 9687        break;
 9688     case 32:
 9689        knotdl(dst, src);
 9690        break;
 9691     case 64:
 9692        knotql(dst, src);
 9693        break;
 9694     default:
 9695       fatal("Unexpected vector length %d", masklen);
 9696       break;
 9697   }
 9698 }
 9699 
 9700 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9701   switch(type) {
 9702     case T_BOOLEAN:
 9703     case T_BYTE:
 9704        kandbl(dst, src1, src2);
 9705        break;
 9706     case T_CHAR:
 9707     case T_SHORT:
 9708        kandwl(dst, src1, src2);
 9709        break;
 9710     case T_INT:
 9711     case T_FLOAT:
 9712        kanddl(dst, src1, src2);
 9713        break;
 9714     case T_LONG:
 9715     case T_DOUBLE:
 9716        kandql(dst, src1, src2);
 9717        break;
 9718     default:
 9719       fatal("Unexpected type argument %s", type2name(type));
 9720       break;
 9721   }
 9722 }
 9723 
 9724 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9725   switch(type) {
 9726     case T_BOOLEAN:
 9727     case T_BYTE:
 9728        korbl(dst, src1, src2);
 9729        break;
 9730     case T_CHAR:
 9731     case T_SHORT:
 9732        korwl(dst, src1, src2);
 9733        break;
 9734     case T_INT:
 9735     case T_FLOAT:
 9736        kordl(dst, src1, src2);
 9737        break;
 9738     case T_LONG:
 9739     case T_DOUBLE:
 9740        korql(dst, src1, src2);
 9741        break;
 9742     default:
 9743       fatal("Unexpected type argument %s", type2name(type));
 9744       break;
 9745   }
 9746 }
 9747 
 9748 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9749   switch(type) {
 9750     case T_BOOLEAN:
 9751     case T_BYTE:
 9752        kxorbl(dst, src1, src2);
 9753        break;
 9754     case T_CHAR:
 9755     case T_SHORT:
 9756        kxorwl(dst, src1, src2);
 9757        break;
 9758     case T_INT:
 9759     case T_FLOAT:
 9760        kxordl(dst, src1, src2);
 9761        break;
 9762     case T_LONG:
 9763     case T_DOUBLE:
 9764        kxorql(dst, src1, src2);
 9765        break;
 9766     default:
 9767       fatal("Unexpected type argument %s", type2name(type));
 9768       break;
 9769   }
 9770 }
 9771 
 9772 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9773   switch(type) {
 9774     case T_BOOLEAN:
 9775     case T_BYTE:
 9776       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9777     case T_CHAR:
 9778     case T_SHORT:
 9779       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9780     case T_INT:
 9781     case T_FLOAT:
 9782       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9783     case T_LONG:
 9784     case T_DOUBLE:
 9785       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9786     default:
 9787       fatal("Unexpected type argument %s", type2name(type)); break;
 9788   }
 9789 }
 9790 
 9791 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9792   switch(type) {
 9793     case T_BOOLEAN:
 9794     case T_BYTE:
 9795       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9796     case T_CHAR:
 9797     case T_SHORT:
 9798       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9799     case T_INT:
 9800     case T_FLOAT:
 9801       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9802     case T_LONG:
 9803     case T_DOUBLE:
 9804       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9805     default:
 9806       fatal("Unexpected type argument %s", type2name(type)); break;
 9807   }
 9808 }
 9809 
 9810 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9811   switch(type) {
 9812     case T_BYTE:
 9813       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9814     case T_SHORT:
 9815       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9816     case T_INT:
 9817       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9818     case T_LONG:
 9819       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9820     default:
 9821       fatal("Unexpected type argument %s", type2name(type)); break;
 9822   }
 9823 }
 9824 
 9825 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9826   switch(type) {
 9827     case T_BYTE:
 9828       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9829     case T_SHORT:
 9830       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9831     case T_INT:
 9832       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9833     case T_LONG:
 9834       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9835     default:
 9836       fatal("Unexpected type argument %s", type2name(type)); break;
 9837   }
 9838 }
 9839 
 9840 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9841   switch(type) {
 9842     case T_BYTE:
 9843       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9844     case T_SHORT:
 9845       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9846     case T_INT:
 9847       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9848     case T_LONG:
 9849       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9850     default:
 9851       fatal("Unexpected type argument %s", type2name(type)); break;
 9852   }
 9853 }
 9854 
 9855 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9856   switch(type) {
 9857     case T_BYTE:
 9858       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9859     case T_SHORT:
 9860       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9861     case T_INT:
 9862       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9863     case T_LONG:
 9864       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9865     default:
 9866       fatal("Unexpected type argument %s", type2name(type)); break;
 9867   }
 9868 }
 9869 
 9870 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9871   switch(type) {
 9872     case T_INT:
 9873       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9874     case T_LONG:
 9875       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9876     default:
 9877       fatal("Unexpected type argument %s", type2name(type)); break;
 9878   }
 9879 }
 9880 
 9881 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9882   switch(type) {
 9883     case T_INT:
 9884       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9885     case T_LONG:
 9886       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9887     default:
 9888       fatal("Unexpected type argument %s", type2name(type)); break;
 9889   }
 9890 }
 9891 
 9892 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9893   switch(type) {
 9894     case T_INT:
 9895       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9896     case T_LONG:
 9897       evporq(dst, mask, nds, src, merge, vector_len); break;
 9898     default:
 9899       fatal("Unexpected type argument %s", type2name(type)); break;
 9900   }
 9901 }
 9902 
 9903 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9904   switch(type) {
 9905     case T_INT:
 9906       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9907     case T_LONG:
 9908       evporq(dst, mask, nds, src, merge, vector_len); break;
 9909     default:
 9910       fatal("Unexpected type argument %s", type2name(type)); break;
 9911   }
 9912 }
 9913 
 9914 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9915   switch(type) {
 9916     case T_INT:
 9917       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9918     case T_LONG:
 9919       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9920     default:
 9921       fatal("Unexpected type argument %s", type2name(type)); break;
 9922   }
 9923 }
 9924 
 9925 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9926   switch(type) {
 9927     case T_INT:
 9928       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9929     case T_LONG:
 9930       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9931     default:
 9932       fatal("Unexpected type argument %s", type2name(type)); break;
 9933   }
 9934 }
 9935 
 9936 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
 9937   switch(masklen) {
 9938     case 8:
 9939        kortestbl(src1, src2);
 9940        break;
 9941     case 16:
 9942        kortestwl(src1, src2);
 9943        break;
 9944     case 32:
 9945        kortestdl(src1, src2);
 9946        break;
 9947     case 64:
 9948        kortestql(src1, src2);
 9949        break;
 9950     default:
 9951       fatal("Unexpected mask length %d", masklen);
 9952       break;
 9953   }
 9954 }
 9955 
 9956 
 9957 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
 9958   switch(masklen)  {
 9959     case 8:
 9960        ktestbl(src1, src2);
 9961        break;
 9962     case 16:
 9963        ktestwl(src1, src2);
 9964        break;
 9965     case 32:
 9966        ktestdl(src1, src2);
 9967        break;
 9968     case 64:
 9969        ktestql(src1, src2);
 9970        break;
 9971     default:
 9972       fatal("Unexpected mask length %d", masklen);
 9973       break;
 9974   }
 9975 }
 9976 
 9977 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9978   switch(type) {
 9979     case T_INT:
 9980       evprold(dst, mask, src, shift, merge, vlen_enc); break;
 9981     case T_LONG:
 9982       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
 9983     default:
 9984       fatal("Unexpected type argument %s", type2name(type)); break;
 9985       break;
 9986   }
 9987 }
 9988 
 9989 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9990   switch(type) {
 9991     case T_INT:
 9992       evprord(dst, mask, src, shift, merge, vlen_enc); break;
 9993     case T_LONG:
 9994       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
 9995     default:
 9996       fatal("Unexpected type argument %s", type2name(type)); break;
 9997   }
 9998 }
 9999 
10000 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10001   switch(type) {
10002     case T_INT:
10003       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
10004     case T_LONG:
10005       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
10006     default:
10007       fatal("Unexpected type argument %s", type2name(type)); break;
10008   }
10009 }
10010 
10011 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10012   switch(type) {
10013     case T_INT:
10014       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
10015     case T_LONG:
10016       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
10017     default:
10018       fatal("Unexpected type argument %s", type2name(type)); break;
10019   }
10020 }
10021 
10022 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10023   assert(rscratch != noreg || always_reachable(src), "missing");
10024 
10025   if (reachable(src)) {
10026     evpandq(dst, nds, as_Address(src), vector_len);
10027   } else {
10028     lea(rscratch, src);
10029     evpandq(dst, nds, Address(rscratch, 0), vector_len);
10030   }
10031 }
10032 
10033 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
10034   assert(rscratch != noreg || always_reachable(src), "missing");
10035 
10036   if (reachable(src)) {
10037     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
10038   } else {
10039     lea(rscratch, src);
10040     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
10041   }
10042 }
10043 
10044 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10045   assert(rscratch != noreg || always_reachable(src), "missing");
10046 
10047   if (reachable(src)) {
10048     evporq(dst, nds, as_Address(src), vector_len);
10049   } else {
10050     lea(rscratch, src);
10051     evporq(dst, nds, Address(rscratch, 0), vector_len);
10052   }
10053 }
10054 
10055 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
10056   assert(rscratch != noreg || always_reachable(src3), "missing");
10057 
10058   if (reachable(src3)) {
10059     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
10060   } else {
10061     lea(rscratch, src3);
10062     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
10063   }
10064 }
10065 
10066 #if COMPILER2_OR_JVMCI
10067 
10068 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
10069                                  Register length, Register temp, int vec_enc) {
10070   // Computing mask for predicated vector store.
10071   movptr(temp, -1);
10072   bzhiq(temp, temp, length);
10073   kmov(mask, temp);
10074   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
10075 }
10076 
10077 // Set memory operation for length "less than" 64 bytes.
10078 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
10079                                        XMMRegister xmm, KRegister mask, Register length,
10080                                        Register temp, bool use64byteVector) {
10081   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10082   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10083   if (!use64byteVector) {
10084     fill32(dst, disp, xmm);
10085     subptr(length, 32 >> shift);
10086     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
10087   } else {
10088     assert(MaxVectorSize == 64, "vector length != 64");
10089     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
10090   }
10091 }
10092 
10093 
10094 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
10095                                        XMMRegister xmm, KRegister mask, Register length,
10096                                        Register temp) {
10097   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10098   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10099   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
10100 }
10101 
10102 
10103 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
10104   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10105   vmovdqu(dst, xmm);
10106 }
10107 
10108 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
10109   fill32(Address(dst, disp), xmm);
10110 }
10111 
10112 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
10113   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10114   if (!use64byteVector) {
10115     fill32(dst, xmm);
10116     fill32(dst.plus_disp(32), xmm);
10117   } else {
10118     evmovdquq(dst, xmm, Assembler::AVX_512bit);
10119   }
10120 }
10121 
10122 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
10123   fill64(Address(dst, disp), xmm, use64byteVector);
10124 }
10125 
10126 #ifdef _LP64
10127 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
10128                                         Register count, Register rtmp, XMMRegister xtmp) {
10129   Label L_exit;
10130   Label L_fill_start;
10131   Label L_fill_64_bytes;
10132   Label L_fill_96_bytes;
10133   Label L_fill_128_bytes;
10134   Label L_fill_128_bytes_loop;
10135   Label L_fill_128_loop_header;
10136   Label L_fill_128_bytes_loop_header;
10137   Label L_fill_128_bytes_loop_pre_header;
10138   Label L_fill_zmm_sequence;
10139 
10140   int shift = -1;
10141   int avx3threshold = VM_Version::avx3_threshold();
10142   switch(type) {
10143     case T_BYTE:  shift = 0;
10144       break;
10145     case T_SHORT: shift = 1;
10146       break;
10147     case T_INT:   shift = 2;
10148       break;
10149     /* Uncomment when LONG fill stubs are supported.
10150     case T_LONG:  shift = 3;
10151       break;
10152     */
10153     default:
10154       fatal("Unhandled type: %s\n", type2name(type));
10155   }
10156 
10157   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
10158 
10159     if (MaxVectorSize == 64) {
10160       cmpq(count, avx3threshold >> shift);
10161       jcc(Assembler::greater, L_fill_zmm_sequence);
10162     }
10163 
10164     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
10165 
10166     bind(L_fill_start);
10167 
10168     cmpq(count, 32 >> shift);
10169     jccb(Assembler::greater, L_fill_64_bytes);
10170     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
10171     jmp(L_exit);
10172 
10173     bind(L_fill_64_bytes);
10174     cmpq(count, 64 >> shift);
10175     jccb(Assembler::greater, L_fill_96_bytes);
10176     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
10177     jmp(L_exit);
10178 
10179     bind(L_fill_96_bytes);
10180     cmpq(count, 96 >> shift);
10181     jccb(Assembler::greater, L_fill_128_bytes);
10182     fill64(to, 0, xtmp);
10183     subq(count, 64 >> shift);
10184     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
10185     jmp(L_exit);
10186 
10187     bind(L_fill_128_bytes);
10188     cmpq(count, 128 >> shift);
10189     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
10190     fill64(to, 0, xtmp);
10191     fill32(to, 64, xtmp);
10192     subq(count, 96 >> shift);
10193     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
10194     jmp(L_exit);
10195 
10196     bind(L_fill_128_bytes_loop_pre_header);
10197     {
10198       mov(rtmp, to);
10199       andq(rtmp, 31);
10200       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
10201       negq(rtmp);
10202       addq(rtmp, 32);
10203       mov64(r8, -1L);
10204       bzhiq(r8, r8, rtmp);
10205       kmovql(k2, r8);
10206       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
10207       addq(to, rtmp);
10208       shrq(rtmp, shift);
10209       subq(count, rtmp);
10210     }
10211 
10212     cmpq(count, 128 >> shift);
10213     jcc(Assembler::less, L_fill_start);
10214 
10215     bind(L_fill_128_bytes_loop_header);
10216     subq(count, 128 >> shift);
10217 
10218     align32();
10219     bind(L_fill_128_bytes_loop);
10220       fill64(to, 0, xtmp);
10221       fill64(to, 64, xtmp);
10222       addq(to, 128);
10223       subq(count, 128 >> shift);
10224       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
10225 
10226     addq(count, 128 >> shift);
10227     jcc(Assembler::zero, L_exit);
10228     jmp(L_fill_start);
10229   }
10230 
10231   if (MaxVectorSize == 64) {
10232     // Sequence using 64 byte ZMM register.
10233     Label L_fill_128_bytes_zmm;
10234     Label L_fill_192_bytes_zmm;
10235     Label L_fill_192_bytes_loop_zmm;
10236     Label L_fill_192_bytes_loop_header_zmm;
10237     Label L_fill_192_bytes_loop_pre_header_zmm;
10238     Label L_fill_start_zmm_sequence;
10239 
10240     bind(L_fill_zmm_sequence);
10241     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
10242 
10243     bind(L_fill_start_zmm_sequence);
10244     cmpq(count, 64 >> shift);
10245     jccb(Assembler::greater, L_fill_128_bytes_zmm);
10246     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
10247     jmp(L_exit);
10248 
10249     bind(L_fill_128_bytes_zmm);
10250     cmpq(count, 128 >> shift);
10251     jccb(Assembler::greater, L_fill_192_bytes_zmm);
10252     fill64(to, 0, xtmp, true);
10253     subq(count, 64 >> shift);
10254     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
10255     jmp(L_exit);
10256 
10257     bind(L_fill_192_bytes_zmm);
10258     cmpq(count, 192 >> shift);
10259     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
10260     fill64(to, 0, xtmp, true);
10261     fill64(to, 64, xtmp, true);
10262     subq(count, 128 >> shift);
10263     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
10264     jmp(L_exit);
10265 
10266     bind(L_fill_192_bytes_loop_pre_header_zmm);
10267     {
10268       movq(rtmp, to);
10269       andq(rtmp, 63);
10270       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
10271       negq(rtmp);
10272       addq(rtmp, 64);
10273       mov64(r8, -1L);
10274       bzhiq(r8, r8, rtmp);
10275       kmovql(k2, r8);
10276       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
10277       addq(to, rtmp);
10278       shrq(rtmp, shift);
10279       subq(count, rtmp);
10280     }
10281 
10282     cmpq(count, 192 >> shift);
10283     jcc(Assembler::less, L_fill_start_zmm_sequence);
10284 
10285     bind(L_fill_192_bytes_loop_header_zmm);
10286     subq(count, 192 >> shift);
10287 
10288     align32();
10289     bind(L_fill_192_bytes_loop_zmm);
10290       fill64(to, 0, xtmp, true);
10291       fill64(to, 64, xtmp, true);
10292       fill64(to, 128, xtmp, true);
10293       addq(to, 192);
10294       subq(count, 192 >> shift);
10295       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10296 
10297     addq(count, 192 >> shift);
10298     jcc(Assembler::zero, L_exit);
10299     jmp(L_fill_start_zmm_sequence);
10300   }
10301   bind(L_exit);
10302 }
10303 #endif
10304 #endif //COMPILER2_OR_JVMCI
10305 
10306 
10307 #ifdef _LP64
10308 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10309   Label done;
10310   cvttss2sil(dst, src);
10311   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10312   cmpl(dst, 0x80000000); // float_sign_flip
10313   jccb(Assembler::notEqual, done);
10314   subptr(rsp, 8);
10315   movflt(Address(rsp, 0), src);
10316   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10317   pop(dst);
10318   bind(done);
10319 }
10320 
10321 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10322   Label done;
10323   cvttsd2sil(dst, src);
10324   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10325   cmpl(dst, 0x80000000); // float_sign_flip
10326   jccb(Assembler::notEqual, done);
10327   subptr(rsp, 8);
10328   movdbl(Address(rsp, 0), src);
10329   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10330   pop(dst);
10331   bind(done);
10332 }
10333 
10334 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10335   Label done;
10336   cvttss2siq(dst, src);
10337   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10338   jccb(Assembler::notEqual, done);
10339   subptr(rsp, 8);
10340   movflt(Address(rsp, 0), src);
10341   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10342   pop(dst);
10343   bind(done);
10344 }
10345 
10346 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10347   // Following code is line by line assembly translation rounding algorithm.
10348   // Please refer to java.lang.Math.round(float) algorithm for details.
10349   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10350   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10351   const int32_t FloatConsts_EXP_BIAS = 127;
10352   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10353   const int32_t MINUS_32 = 0xFFFFFFE0;
10354   Label L_special_case, L_block1, L_exit;
10355   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10356   movdl(dst, src);
10357   andl(dst, rtmp);
10358   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10359   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10360   subl(rtmp, dst);
10361   movl(rcx, rtmp);
10362   movl(dst, MINUS_32);
10363   testl(rtmp, dst);
10364   jccb(Assembler::notEqual, L_special_case);
10365   movdl(dst, src);
10366   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10367   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10368   movdl(rtmp, src);
10369   testl(rtmp, rtmp);
10370   jccb(Assembler::greaterEqual, L_block1);
10371   negl(dst);
10372   bind(L_block1);
10373   sarl(dst);
10374   addl(dst, 0x1);
10375   sarl(dst, 0x1);
10376   jmp(L_exit);
10377   bind(L_special_case);
10378   convert_f2i(dst, src);
10379   bind(L_exit);
10380 }
10381 
10382 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10383   // Following code is line by line assembly translation rounding algorithm.
10384   // Please refer to java.lang.Math.round(double) algorithm for details.
10385   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10386   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10387   const int64_t DoubleConsts_EXP_BIAS = 1023;
10388   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10389   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10390   Label L_special_case, L_block1, L_exit;
10391   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10392   movq(dst, src);
10393   andq(dst, rtmp);
10394   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10395   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10396   subq(rtmp, dst);
10397   movq(rcx, rtmp);
10398   mov64(dst, MINUS_64);
10399   testq(rtmp, dst);
10400   jccb(Assembler::notEqual, L_special_case);
10401   movq(dst, src);
10402   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10403   andq(dst, rtmp);
10404   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10405   orq(dst, rtmp);
10406   movq(rtmp, src);
10407   testq(rtmp, rtmp);
10408   jccb(Assembler::greaterEqual, L_block1);
10409   negq(dst);
10410   bind(L_block1);
10411   sarq(dst);
10412   addq(dst, 0x1);
10413   sarq(dst, 0x1);
10414   jmp(L_exit);
10415   bind(L_special_case);
10416   convert_d2l(dst, src);
10417   bind(L_exit);
10418 }
10419 
10420 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10421   Label done;
10422   cvttsd2siq(dst, src);
10423   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10424   jccb(Assembler::notEqual, done);
10425   subptr(rsp, 8);
10426   movdbl(Address(rsp, 0), src);
10427   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10428   pop(dst);
10429   bind(done);
10430 }
10431 
10432 void MacroAssembler::cache_wb(Address line)
10433 {
10434   // 64 bit cpus always support clflush
10435   assert(VM_Version::supports_clflush(), "clflush should be available");
10436   bool optimized = VM_Version::supports_clflushopt();
10437   bool no_evict = VM_Version::supports_clwb();
10438 
10439   // prefer clwb (writeback without evict) otherwise
10440   // prefer clflushopt (potentially parallel writeback with evict)
10441   // otherwise fallback on clflush (serial writeback with evict)
10442 
10443   if (optimized) {
10444     if (no_evict) {
10445       clwb(line);
10446     } else {
10447       clflushopt(line);
10448     }
10449   } else {
10450     // no need for fence when using CLFLUSH
10451     clflush(line);
10452   }
10453 }
10454 
10455 void MacroAssembler::cache_wbsync(bool is_pre)
10456 {
10457   assert(VM_Version::supports_clflush(), "clflush should be available");
10458   bool optimized = VM_Version::supports_clflushopt();
10459   bool no_evict = VM_Version::supports_clwb();
10460 
10461   // pick the correct implementation
10462 
10463   if (!is_pre && (optimized || no_evict)) {
10464     // need an sfence for post flush when using clflushopt or clwb
10465     // otherwise no no need for any synchroniaztion
10466 
10467     sfence();
10468   }
10469 }
10470 
10471 #endif // _LP64
10472 
10473 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10474   switch (cond) {
10475     // Note some conditions are synonyms for others
10476     case Assembler::zero:         return Assembler::notZero;
10477     case Assembler::notZero:      return Assembler::zero;
10478     case Assembler::less:         return Assembler::greaterEqual;
10479     case Assembler::lessEqual:    return Assembler::greater;
10480     case Assembler::greater:      return Assembler::lessEqual;
10481     case Assembler::greaterEqual: return Assembler::less;
10482     case Assembler::below:        return Assembler::aboveEqual;
10483     case Assembler::belowEqual:   return Assembler::above;
10484     case Assembler::above:        return Assembler::belowEqual;
10485     case Assembler::aboveEqual:   return Assembler::below;
10486     case Assembler::overflow:     return Assembler::noOverflow;
10487     case Assembler::noOverflow:   return Assembler::overflow;
10488     case Assembler::negative:     return Assembler::positive;
10489     case Assembler::positive:     return Assembler::negative;
10490     case Assembler::parity:       return Assembler::noParity;
10491     case Assembler::noParity:     return Assembler::parity;
10492   }
10493   ShouldNotReachHere(); return Assembler::overflow;
10494 }
10495 
10496 SkipIfEqual::SkipIfEqual(
10497     MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
10498   _masm = masm;
10499   _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
10500   _masm->jcc(Assembler::equal, _label);
10501 }
10502 
10503 SkipIfEqual::~SkipIfEqual() {
10504   _masm->bind(_label);
10505 }
10506 
10507 // 32-bit Windows has its own fast-path implementation
10508 // of get_thread
10509 #if !defined(WIN32) || defined(_LP64)
10510 
10511 // This is simply a call to Thread::current()
10512 void MacroAssembler::get_thread(Register thread) {
10513   if (thread != rax) {
10514     push(rax);
10515   }
10516   LP64_ONLY(push(rdi);)
10517   LP64_ONLY(push(rsi);)
10518   push(rdx);
10519   push(rcx);
10520 #ifdef _LP64
10521   push(r8);
10522   push(r9);
10523   push(r10);
10524   push(r11);
10525 #endif
10526 
10527   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10528 
10529 #ifdef _LP64
10530   pop(r11);
10531   pop(r10);
10532   pop(r9);
10533   pop(r8);
10534 #endif
10535   pop(rcx);
10536   pop(rdx);
10537   LP64_ONLY(pop(rsi);)
10538   LP64_ONLY(pop(rdi);)
10539   if (thread != rax) {
10540     mov(thread, rax);
10541     pop(rax);
10542   }
10543 }
10544 
10545 
10546 #endif // !WIN32 || _LP64
10547 
10548 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10549   Label L_stack_ok;
10550   if (bias == 0) {
10551     testptr(sp, 2 * wordSize - 1);
10552   } else {
10553     // lea(tmp, Address(rsp, bias);
10554     mov(tmp, sp);
10555     addptr(tmp, bias);
10556     testptr(tmp, 2 * wordSize - 1);
10557   }
10558   jcc(Assembler::equal, L_stack_ok);
10559   block_comment(msg);
10560   stop(msg);
10561   bind(L_stack_ok);
10562 }
10563 
10564 // Implements fast-locking.
10565 // Branches to slow upon failure to lock the object, with ZF cleared.
10566 // Falls through upon success with unspecified ZF.
10567 //
10568 // obj: the object to be locked
10569 // hdr: the (pre-loaded) header of the object, must be rax
10570 // thread: the thread which attempts to lock obj
10571 // tmp: a temporary register
10572 void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
10573   assert(hdr == rax, "header must be in rax for cmpxchg");
10574   assert_different_registers(obj, hdr, thread, tmp);
10575 
10576   // First we need to check if the lock-stack has room for pushing the object reference.
10577   // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
10578   // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
10579   // avoids one branch.
10580   cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
10581   jcc(Assembler::greater, slow);
10582 
10583   // Now we attempt to take the fast-lock.
10584   // Clear lock_mask bits (locked state).
10585   andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
10586   movptr(tmp, hdr);
10587   // Set unlocked_value bit.
10588   orptr(hdr, markWord::unlocked_value);
10589   lock();
10590   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10591   jcc(Assembler::notEqual, slow);
10592 
10593   // If successful, push object to lock-stack.
10594   movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
10595   movptr(Address(thread, tmp), obj);
10596   incrementl(tmp, oopSize);
10597   movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
10598 }
10599 
10600 // Implements fast-unlocking.
10601 // Branches to slow upon failure, with ZF cleared.
10602 // Falls through upon success, with unspecified ZF.
10603 //
10604 // obj: the object to be unlocked
10605 // hdr: the (pre-loaded) header of the object, must be rax
10606 // tmp: a temporary register
10607 void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) {
10608   assert(hdr == rax, "header must be in rax for cmpxchg");
10609   assert_different_registers(obj, hdr, tmp);
10610 
10611   // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
10612   movptr(tmp, hdr); // The expected old value
10613   orptr(tmp, markWord::unlocked_value);
10614   lock();
10615   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10616   jcc(Assembler::notEqual, slow);
10617   // Pop the lock object from the lock-stack.
10618 #ifdef _LP64
10619   const Register thread = r15_thread;
10620 #else
10621   const Register thread = rax;
10622   get_thread(thread);
10623 #endif
10624   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10625 #ifdef ASSERT
10626   movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
10627   movptr(Address(thread, tmp), 0);
10628 #endif
10629 }