1 /*
    2  * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "code/SCCache.hpp"
   29 #include "code/compiledIC.hpp"
   30 #include "compiler/compiler_globals.hpp"
   31 #include "compiler/disassembler.hpp"
   32 #include "crc32c.h"
   33 #include "gc/shared/barrierSet.hpp"
   34 #include "gc/shared/barrierSetAssembler.hpp"
   35 #include "gc/shared/collectedHeap.inline.hpp"
   36 #include "gc/shared/tlab_globals.hpp"
   37 #include "interpreter/bytecodeHistogram.hpp"
   38 #include "interpreter/interpreter.hpp"
   39 #include "jvm.h"
   40 #include "memory/resourceArea.hpp"
   41 #include "memory/universe.hpp"
   42 #include "oops/accessDecorators.hpp"
   43 #include "oops/compressedKlass.inline.hpp"
   44 #include "oops/compressedOops.inline.hpp"
   45 #include "oops/klass.inline.hpp"
   46 #include "prims/methodHandles.hpp"
   47 #include "runtime/continuation.hpp"
   48 #include "runtime/interfaceSupport.inline.hpp"
   49 #include "runtime/javaThread.hpp"
   50 #include "runtime/jniHandles.hpp"
   51 #include "runtime/objectMonitor.hpp"
   52 #include "runtime/os.hpp"
   53 #include "runtime/safepoint.hpp"
   54 #include "runtime/safepointMechanism.hpp"
   55 #include "runtime/sharedRuntime.hpp"
   56 #include "runtime/stubRoutines.hpp"
   57 #include "utilities/checkedCast.hpp"
   58 #include "utilities/macros.hpp"
   59 
   60 #ifdef PRODUCT
   61 #define BLOCK_COMMENT(str) /* nothing */
   62 #define STOP(error) stop(error)
   63 #else
   64 #define BLOCK_COMMENT(str) block_comment(str)
   65 #define STOP(error) block_comment(error); stop(error)
   66 #endif
   67 
   68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   69 
   70 #ifdef ASSERT
   71 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   72 #endif
   73 
   74 static const Assembler::Condition reverse[] = {
   75     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   76     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   77     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   78     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   79     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   80     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   81     Assembler::above          /* belowEqual    = 0x6 */ ,
   82     Assembler::belowEqual     /* above         = 0x7 */ ,
   83     Assembler::positive       /* negative      = 0x8 */ ,
   84     Assembler::negative       /* positive      = 0x9 */ ,
   85     Assembler::noParity       /* parity        = 0xa */ ,
   86     Assembler::parity         /* noParity      = 0xb */ ,
   87     Assembler::greaterEqual   /* less          = 0xc */ ,
   88     Assembler::less           /* greaterEqual  = 0xd */ ,
   89     Assembler::greater        /* lessEqual     = 0xe */ ,
   90     Assembler::lessEqual      /* greater       = 0xf, */
   91 
   92 };
   93 
   94 
   95 // Implementation of MacroAssembler
   96 
   97 // First all the versions that have distinct versions depending on 32/64 bit
   98 // Unless the difference is trivial (1 line or so).
   99 
  100 #ifndef _LP64
  101 
  102 // 32bit versions
  103 
  104 Address MacroAssembler::as_Address(AddressLiteral adr) {
  105   return Address(adr.target(), adr.rspec());
  106 }
  107 
  108 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  109   assert(rscratch == noreg, "");
  110   return Address::make_array(adr);
  111 }
  112 
  113 void MacroAssembler::call_VM_leaf_base(address entry_point,
  114                                        int number_of_arguments) {
  115   call(RuntimeAddress(entry_point));
  116   increment(rsp, number_of_arguments * wordSize);
  117 }
  118 
  119 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  120   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  121 }
  122 
  123 
  124 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  125   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  126 }
  127 
  128 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  129   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  130 }
  131 
  132 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  133   assert(rscratch == noreg, "redundant");
  134   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  135 }
  136 
  137 void MacroAssembler::extend_sign(Register hi, Register lo) {
  138   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  139   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  140     cdql();
  141   } else {
  142     movl(hi, lo);
  143     sarl(hi, 31);
  144   }
  145 }
  146 
  147 void MacroAssembler::jC2(Register tmp, Label& L) {
  148   // set parity bit if FPU flag C2 is set (via rax)
  149   save_rax(tmp);
  150   fwait(); fnstsw_ax();
  151   sahf();
  152   restore_rax(tmp);
  153   // branch
  154   jcc(Assembler::parity, L);
  155 }
  156 
  157 void MacroAssembler::jnC2(Register tmp, Label& L) {
  158   // set parity bit if FPU flag C2 is set (via rax)
  159   save_rax(tmp);
  160   fwait(); fnstsw_ax();
  161   sahf();
  162   restore_rax(tmp);
  163   // branch
  164   jcc(Assembler::noParity, L);
  165 }
  166 
  167 // 32bit can do a case table jump in one instruction but we no longer allow the base
  168 // to be installed in the Address class
  169 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  170   assert(rscratch == noreg, "not needed");
  171   jmp(as_Address(entry, noreg));
  172 }
  173 
  174 // Note: y_lo will be destroyed
  175 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  176   // Long compare for Java (semantics as described in JVM spec.)
  177   Label high, low, done;
  178 
  179   cmpl(x_hi, y_hi);
  180   jcc(Assembler::less, low);
  181   jcc(Assembler::greater, high);
  182   // x_hi is the return register
  183   xorl(x_hi, x_hi);
  184   cmpl(x_lo, y_lo);
  185   jcc(Assembler::below, low);
  186   jcc(Assembler::equal, done);
  187 
  188   bind(high);
  189   xorl(x_hi, x_hi);
  190   increment(x_hi);
  191   jmp(done);
  192 
  193   bind(low);
  194   xorl(x_hi, x_hi);
  195   decrementl(x_hi);
  196 
  197   bind(done);
  198 }
  199 
  200 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  201   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  202 }
  203 
  204 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  205   assert(rscratch == noreg, "not needed");
  206 
  207   // leal(dst, as_Address(adr));
  208   // see note in movl as to why we must use a move
  209   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  210 }
  211 
  212 void MacroAssembler::leave() {
  213   mov(rsp, rbp);
  214   pop(rbp);
  215 }
  216 
  217 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  218   // Multiplication of two Java long values stored on the stack
  219   // as illustrated below. Result is in rdx:rax.
  220   //
  221   // rsp ---> [  ??  ] \               \
  222   //            ....    | y_rsp_offset  |
  223   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  224   //          [ y_hi ]                  | (in bytes)
  225   //            ....                    |
  226   //          [ x_lo ]                 /
  227   //          [ x_hi ]
  228   //            ....
  229   //
  230   // Basic idea: lo(result) = lo(x_lo * y_lo)
  231   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  232   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  233   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  234   Label quick;
  235   // load x_hi, y_hi and check if quick
  236   // multiplication is possible
  237   movl(rbx, x_hi);
  238   movl(rcx, y_hi);
  239   movl(rax, rbx);
  240   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  241   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  242   // do full multiplication
  243   // 1st step
  244   mull(y_lo);                                    // x_hi * y_lo
  245   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  246   // 2nd step
  247   movl(rax, x_lo);
  248   mull(rcx);                                     // x_lo * y_hi
  249   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  250   // 3rd step
  251   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  252   movl(rax, x_lo);
  253   mull(y_lo);                                    // x_lo * y_lo
  254   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  255 }
  256 
  257 void MacroAssembler::lneg(Register hi, Register lo) {
  258   negl(lo);
  259   adcl(hi, 0);
  260   negl(hi);
  261 }
  262 
  263 void MacroAssembler::lshl(Register hi, Register lo) {
  264   // Java shift left long support (semantics as described in JVM spec., p.305)
  265   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  266   // shift value is in rcx !
  267   assert(hi != rcx, "must not use rcx");
  268   assert(lo != rcx, "must not use rcx");
  269   const Register s = rcx;                        // shift count
  270   const int      n = BitsPerWord;
  271   Label L;
  272   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  273   cmpl(s, n);                                    // if (s < n)
  274   jcc(Assembler::less, L);                       // else (s >= n)
  275   movl(hi, lo);                                  // x := x << n
  276   xorl(lo, lo);
  277   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  278   bind(L);                                       // s (mod n) < n
  279   shldl(hi, lo);                                 // x := x << s
  280   shll(lo);
  281 }
  282 
  283 
  284 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  285   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  286   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  287   assert(hi != rcx, "must not use rcx");
  288   assert(lo != rcx, "must not use rcx");
  289   const Register s = rcx;                        // shift count
  290   const int      n = BitsPerWord;
  291   Label L;
  292   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  293   cmpl(s, n);                                    // if (s < n)
  294   jcc(Assembler::less, L);                       // else (s >= n)
  295   movl(lo, hi);                                  // x := x >> n
  296   if (sign_extension) sarl(hi, 31);
  297   else                xorl(hi, hi);
  298   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  299   bind(L);                                       // s (mod n) < n
  300   shrdl(lo, hi);                                 // x := x >> s
  301   if (sign_extension) sarl(hi);
  302   else                shrl(hi);
  303 }
  304 
  305 void MacroAssembler::movoop(Register dst, jobject obj) {
  306   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  307 }
  308 
  309 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  310   assert(rscratch == noreg, "redundant");
  311   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  312 }
  313 
  314 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  315   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  316 }
  317 
  318 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  319   assert(rscratch == noreg, "redundant");
  320   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  321 }
  322 
  323 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  324   if (src.is_lval()) {
  325     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  326   } else {
  327     movl(dst, as_Address(src));
  328   }
  329 }
  330 
  331 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  332   assert(rscratch == noreg, "redundant");
  333   movl(as_Address(dst, noreg), src);
  334 }
  335 
  336 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  337   movl(dst, as_Address(src, noreg));
  338 }
  339 
  340 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  341   assert(rscratch == noreg, "redundant");
  342   movl(dst, src);
  343 }
  344 
  345 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  346   assert(rscratch == noreg, "redundant");
  347   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  348 }
  349 
  350 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  351   assert(rscratch == noreg, "redundant");
  352   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  353 }
  354 
  355 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  356   assert(rscratch == noreg, "redundant");
  357   if (src.is_lval()) {
  358     push_literal32((int32_t)src.target(), src.rspec());
  359   } else {
  360     pushl(as_Address(src));
  361   }
  362 }
  363 
  364 static void pass_arg0(MacroAssembler* masm, Register arg) {
  365   masm->push(arg);
  366 }
  367 
  368 static void pass_arg1(MacroAssembler* masm, Register arg) {
  369   masm->push(arg);
  370 }
  371 
  372 static void pass_arg2(MacroAssembler* masm, Register arg) {
  373   masm->push(arg);
  374 }
  375 
  376 static void pass_arg3(MacroAssembler* masm, Register arg) {
  377   masm->push(arg);
  378 }
  379 
  380 #ifndef PRODUCT
  381 extern "C" void findpc(intptr_t x);
  382 #endif
  383 
  384 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  385   // In order to get locks to work, we need to fake a in_VM state
  386   JavaThread* thread = JavaThread::current();
  387   JavaThreadState saved_state = thread->thread_state();
  388   thread->set_thread_state(_thread_in_vm);
  389   if (ShowMessageBoxOnError) {
  390     JavaThread* thread = JavaThread::current();
  391     JavaThreadState saved_state = thread->thread_state();
  392     thread->set_thread_state(_thread_in_vm);
  393     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  394       ttyLocker ttyl;
  395       BytecodeCounter::print();
  396     }
  397     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  398     // This is the value of eip which points to where verify_oop will return.
  399     if (os::message_box(msg, "Execution stopped, print registers?")) {
  400       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  401       BREAKPOINT;
  402     }
  403   }
  404   fatal("DEBUG MESSAGE: %s", msg);
  405 }
  406 
  407 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  408   ttyLocker ttyl;
  409   DebuggingContext debugging{};
  410   tty->print_cr("eip = 0x%08x", eip);
  411 #ifndef PRODUCT
  412   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  413     tty->cr();
  414     findpc(eip);
  415     tty->cr();
  416   }
  417 #endif
  418 #define PRINT_REG(rax) \
  419   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  420   PRINT_REG(rax);
  421   PRINT_REG(rbx);
  422   PRINT_REG(rcx);
  423   PRINT_REG(rdx);
  424   PRINT_REG(rdi);
  425   PRINT_REG(rsi);
  426   PRINT_REG(rbp);
  427   PRINT_REG(rsp);
  428 #undef PRINT_REG
  429   // Print some words near top of staack.
  430   int* dump_sp = (int*) rsp;
  431   for (int col1 = 0; col1 < 8; col1++) {
  432     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  433     os::print_location(tty, *dump_sp++);
  434   }
  435   for (int row = 0; row < 16; row++) {
  436     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  437     for (int col = 0; col < 8; col++) {
  438       tty->print(" 0x%08x", *dump_sp++);
  439     }
  440     tty->cr();
  441   }
  442   // Print some instructions around pc:
  443   Disassembler::decode((address)eip-64, (address)eip);
  444   tty->print_cr("--------");
  445   Disassembler::decode((address)eip, (address)eip+32);
  446 }
  447 
  448 void MacroAssembler::stop(const char* msg) {
  449   // push address of message
  450   ExternalAddress message((address)msg);
  451   pushptr(message.addr(), noreg);
  452   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  453   pusha();                                            // push registers
  454   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  455   hlt();
  456 }
  457 
  458 void MacroAssembler::warn(const char* msg) {
  459   push_CPU_state();
  460 
  461   // push address of message
  462   ExternalAddress message((address)msg);
  463   pushptr(message.addr(), noreg);
  464 
  465   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  466   addl(rsp, wordSize);       // discard argument
  467   pop_CPU_state();
  468 }
  469 
  470 void MacroAssembler::print_state() {
  471   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  472   pusha();                                            // push registers
  473 
  474   push_CPU_state();
  475   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  476   pop_CPU_state();
  477 
  478   popa();
  479   addl(rsp, wordSize);
  480 }
  481 
  482 #else // _LP64
  483 
  484 // 64 bit versions
  485 
  486 Address MacroAssembler::as_Address(AddressLiteral adr) {
  487   // amd64 always does this as a pc-rel
  488   // we can be absolute or disp based on the instruction type
  489   // jmp/call are displacements others are absolute
  490   assert(!adr.is_lval(), "must be rval");
  491   assert(reachable(adr), "must be");
  492   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  493 
  494 }
  495 
  496 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  497   AddressLiteral base = adr.base();
  498   lea(rscratch, base);
  499   Address index = adr.index();
  500   assert(index._disp == 0, "must not have disp"); // maybe it can?
  501   Address array(rscratch, index._index, index._scale, index._disp);
  502   return array;
  503 }
  504 
  505 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  506   Label L, E;
  507 
  508 #ifdef _WIN64
  509   // Windows always allocates space for it's register args
  510   assert(num_args <= 4, "only register arguments supported");
  511   subq(rsp,  frame::arg_reg_save_area_bytes);
  512 #endif
  513 
  514   // Align stack if necessary
  515   testl(rsp, 15);
  516   jcc(Assembler::zero, L);
  517 
  518   subq(rsp, 8);
  519   call(RuntimeAddress(entry_point));
  520   addq(rsp, 8);
  521   jmp(E);
  522 
  523   bind(L);
  524   call(RuntimeAddress(entry_point));
  525 
  526   bind(E);
  527 
  528 #ifdef _WIN64
  529   // restore stack pointer
  530   addq(rsp, frame::arg_reg_save_area_bytes);
  531 #endif
  532 
  533 }
  534 
  535 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  536   assert(!src2.is_lval(), "should use cmpptr");
  537   assert(rscratch != noreg || always_reachable(src2), "missing");
  538 
  539   if (reachable(src2)) {
  540     cmpq(src1, as_Address(src2));
  541   } else {
  542     lea(rscratch, src2);
  543     Assembler::cmpq(src1, Address(rscratch, 0));
  544   }
  545 }
  546 
  547 int MacroAssembler::corrected_idivq(Register reg) {
  548   // Full implementation of Java ldiv and lrem; checks for special
  549   // case as described in JVM spec., p.243 & p.271.  The function
  550   // returns the (pc) offset of the idivl instruction - may be needed
  551   // for implicit exceptions.
  552   //
  553   //         normal case                           special case
  554   //
  555   // input : rax: dividend                         min_long
  556   //         reg: divisor   (may not be eax/edx)   -1
  557   //
  558   // output: rax: quotient  (= rax idiv reg)       min_long
  559   //         rdx: remainder (= rax irem reg)       0
  560   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  561   static const int64_t min_long = 0x8000000000000000;
  562   Label normal_case, special_case;
  563 
  564   // check for special case
  565   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  566   jcc(Assembler::notEqual, normal_case);
  567   xorl(rdx, rdx); // prepare rdx for possible special case (where
  568                   // remainder = 0)
  569   cmpq(reg, -1);
  570   jcc(Assembler::equal, special_case);
  571 
  572   // handle normal case
  573   bind(normal_case);
  574   cdqq();
  575   int idivq_offset = offset();
  576   idivq(reg);
  577 
  578   // normal and special case exit
  579   bind(special_case);
  580 
  581   return idivq_offset;
  582 }
  583 
  584 void MacroAssembler::decrementq(Register reg, int value) {
  585   if (value == min_jint) { subq(reg, value); return; }
  586   if (value <  0) { incrementq(reg, -value); return; }
  587   if (value == 0) {                        ; return; }
  588   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  589   /* else */      { subq(reg, value)       ; return; }
  590 }
  591 
  592 void MacroAssembler::decrementq(Address dst, int value) {
  593   if (value == min_jint) { subq(dst, value); return; }
  594   if (value <  0) { incrementq(dst, -value); return; }
  595   if (value == 0) {                        ; return; }
  596   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  597   /* else */      { subq(dst, value)       ; return; }
  598 }
  599 
  600 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  601   assert(rscratch != noreg || always_reachable(dst), "missing");
  602 
  603   if (reachable(dst)) {
  604     incrementq(as_Address(dst));
  605   } else {
  606     lea(rscratch, dst);
  607     incrementq(Address(rscratch, 0));
  608   }
  609 }
  610 
  611 void MacroAssembler::incrementq(Register reg, int value) {
  612   if (value == min_jint) { addq(reg, value); return; }
  613   if (value <  0) { decrementq(reg, -value); return; }
  614   if (value == 0) {                        ; return; }
  615   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  616   /* else */      { addq(reg, value)       ; return; }
  617 }
  618 
  619 void MacroAssembler::incrementq(Address dst, int value) {
  620   if (value == min_jint) { addq(dst, value); return; }
  621   if (value <  0) { decrementq(dst, -value); return; }
  622   if (value == 0) {                        ; return; }
  623   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  624   /* else */      { addq(dst, value)       ; return; }
  625 }
  626 
  627 // 32bit can do a case table jump in one instruction but we no longer allow the base
  628 // to be installed in the Address class
  629 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  630   lea(rscratch, entry.base());
  631   Address dispatch = entry.index();
  632   assert(dispatch._base == noreg, "must be");
  633   dispatch._base = rscratch;
  634   jmp(dispatch);
  635 }
  636 
  637 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  638   ShouldNotReachHere(); // 64bit doesn't use two regs
  639   cmpq(x_lo, y_lo);
  640 }
  641 
  642 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  643   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  644 }
  645 
  646 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  647   lea(rscratch, adr);
  648   movptr(dst, rscratch);
  649 }
  650 
  651 void MacroAssembler::leave() {
  652   // %%% is this really better? Why not on 32bit too?
  653   emit_int8((unsigned char)0xC9); // LEAVE
  654 }
  655 
  656 void MacroAssembler::lneg(Register hi, Register lo) {
  657   ShouldNotReachHere(); // 64bit doesn't use two regs
  658   negq(lo);
  659 }
  660 
  661 void MacroAssembler::movoop(Register dst, jobject obj) {
  662   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  663 }
  664 
  665 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  666   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  667   movq(dst, rscratch);
  668 }
  669 
  670 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  671   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  672 }
  673 
  674 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  675   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  676   movq(dst, rscratch);
  677 }
  678 
  679 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  680   if (src.is_lval()) {
  681     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  682   } else {
  683     if (reachable(src)) {
  684       movq(dst, as_Address(src));
  685     } else {
  686       lea(dst, src);
  687       movq(dst, Address(dst, 0));
  688     }
  689   }
  690 }
  691 
  692 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  693   movq(as_Address(dst, rscratch), src);
  694 }
  695 
  696 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  697   movq(dst, as_Address(src, dst /*rscratch*/));
  698 }
  699 
  700 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  701 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  702   if (is_simm32(src)) {
  703     movptr(dst, checked_cast<int32_t>(src));
  704   } else {
  705     mov64(rscratch, src);
  706     movq(dst, rscratch);
  707   }
  708 }
  709 
  710 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  711   movoop(rscratch, obj);
  712   push(rscratch);
  713 }
  714 
  715 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  716   mov_metadata(rscratch, obj);
  717   push(rscratch);
  718 }
  719 
  720 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  721   lea(rscratch, src);
  722   if (src.is_lval()) {
  723     push(rscratch);
  724   } else {
  725     pushq(Address(rscratch, 0));
  726   }
  727 }
  728 
  729 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  730   reset_last_Java_frame(r15_thread, clear_fp);
  731 }
  732 
  733 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  734                                          Register last_java_fp,
  735                                          address  last_java_pc,
  736                                          Register rscratch) {
  737   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  738 }
  739 
  740 static void pass_arg0(MacroAssembler* masm, Register arg) {
  741   if (c_rarg0 != arg ) {
  742     masm->mov(c_rarg0, arg);
  743   }
  744 }
  745 
  746 static void pass_arg1(MacroAssembler* masm, Register arg) {
  747   if (c_rarg1 != arg ) {
  748     masm->mov(c_rarg1, arg);
  749   }
  750 }
  751 
  752 static void pass_arg2(MacroAssembler* masm, Register arg) {
  753   if (c_rarg2 != arg ) {
  754     masm->mov(c_rarg2, arg);
  755   }
  756 }
  757 
  758 static void pass_arg3(MacroAssembler* masm, Register arg) {
  759   if (c_rarg3 != arg ) {
  760     masm->mov(c_rarg3, arg);
  761   }
  762 }
  763 
  764 void MacroAssembler::stop(const char* msg) {
  765   if (ShowMessageBoxOnError) {
  766     address rip = pc();
  767     pusha(); // get regs on stack
  768     lea(c_rarg1, InternalAddress(rip));
  769     movq(c_rarg2, rsp); // pass pointer to regs array
  770   }
  771   lea(c_rarg0, ExternalAddress((address) msg));
  772   andq(rsp, -16); // align stack as required by ABI
  773   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  774   hlt();
  775   SCCache::add_C_string(msg);
  776 }
  777 
  778 void MacroAssembler::warn(const char* msg) {
  779   push(rbp);
  780   movq(rbp, rsp);
  781   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  782   push_CPU_state();   // keeps alignment at 16 bytes
  783 
  784   lea(c_rarg0, ExternalAddress((address) msg));
  785   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  786 
  787   pop_CPU_state();
  788   mov(rsp, rbp);
  789   pop(rbp);
  790 }
  791 
  792 void MacroAssembler::print_state() {
  793   address rip = pc();
  794   pusha();            // get regs on stack
  795   push(rbp);
  796   movq(rbp, rsp);
  797   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  798   push_CPU_state();   // keeps alignment at 16 bytes
  799 
  800   lea(c_rarg0, InternalAddress(rip));
  801   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  802   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  803 
  804   pop_CPU_state();
  805   mov(rsp, rbp);
  806   pop(rbp);
  807   popa();
  808 }
  809 
  810 #ifndef PRODUCT
  811 extern "C" void findpc(intptr_t x);
  812 #endif
  813 
  814 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  815   // In order to get locks to work, we need to fake a in_VM state
  816   if (ShowMessageBoxOnError) {
  817     JavaThread* thread = JavaThread::current();
  818     JavaThreadState saved_state = thread->thread_state();
  819     thread->set_thread_state(_thread_in_vm);
  820 #ifndef PRODUCT
  821     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  822       ttyLocker ttyl;
  823       BytecodeCounter::print();
  824     }
  825 #endif
  826     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  827     // XXX correct this offset for amd64
  828     // This is the value of eip which points to where verify_oop will return.
  829     if (os::message_box(msg, "Execution stopped, print registers?")) {
  830       print_state64(pc, regs);
  831       BREAKPOINT;
  832     }
  833   }
  834   fatal("DEBUG MESSAGE: %s", msg);
  835 }
  836 
  837 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  838   ttyLocker ttyl;
  839   DebuggingContext debugging{};
  840   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  841 #ifndef PRODUCT
  842   tty->cr();
  843   findpc(pc);
  844   tty->cr();
  845 #endif
  846 #define PRINT_REG(rax, value) \
  847   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  848   PRINT_REG(rax, regs[15]);
  849   PRINT_REG(rbx, regs[12]);
  850   PRINT_REG(rcx, regs[14]);
  851   PRINT_REG(rdx, regs[13]);
  852   PRINT_REG(rdi, regs[8]);
  853   PRINT_REG(rsi, regs[9]);
  854   PRINT_REG(rbp, regs[10]);
  855   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  856   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  857   PRINT_REG(r8 , regs[7]);
  858   PRINT_REG(r9 , regs[6]);
  859   PRINT_REG(r10, regs[5]);
  860   PRINT_REG(r11, regs[4]);
  861   PRINT_REG(r12, regs[3]);
  862   PRINT_REG(r13, regs[2]);
  863   PRINT_REG(r14, regs[1]);
  864   PRINT_REG(r15, regs[0]);
  865 #undef PRINT_REG
  866   // Print some words near the top of the stack.
  867   int64_t* rsp = &regs[16];
  868   int64_t* dump_sp = rsp;
  869   for (int col1 = 0; col1 < 8; col1++) {
  870     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  871     os::print_location(tty, *dump_sp++);
  872   }
  873   for (int row = 0; row < 25; row++) {
  874     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  875     for (int col = 0; col < 4; col++) {
  876       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  877     }
  878     tty->cr();
  879   }
  880   // Print some instructions around pc:
  881   Disassembler::decode((address)pc-64, (address)pc);
  882   tty->print_cr("--------");
  883   Disassembler::decode((address)pc, (address)pc+32);
  884 }
  885 
  886 // The java_calling_convention describes stack locations as ideal slots on
  887 // a frame with no abi restrictions. Since we must observe abi restrictions
  888 // (like the placement of the register window) the slots must be biased by
  889 // the following value.
  890 static int reg2offset_in(VMReg r) {
  891   // Account for saved rbp and return address
  892   // This should really be in_preserve_stack_slots
  893   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  894 }
  895 
  896 static int reg2offset_out(VMReg r) {
  897   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  898 }
  899 
  900 // A long move
  901 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  902 
  903   // The calling conventions assures us that each VMregpair is either
  904   // all really one physical register or adjacent stack slots.
  905 
  906   if (src.is_single_phys_reg() ) {
  907     if (dst.is_single_phys_reg()) {
  908       if (dst.first() != src.first()) {
  909         mov(dst.first()->as_Register(), src.first()->as_Register());
  910       }
  911     } else {
  912       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  913              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  914       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  915     }
  916   } else if (dst.is_single_phys_reg()) {
  917     assert(src.is_single_reg(),  "not a stack pair");
  918     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  919   } else {
  920     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  921     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  922     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  923   }
  924 }
  925 
  926 // A double move
  927 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  928 
  929   // The calling conventions assures us that each VMregpair is either
  930   // all really one physical register or adjacent stack slots.
  931 
  932   if (src.is_single_phys_reg() ) {
  933     if (dst.is_single_phys_reg()) {
  934       // In theory these overlap but the ordering is such that this is likely a nop
  935       if ( src.first() != dst.first()) {
  936         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  937       }
  938     } else {
  939       assert(dst.is_single_reg(), "not a stack pair");
  940       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  941     }
  942   } else if (dst.is_single_phys_reg()) {
  943     assert(src.is_single_reg(),  "not a stack pair");
  944     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  945   } else {
  946     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  947     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  948     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  949   }
  950 }
  951 
  952 
  953 // A float arg may have to do float reg int reg conversion
  954 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  955   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  956 
  957   // The calling conventions assures us that each VMregpair is either
  958   // all really one physical register or adjacent stack slots.
  959 
  960   if (src.first()->is_stack()) {
  961     if (dst.first()->is_stack()) {
  962       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  963       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  964     } else {
  965       // stack to reg
  966       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  967       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  968     }
  969   } else if (dst.first()->is_stack()) {
  970     // reg to stack
  971     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  972     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  973   } else {
  974     // reg to reg
  975     // In theory these overlap but the ordering is such that this is likely a nop
  976     if ( src.first() != dst.first()) {
  977       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  978     }
  979   }
  980 }
  981 
  982 // On 64 bit we will store integer like items to the stack as
  983 // 64 bits items (x86_32/64 abi) even though java would only store
  984 // 32bits for a parameter. On 32bit it will simply be 32 bits
  985 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  986 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  987   if (src.first()->is_stack()) {
  988     if (dst.first()->is_stack()) {
  989       // stack to stack
  990       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  991       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  992     } else {
  993       // stack to reg
  994       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  995     }
  996   } else if (dst.first()->is_stack()) {
  997     // reg to stack
  998     // Do we really have to sign extend???
  999     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 1000     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 1001   } else {
 1002     // Do we really have to sign extend???
 1003     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1004     if (dst.first() != src.first()) {
 1005       movq(dst.first()->as_Register(), src.first()->as_Register());
 1006     }
 1007   }
 1008 }
 1009 
 1010 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1011   if (src.first()->is_stack()) {
 1012     if (dst.first()->is_stack()) {
 1013       // stack to stack
 1014       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1015       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1016     } else {
 1017       // stack to reg
 1018       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1019     }
 1020   } else if (dst.first()->is_stack()) {
 1021     // reg to stack
 1022     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1023   } else {
 1024     if (dst.first() != src.first()) {
 1025       movq(dst.first()->as_Register(), src.first()->as_Register());
 1026     }
 1027   }
 1028 }
 1029 
 1030 // An oop arg. Must pass a handle not the oop itself
 1031 void MacroAssembler::object_move(OopMap* map,
 1032                         int oop_handle_offset,
 1033                         int framesize_in_slots,
 1034                         VMRegPair src,
 1035                         VMRegPair dst,
 1036                         bool is_receiver,
 1037                         int* receiver_offset) {
 1038 
 1039   // must pass a handle. First figure out the location we use as a handle
 1040 
 1041   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1042 
 1043   // See if oop is null if it is we need no handle
 1044 
 1045   if (src.first()->is_stack()) {
 1046 
 1047     // Oop is already on the stack as an argument
 1048     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1049     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1050     if (is_receiver) {
 1051       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1052     }
 1053 
 1054     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1055     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1056     // conditionally move a null
 1057     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1058   } else {
 1059 
 1060     // Oop is in a register we must store it to the space we reserve
 1061     // on the stack for oop_handles and pass a handle if oop is non-null
 1062 
 1063     const Register rOop = src.first()->as_Register();
 1064     int oop_slot;
 1065     if (rOop == j_rarg0)
 1066       oop_slot = 0;
 1067     else if (rOop == j_rarg1)
 1068       oop_slot = 1;
 1069     else if (rOop == j_rarg2)
 1070       oop_slot = 2;
 1071     else if (rOop == j_rarg3)
 1072       oop_slot = 3;
 1073     else if (rOop == j_rarg4)
 1074       oop_slot = 4;
 1075     else {
 1076       assert(rOop == j_rarg5, "wrong register");
 1077       oop_slot = 5;
 1078     }
 1079 
 1080     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1081     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1082 
 1083     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1084     // Store oop in handle area, may be null
 1085     movptr(Address(rsp, offset), rOop);
 1086     if (is_receiver) {
 1087       *receiver_offset = offset;
 1088     }
 1089 
 1090     cmpptr(rOop, NULL_WORD);
 1091     lea(rHandle, Address(rsp, offset));
 1092     // conditionally move a null from the handle area where it was just stored
 1093     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1094   }
 1095 
 1096   // If arg is on the stack then place it otherwise it is already in correct reg.
 1097   if (dst.first()->is_stack()) {
 1098     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1099   }
 1100 }
 1101 
 1102 #endif // _LP64
 1103 
 1104 // Now versions that are common to 32/64 bit
 1105 
 1106 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1107   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1108 }
 1109 
 1110 void MacroAssembler::addptr(Register dst, Register src) {
 1111   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1112 }
 1113 
 1114 void MacroAssembler::addptr(Address dst, Register src) {
 1115   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1116 }
 1117 
 1118 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1119   assert(rscratch != noreg || always_reachable(src), "missing");
 1120 
 1121   if (reachable(src)) {
 1122     Assembler::addsd(dst, as_Address(src));
 1123   } else {
 1124     lea(rscratch, src);
 1125     Assembler::addsd(dst, Address(rscratch, 0));
 1126   }
 1127 }
 1128 
 1129 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1130   assert(rscratch != noreg || always_reachable(src), "missing");
 1131 
 1132   if (reachable(src)) {
 1133     addss(dst, as_Address(src));
 1134   } else {
 1135     lea(rscratch, src);
 1136     addss(dst, Address(rscratch, 0));
 1137   }
 1138 }
 1139 
 1140 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1141   assert(rscratch != noreg || always_reachable(src), "missing");
 1142 
 1143   if (reachable(src)) {
 1144     Assembler::addpd(dst, as_Address(src));
 1145   } else {
 1146     lea(rscratch, src);
 1147     Assembler::addpd(dst, Address(rscratch, 0));
 1148   }
 1149 }
 1150 
 1151 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1152 // Stub code is generated once and never copied.
 1153 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1154 void MacroAssembler::align64() {
 1155   align(64, (unsigned long long) pc());
 1156 }
 1157 
 1158 void MacroAssembler::align32() {
 1159   align(32, (unsigned long long) pc());
 1160 }
 1161 
 1162 void MacroAssembler::align(int modulus) {
 1163   // 8273459: Ensure alignment is possible with current segment alignment
 1164   assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1165   align(modulus, offset());
 1166 }
 1167 
 1168 void MacroAssembler::align(int modulus, int target) {
 1169   if (target % modulus != 0) {
 1170     nop(modulus - (target % modulus));
 1171   }
 1172 }
 1173 
 1174 void MacroAssembler::push_f(XMMRegister r) {
 1175   subptr(rsp, wordSize);
 1176   movflt(Address(rsp, 0), r);
 1177 }
 1178 
 1179 void MacroAssembler::pop_f(XMMRegister r) {
 1180   movflt(r, Address(rsp, 0));
 1181   addptr(rsp, wordSize);
 1182 }
 1183 
 1184 void MacroAssembler::push_d(XMMRegister r) {
 1185   subptr(rsp, 2 * wordSize);
 1186   movdbl(Address(rsp, 0), r);
 1187 }
 1188 
 1189 void MacroAssembler::pop_d(XMMRegister r) {
 1190   movdbl(r, Address(rsp, 0));
 1191   addptr(rsp, 2 * Interpreter::stackElementSize);
 1192 }
 1193 
 1194 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1195   // Used in sign-masking with aligned address.
 1196   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1197   assert(rscratch != noreg || always_reachable(src), "missing");
 1198 
 1199   if (reachable(src)) {
 1200     Assembler::andpd(dst, as_Address(src));
 1201   } else {
 1202     lea(rscratch, src);
 1203     Assembler::andpd(dst, Address(rscratch, 0));
 1204   }
 1205 }
 1206 
 1207 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1208   // Used in sign-masking with aligned address.
 1209   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1210   assert(rscratch != noreg || always_reachable(src), "missing");
 1211 
 1212   if (reachable(src)) {
 1213     Assembler::andps(dst, as_Address(src));
 1214   } else {
 1215     lea(rscratch, src);
 1216     Assembler::andps(dst, Address(rscratch, 0));
 1217   }
 1218 }
 1219 
 1220 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1221   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1222 }
 1223 
 1224 #ifdef _LP64
 1225 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1226   assert(rscratch != noreg || always_reachable(src), "missing");
 1227 
 1228   if (reachable(src)) {
 1229     andq(dst, as_Address(src));
 1230   } else {
 1231     lea(rscratch, src);
 1232     andq(dst, Address(rscratch, 0));
 1233   }
 1234 }
 1235 #endif
 1236 
 1237 void MacroAssembler::atomic_incl(Address counter_addr) {
 1238   lock();
 1239   incrementl(counter_addr);
 1240 }
 1241 
 1242 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1243   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1244 
 1245   if (reachable(counter_addr)) {
 1246     atomic_incl(as_Address(counter_addr));
 1247   } else {
 1248     lea(rscratch, counter_addr);
 1249     atomic_incl(Address(rscratch, 0));
 1250   }
 1251 }
 1252 
 1253 #ifdef _LP64
 1254 void MacroAssembler::atomic_incq(Address counter_addr) {
 1255   lock();
 1256   incrementq(counter_addr);
 1257 }
 1258 
 1259 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1260   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1261 
 1262   if (reachable(counter_addr)) {
 1263     atomic_incq(as_Address(counter_addr));
 1264   } else {
 1265     lea(rscratch, counter_addr);
 1266     atomic_incq(Address(rscratch, 0));
 1267   }
 1268 }
 1269 #endif
 1270 
 1271 // Writes to stack successive pages until offset reached to check for
 1272 // stack overflow + shadow pages.  This clobbers tmp.
 1273 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1274   movptr(tmp, rsp);
 1275   // Bang stack for total size given plus shadow page size.
 1276   // Bang one page at a time because large size can bang beyond yellow and
 1277   // red zones.
 1278   Label loop;
 1279   bind(loop);
 1280   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1281   subptr(tmp, (int)os::vm_page_size());
 1282   subl(size, (int)os::vm_page_size());
 1283   jcc(Assembler::greater, loop);
 1284 
 1285   // Bang down shadow pages too.
 1286   // At this point, (tmp-0) is the last address touched, so don't
 1287   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1288   // was post-decremented.)  Skip this address by starting at i=1, and
 1289   // touch a few more pages below.  N.B.  It is important to touch all
 1290   // the way down including all pages in the shadow zone.
 1291   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1292     // this could be any sized move but this is can be a debugging crumb
 1293     // so the bigger the better.
 1294     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1295   }
 1296 }
 1297 
 1298 void MacroAssembler::reserved_stack_check() {
 1299   // testing if reserved zone needs to be enabled
 1300   Label no_reserved_zone_enabling;
 1301   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1302   NOT_LP64(get_thread(rsi);)
 1303 
 1304   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1305   jcc(Assembler::below, no_reserved_zone_enabling);
 1306 
 1307   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1308   jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
 1309   should_not_reach_here();
 1310 
 1311   bind(no_reserved_zone_enabling);
 1312 }
 1313 
 1314 void MacroAssembler::c2bool(Register x) {
 1315   // implements x == 0 ? 0 : 1
 1316   // note: must only look at least-significant byte of x
 1317   //       since C-style booleans are stored in one byte
 1318   //       only! (was bug)
 1319   andl(x, 0xFF);
 1320   setb(Assembler::notZero, x);
 1321 }
 1322 
 1323 // Wouldn't need if AddressLiteral version had new name
 1324 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1325   Assembler::call(L, rtype);
 1326 }
 1327 
 1328 void MacroAssembler::call(Register entry) {
 1329   Assembler::call(entry);
 1330 }
 1331 
 1332 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1333   assert(rscratch != noreg || always_reachable(entry), "missing");
 1334 
 1335   if (reachable(entry)) {
 1336     Assembler::call_literal(entry.target(), entry.rspec());
 1337   } else {
 1338     lea(rscratch, entry);
 1339     Assembler::call(rscratch);
 1340   }
 1341 }
 1342 
 1343 void MacroAssembler::ic_call(address entry, jint method_index) {
 1344   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1345 #ifdef _LP64
 1346   // Needs full 64-bit immediate for later patching.
 1347   mov64(rax, (int64_t)Universe::non_oop_word());
 1348 #else
 1349   movptr(rax, (intptr_t)Universe::non_oop_word());
 1350 #endif
 1351   call(AddressLiteral(entry, rh));
 1352 }
 1353 
 1354 int MacroAssembler::ic_check_size() {
 1355   return LP64_ONLY(14) NOT_LP64(12);
 1356 }
 1357 
 1358 int MacroAssembler::ic_check(int end_alignment) {
 1359   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1360   Register data = rax;
 1361   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1362 
 1363   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1364   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1365   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1366   // before the inline cache check here, and not after
 1367   align(end_alignment, offset() + ic_check_size());
 1368 
 1369   int uep_offset = offset();
 1370 
 1371   if (UseCompressedClassPointers) {
 1372     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1373     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1374   } else {
 1375     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1376     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1377   }
 1378 
 1379   // if inline cache check fails, then jump to runtime routine
 1380   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1381   assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
 1382 
 1383   return uep_offset;
 1384 }
 1385 
 1386 void MacroAssembler::emit_static_call_stub() {
 1387   // Static stub relocation also tags the Method* in the code-stream.
 1388   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1389   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1390   jump(RuntimeAddress(pc()));
 1391 }
 1392 
 1393 // Implementation of call_VM versions
 1394 
 1395 void MacroAssembler::call_VM(Register oop_result,
 1396                              address entry_point,
 1397                              bool check_exceptions) {
 1398   Label C, E;
 1399   call(C, relocInfo::none);
 1400   jmp(E);
 1401 
 1402   bind(C);
 1403   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1404   ret(0);
 1405 
 1406   bind(E);
 1407 }
 1408 
 1409 void MacroAssembler::call_VM(Register oop_result,
 1410                              address entry_point,
 1411                              Register arg_1,
 1412                              bool check_exceptions) {
 1413   Label C, E;
 1414   call(C, relocInfo::none);
 1415   jmp(E);
 1416 
 1417   bind(C);
 1418   pass_arg1(this, arg_1);
 1419   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1420   ret(0);
 1421 
 1422   bind(E);
 1423 }
 1424 
 1425 void MacroAssembler::call_VM(Register oop_result,
 1426                              address entry_point,
 1427                              Register arg_1,
 1428                              Register arg_2,
 1429                              bool check_exceptions) {
 1430   Label C, E;
 1431   call(C, relocInfo::none);
 1432   jmp(E);
 1433 
 1434   bind(C);
 1435 
 1436   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1437 
 1438   pass_arg2(this, arg_2);
 1439   pass_arg1(this, arg_1);
 1440   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1441   ret(0);
 1442 
 1443   bind(E);
 1444 }
 1445 
 1446 void MacroAssembler::call_VM(Register oop_result,
 1447                              address entry_point,
 1448                              Register arg_1,
 1449                              Register arg_2,
 1450                              Register arg_3,
 1451                              bool check_exceptions) {
 1452   Label C, E;
 1453   call(C, relocInfo::none);
 1454   jmp(E);
 1455 
 1456   bind(C);
 1457 
 1458   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1459   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1460   pass_arg3(this, arg_3);
 1461   pass_arg2(this, arg_2);
 1462   pass_arg1(this, arg_1);
 1463   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1464   ret(0);
 1465 
 1466   bind(E);
 1467 }
 1468 
 1469 void MacroAssembler::call_VM(Register oop_result,
 1470                              Register last_java_sp,
 1471                              address entry_point,
 1472                              int number_of_arguments,
 1473                              bool check_exceptions) {
 1474   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1475   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1476 }
 1477 
 1478 void MacroAssembler::call_VM(Register oop_result,
 1479                              Register last_java_sp,
 1480                              address entry_point,
 1481                              Register arg_1,
 1482                              bool check_exceptions) {
 1483   pass_arg1(this, arg_1);
 1484   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1485 }
 1486 
 1487 void MacroAssembler::call_VM(Register oop_result,
 1488                              Register last_java_sp,
 1489                              address entry_point,
 1490                              Register arg_1,
 1491                              Register arg_2,
 1492                              bool check_exceptions) {
 1493 
 1494   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1495   pass_arg2(this, arg_2);
 1496   pass_arg1(this, arg_1);
 1497   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1498 }
 1499 
 1500 void MacroAssembler::call_VM(Register oop_result,
 1501                              Register last_java_sp,
 1502                              address entry_point,
 1503                              Register arg_1,
 1504                              Register arg_2,
 1505                              Register arg_3,
 1506                              bool check_exceptions) {
 1507   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1508   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1509   pass_arg3(this, arg_3);
 1510   pass_arg2(this, arg_2);
 1511   pass_arg1(this, arg_1);
 1512   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1513 }
 1514 
 1515 void MacroAssembler::super_call_VM(Register oop_result,
 1516                                    Register last_java_sp,
 1517                                    address entry_point,
 1518                                    int number_of_arguments,
 1519                                    bool check_exceptions) {
 1520   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1521   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1522 }
 1523 
 1524 void MacroAssembler::super_call_VM(Register oop_result,
 1525                                    Register last_java_sp,
 1526                                    address entry_point,
 1527                                    Register arg_1,
 1528                                    bool check_exceptions) {
 1529   pass_arg1(this, arg_1);
 1530   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1531 }
 1532 
 1533 void MacroAssembler::super_call_VM(Register oop_result,
 1534                                    Register last_java_sp,
 1535                                    address entry_point,
 1536                                    Register arg_1,
 1537                                    Register arg_2,
 1538                                    bool check_exceptions) {
 1539 
 1540   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1541   pass_arg2(this, arg_2);
 1542   pass_arg1(this, arg_1);
 1543   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1544 }
 1545 
 1546 void MacroAssembler::super_call_VM(Register oop_result,
 1547                                    Register last_java_sp,
 1548                                    address entry_point,
 1549                                    Register arg_1,
 1550                                    Register arg_2,
 1551                                    Register arg_3,
 1552                                    bool check_exceptions) {
 1553   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1554   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1555   pass_arg3(this, arg_3);
 1556   pass_arg2(this, arg_2);
 1557   pass_arg1(this, arg_1);
 1558   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1559 }
 1560 
 1561 void MacroAssembler::call_VM_base(Register oop_result,
 1562                                   Register java_thread,
 1563                                   Register last_java_sp,
 1564                                   address  entry_point,
 1565                                   int      number_of_arguments,
 1566                                   bool     check_exceptions) {
 1567   // determine java_thread register
 1568   if (!java_thread->is_valid()) {
 1569 #ifdef _LP64
 1570     java_thread = r15_thread;
 1571 #else
 1572     java_thread = rdi;
 1573     get_thread(java_thread);
 1574 #endif // LP64
 1575   }
 1576   // determine last_java_sp register
 1577   if (!last_java_sp->is_valid()) {
 1578     last_java_sp = rsp;
 1579   }
 1580   // debugging support
 1581   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1582   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1583 #ifdef ASSERT
 1584   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1585   // r12 is the heapbase.
 1586   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1587 #endif // ASSERT
 1588 
 1589   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1590   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1591 
 1592   // push java thread (becomes first argument of C function)
 1593 
 1594   NOT_LP64(push(java_thread); number_of_arguments++);
 1595   LP64_ONLY(mov(c_rarg0, r15_thread));
 1596 
 1597   // set last Java frame before call
 1598   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1599 
 1600   // Only interpreter should have to set fp
 1601   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1602 
 1603   // do the call, remove parameters
 1604   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1605 
 1606   // restore the thread (cannot use the pushed argument since arguments
 1607   // may be overwritten by C code generated by an optimizing compiler);
 1608   // however can use the register value directly if it is callee saved.
 1609   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1610     // rdi & rsi (also r15) are callee saved -> nothing to do
 1611 #ifdef ASSERT
 1612     guarantee(java_thread != rax, "change this code");
 1613     push(rax);
 1614     { Label L;
 1615       get_thread(rax);
 1616       cmpptr(java_thread, rax);
 1617       jcc(Assembler::equal, L);
 1618       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1619       bind(L);
 1620     }
 1621     pop(rax);
 1622 #endif
 1623   } else {
 1624     get_thread(java_thread);
 1625   }
 1626   // reset last Java frame
 1627   // Only interpreter should have to clear fp
 1628   reset_last_Java_frame(java_thread, true);
 1629 
 1630    // C++ interp handles this in the interpreter
 1631   check_and_handle_popframe(java_thread);
 1632   check_and_handle_earlyret(java_thread);
 1633 
 1634   if (check_exceptions) {
 1635     // check for pending exceptions (java_thread is set upon return)
 1636     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1637 #ifndef _LP64
 1638     jump_cc(Assembler::notEqual,
 1639             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1640 #else
 1641     // This used to conditionally jump to forward_exception however it is
 1642     // possible if we relocate that the branch will not reach. So we must jump
 1643     // around so we can always reach
 1644 
 1645     Label ok;
 1646     jcc(Assembler::equal, ok);
 1647     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1648     bind(ok);
 1649 #endif // LP64
 1650   }
 1651 
 1652   // get oop result if there is one and reset the value in the thread
 1653   if (oop_result->is_valid()) {
 1654     get_vm_result(oop_result, java_thread);
 1655   }
 1656 }
 1657 
 1658 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1659 
 1660   // Calculate the value for last_Java_sp
 1661   // somewhat subtle. call_VM does an intermediate call
 1662   // which places a return address on the stack just under the
 1663   // stack pointer as the user finished with it. This allows
 1664   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1665   // On 32bit we then have to push additional args on the stack to accomplish
 1666   // the actual requested call. On 64bit call_VM only can use register args
 1667   // so the only extra space is the return address that call_VM created.
 1668   // This hopefully explains the calculations here.
 1669 
 1670 #ifdef _LP64
 1671   // We've pushed one address, correct last_Java_sp
 1672   lea(rax, Address(rsp, wordSize));
 1673 #else
 1674   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1675 #endif // LP64
 1676 
 1677   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1678 
 1679 }
 1680 
 1681 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1682 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1683   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1684 }
 1685 
 1686 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1687   call_VM_leaf_base(entry_point, number_of_arguments);
 1688 }
 1689 
 1690 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1691   pass_arg0(this, arg_0);
 1692   call_VM_leaf(entry_point, 1);
 1693 }
 1694 
 1695 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1696 
 1697   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1698   pass_arg1(this, arg_1);
 1699   pass_arg0(this, arg_0);
 1700   call_VM_leaf(entry_point, 2);
 1701 }
 1702 
 1703 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1704   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1705   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1706   pass_arg2(this, arg_2);
 1707   pass_arg1(this, arg_1);
 1708   pass_arg0(this, arg_0);
 1709   call_VM_leaf(entry_point, 3);
 1710 }
 1711 
 1712 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1713   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1714   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1715   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1716   pass_arg3(this, arg_3);
 1717   pass_arg2(this, arg_2);
 1718   pass_arg1(this, arg_1);
 1719   pass_arg0(this, arg_0);
 1720   call_VM_leaf(entry_point, 3);
 1721 }
 1722 
 1723 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1724   pass_arg0(this, arg_0);
 1725   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1726 }
 1727 
 1728 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1729   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1730   pass_arg1(this, arg_1);
 1731   pass_arg0(this, arg_0);
 1732   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1733 }
 1734 
 1735 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1736   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1737   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1738   pass_arg2(this, arg_2);
 1739   pass_arg1(this, arg_1);
 1740   pass_arg0(this, arg_0);
 1741   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1742 }
 1743 
 1744 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1745   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1746   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1747   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1748   pass_arg3(this, arg_3);
 1749   pass_arg2(this, arg_2);
 1750   pass_arg1(this, arg_1);
 1751   pass_arg0(this, arg_0);
 1752   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1753 }
 1754 
 1755 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1756   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1757   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1758   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1759 }
 1760 
 1761 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1762   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1763   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1764 }
 1765 
 1766 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1767 }
 1768 
 1769 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1770 }
 1771 
 1772 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1773   assert(rscratch != noreg || always_reachable(src1), "missing");
 1774 
 1775   if (reachable(src1)) {
 1776     cmpl(as_Address(src1), imm);
 1777   } else {
 1778     lea(rscratch, src1);
 1779     cmpl(Address(rscratch, 0), imm);
 1780   }
 1781 }
 1782 
 1783 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1784   assert(!src2.is_lval(), "use cmpptr");
 1785   assert(rscratch != noreg || always_reachable(src2), "missing");
 1786 
 1787   if (reachable(src2)) {
 1788     cmpl(src1, as_Address(src2));
 1789   } else {
 1790     lea(rscratch, src2);
 1791     cmpl(src1, Address(rscratch, 0));
 1792   }
 1793 }
 1794 
 1795 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1796   Assembler::cmpl(src1, imm);
 1797 }
 1798 
 1799 void MacroAssembler::cmp32(Register src1, Address src2) {
 1800   Assembler::cmpl(src1, src2);
 1801 }
 1802 
 1803 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1804   ucomisd(opr1, opr2);
 1805 
 1806   Label L;
 1807   if (unordered_is_less) {
 1808     movl(dst, -1);
 1809     jcc(Assembler::parity, L);
 1810     jcc(Assembler::below , L);
 1811     movl(dst, 0);
 1812     jcc(Assembler::equal , L);
 1813     increment(dst);
 1814   } else { // unordered is greater
 1815     movl(dst, 1);
 1816     jcc(Assembler::parity, L);
 1817     jcc(Assembler::above , L);
 1818     movl(dst, 0);
 1819     jcc(Assembler::equal , L);
 1820     decrementl(dst);
 1821   }
 1822   bind(L);
 1823 }
 1824 
 1825 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1826   ucomiss(opr1, opr2);
 1827 
 1828   Label L;
 1829   if (unordered_is_less) {
 1830     movl(dst, -1);
 1831     jcc(Assembler::parity, L);
 1832     jcc(Assembler::below , L);
 1833     movl(dst, 0);
 1834     jcc(Assembler::equal , L);
 1835     increment(dst);
 1836   } else { // unordered is greater
 1837     movl(dst, 1);
 1838     jcc(Assembler::parity, L);
 1839     jcc(Assembler::above , L);
 1840     movl(dst, 0);
 1841     jcc(Assembler::equal , L);
 1842     decrementl(dst);
 1843   }
 1844   bind(L);
 1845 }
 1846 
 1847 
 1848 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1849   assert(rscratch != noreg || always_reachable(src1), "missing");
 1850 
 1851   if (reachable(src1)) {
 1852     cmpb(as_Address(src1), imm);
 1853   } else {
 1854     lea(rscratch, src1);
 1855     cmpb(Address(rscratch, 0), imm);
 1856   }
 1857 }
 1858 
 1859 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1860 #ifdef _LP64
 1861   assert(rscratch != noreg || always_reachable(src2), "missing");
 1862 
 1863   if (src2.is_lval()) {
 1864     movptr(rscratch, src2);
 1865     Assembler::cmpq(src1, rscratch);
 1866   } else if (reachable(src2)) {
 1867     cmpq(src1, as_Address(src2));
 1868   } else {
 1869     lea(rscratch, src2);
 1870     Assembler::cmpq(src1, Address(rscratch, 0));
 1871   }
 1872 #else
 1873   assert(rscratch == noreg, "not needed");
 1874   if (src2.is_lval()) {
 1875     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1876   } else {
 1877     cmpl(src1, as_Address(src2));
 1878   }
 1879 #endif // _LP64
 1880 }
 1881 
 1882 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1883   assert(src2.is_lval(), "not a mem-mem compare");
 1884 #ifdef _LP64
 1885   // moves src2's literal address
 1886   movptr(rscratch, src2);
 1887   Assembler::cmpq(src1, rscratch);
 1888 #else
 1889   assert(rscratch == noreg, "not needed");
 1890   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1891 #endif // _LP64
 1892 }
 1893 
 1894 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1895   cmpptr(src1, src2);
 1896 }
 1897 
 1898 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1899   cmpptr(src1, src2);
 1900 }
 1901 
 1902 #ifdef _LP64
 1903 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1904   movoop(rscratch, src2);
 1905   cmpptr(src1, rscratch);
 1906 }
 1907 #endif
 1908 
 1909 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1910   assert(rscratch != noreg || always_reachable(adr), "missing");
 1911 
 1912   if (reachable(adr)) {
 1913     lock();
 1914     cmpxchgptr(reg, as_Address(adr));
 1915   } else {
 1916     lea(rscratch, adr);
 1917     lock();
 1918     cmpxchgptr(reg, Address(rscratch, 0));
 1919   }
 1920 }
 1921 
 1922 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1923   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1924 }
 1925 
 1926 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1927   assert(rscratch != noreg || always_reachable(src), "missing");
 1928 
 1929   if (reachable(src)) {
 1930     Assembler::comisd(dst, as_Address(src));
 1931   } else {
 1932     lea(rscratch, src);
 1933     Assembler::comisd(dst, Address(rscratch, 0));
 1934   }
 1935 }
 1936 
 1937 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1938   assert(rscratch != noreg || always_reachable(src), "missing");
 1939 
 1940   if (reachable(src)) {
 1941     Assembler::comiss(dst, as_Address(src));
 1942   } else {
 1943     lea(rscratch, src);
 1944     Assembler::comiss(dst, Address(rscratch, 0));
 1945   }
 1946 }
 1947 
 1948 
 1949 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1950   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1951 
 1952   Condition negated_cond = negate_condition(cond);
 1953   Label L;
 1954   jcc(negated_cond, L);
 1955   pushf(); // Preserve flags
 1956   atomic_incl(counter_addr, rscratch);
 1957   popf();
 1958   bind(L);
 1959 }
 1960 
 1961 int MacroAssembler::corrected_idivl(Register reg) {
 1962   // Full implementation of Java idiv and irem; checks for
 1963   // special case as described in JVM spec., p.243 & p.271.
 1964   // The function returns the (pc) offset of the idivl
 1965   // instruction - may be needed for implicit exceptions.
 1966   //
 1967   //         normal case                           special case
 1968   //
 1969   // input : rax,: dividend                         min_int
 1970   //         reg: divisor   (may not be rax,/rdx)   -1
 1971   //
 1972   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1973   //         rdx: remainder (= rax, irem reg)       0
 1974   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1975   const int min_int = 0x80000000;
 1976   Label normal_case, special_case;
 1977 
 1978   // check for special case
 1979   cmpl(rax, min_int);
 1980   jcc(Assembler::notEqual, normal_case);
 1981   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1982   cmpl(reg, -1);
 1983   jcc(Assembler::equal, special_case);
 1984 
 1985   // handle normal case
 1986   bind(normal_case);
 1987   cdql();
 1988   int idivl_offset = offset();
 1989   idivl(reg);
 1990 
 1991   // normal and special case exit
 1992   bind(special_case);
 1993 
 1994   return idivl_offset;
 1995 }
 1996 
 1997 
 1998 
 1999 void MacroAssembler::decrementl(Register reg, int value) {
 2000   if (value == min_jint) {subl(reg, value) ; return; }
 2001   if (value <  0) { incrementl(reg, -value); return; }
 2002   if (value == 0) {                        ; return; }
 2003   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2004   /* else */      { subl(reg, value)       ; return; }
 2005 }
 2006 
 2007 void MacroAssembler::decrementl(Address dst, int value) {
 2008   if (value == min_jint) {subl(dst, value) ; return; }
 2009   if (value <  0) { incrementl(dst, -value); return; }
 2010   if (value == 0) {                        ; return; }
 2011   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2012   /* else */      { subl(dst, value)       ; return; }
 2013 }
 2014 
 2015 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2016   assert(shift_value > 0, "illegal shift value");
 2017   Label _is_positive;
 2018   testl (reg, reg);
 2019   jcc (Assembler::positive, _is_positive);
 2020   int offset = (1 << shift_value) - 1 ;
 2021 
 2022   if (offset == 1) {
 2023     incrementl(reg);
 2024   } else {
 2025     addl(reg, offset);
 2026   }
 2027 
 2028   bind (_is_positive);
 2029   sarl(reg, shift_value);
 2030 }
 2031 
 2032 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2033   assert(rscratch != noreg || always_reachable(src), "missing");
 2034 
 2035   if (reachable(src)) {
 2036     Assembler::divsd(dst, as_Address(src));
 2037   } else {
 2038     lea(rscratch, src);
 2039     Assembler::divsd(dst, Address(rscratch, 0));
 2040   }
 2041 }
 2042 
 2043 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2044   assert(rscratch != noreg || always_reachable(src), "missing");
 2045 
 2046   if (reachable(src)) {
 2047     Assembler::divss(dst, as_Address(src));
 2048   } else {
 2049     lea(rscratch, src);
 2050     Assembler::divss(dst, Address(rscratch, 0));
 2051   }
 2052 }
 2053 
 2054 void MacroAssembler::enter() {
 2055   push(rbp);
 2056   mov(rbp, rsp);
 2057 }
 2058 
 2059 void MacroAssembler::post_call_nop() {
 2060   if (!Continuations::enabled()) {
 2061     return;
 2062   }
 2063   InstructionMark im(this);
 2064   relocate(post_call_nop_Relocation::spec());
 2065   InlineSkippedInstructionsCounter skipCounter(this);
 2066   emit_int8((uint8_t)0x0f);
 2067   emit_int8((uint8_t)0x1f);
 2068   emit_int8((uint8_t)0x84);
 2069   emit_int8((uint8_t)0x00);
 2070   emit_int32(0x00);
 2071 }
 2072 
 2073 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2074 void MacroAssembler::fat_nop() {
 2075   if (UseAddressNop) {
 2076     addr_nop_5();
 2077   } else {
 2078     emit_int8((uint8_t)0x26); // es:
 2079     emit_int8((uint8_t)0x2e); // cs:
 2080     emit_int8((uint8_t)0x64); // fs:
 2081     emit_int8((uint8_t)0x65); // gs:
 2082     emit_int8((uint8_t)0x90);
 2083   }
 2084 }
 2085 
 2086 #ifndef _LP64
 2087 void MacroAssembler::fcmp(Register tmp) {
 2088   fcmp(tmp, 1, true, true);
 2089 }
 2090 
 2091 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2092   assert(!pop_right || pop_left, "usage error");
 2093   if (VM_Version::supports_cmov()) {
 2094     assert(tmp == noreg, "unneeded temp");
 2095     if (pop_left) {
 2096       fucomip(index);
 2097     } else {
 2098       fucomi(index);
 2099     }
 2100     if (pop_right) {
 2101       fpop();
 2102     }
 2103   } else {
 2104     assert(tmp != noreg, "need temp");
 2105     if (pop_left) {
 2106       if (pop_right) {
 2107         fcompp();
 2108       } else {
 2109         fcomp(index);
 2110       }
 2111     } else {
 2112       fcom(index);
 2113     }
 2114     // convert FPU condition into eflags condition via rax,
 2115     save_rax(tmp);
 2116     fwait(); fnstsw_ax();
 2117     sahf();
 2118     restore_rax(tmp);
 2119   }
 2120   // condition codes set as follows:
 2121   //
 2122   // CF (corresponds to C0) if x < y
 2123   // PF (corresponds to C2) if unordered
 2124   // ZF (corresponds to C3) if x = y
 2125 }
 2126 
 2127 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2128   fcmp2int(dst, unordered_is_less, 1, true, true);
 2129 }
 2130 
 2131 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2132   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2133   Label L;
 2134   if (unordered_is_less) {
 2135     movl(dst, -1);
 2136     jcc(Assembler::parity, L);
 2137     jcc(Assembler::below , L);
 2138     movl(dst, 0);
 2139     jcc(Assembler::equal , L);
 2140     increment(dst);
 2141   } else { // unordered is greater
 2142     movl(dst, 1);
 2143     jcc(Assembler::parity, L);
 2144     jcc(Assembler::above , L);
 2145     movl(dst, 0);
 2146     jcc(Assembler::equal , L);
 2147     decrementl(dst);
 2148   }
 2149   bind(L);
 2150 }
 2151 
 2152 void MacroAssembler::fld_d(AddressLiteral src) {
 2153   fld_d(as_Address(src));
 2154 }
 2155 
 2156 void MacroAssembler::fld_s(AddressLiteral src) {
 2157   fld_s(as_Address(src));
 2158 }
 2159 
 2160 void MacroAssembler::fldcw(AddressLiteral src) {
 2161   fldcw(as_Address(src));
 2162 }
 2163 
 2164 void MacroAssembler::fpop() {
 2165   ffree();
 2166   fincstp();
 2167 }
 2168 
 2169 void MacroAssembler::fremr(Register tmp) {
 2170   save_rax(tmp);
 2171   { Label L;
 2172     bind(L);
 2173     fprem();
 2174     fwait(); fnstsw_ax();
 2175     sahf();
 2176     jcc(Assembler::parity, L);
 2177   }
 2178   restore_rax(tmp);
 2179   // Result is in ST0.
 2180   // Note: fxch & fpop to get rid of ST1
 2181   // (otherwise FPU stack could overflow eventually)
 2182   fxch(1);
 2183   fpop();
 2184 }
 2185 
 2186 void MacroAssembler::empty_FPU_stack() {
 2187   if (VM_Version::supports_mmx()) {
 2188     emms();
 2189   } else {
 2190     for (int i = 8; i-- > 0; ) ffree(i);
 2191   }
 2192 }
 2193 #endif // !LP64
 2194 
 2195 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2196   assert(rscratch != noreg || always_reachable(src), "missing");
 2197   if (reachable(src)) {
 2198     Assembler::mulpd(dst, as_Address(src));
 2199   } else {
 2200     lea(rscratch, src);
 2201     Assembler::mulpd(dst, Address(rscratch, 0));
 2202   }
 2203 }
 2204 
 2205 void MacroAssembler::load_float(Address src) {
 2206 #ifdef _LP64
 2207   movflt(xmm0, src);
 2208 #else
 2209   if (UseSSE >= 1) {
 2210     movflt(xmm0, src);
 2211   } else {
 2212     fld_s(src);
 2213   }
 2214 #endif // LP64
 2215 }
 2216 
 2217 void MacroAssembler::store_float(Address dst) {
 2218 #ifdef _LP64
 2219   movflt(dst, xmm0);
 2220 #else
 2221   if (UseSSE >= 1) {
 2222     movflt(dst, xmm0);
 2223   } else {
 2224     fstp_s(dst);
 2225   }
 2226 #endif // LP64
 2227 }
 2228 
 2229 void MacroAssembler::load_double(Address src) {
 2230 #ifdef _LP64
 2231   movdbl(xmm0, src);
 2232 #else
 2233   if (UseSSE >= 2) {
 2234     movdbl(xmm0, src);
 2235   } else {
 2236     fld_d(src);
 2237   }
 2238 #endif // LP64
 2239 }
 2240 
 2241 void MacroAssembler::store_double(Address dst) {
 2242 #ifdef _LP64
 2243   movdbl(dst, xmm0);
 2244 #else
 2245   if (UseSSE >= 2) {
 2246     movdbl(dst, xmm0);
 2247   } else {
 2248     fstp_d(dst);
 2249   }
 2250 #endif // LP64
 2251 }
 2252 
 2253 // dst = c = a * b + c
 2254 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2255   Assembler::vfmadd231sd(c, a, b);
 2256   if (dst != c) {
 2257     movdbl(dst, c);
 2258   }
 2259 }
 2260 
 2261 // dst = c = a * b + c
 2262 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2263   Assembler::vfmadd231ss(c, a, b);
 2264   if (dst != c) {
 2265     movflt(dst, c);
 2266   }
 2267 }
 2268 
 2269 // dst = c = a * b + c
 2270 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2271   Assembler::vfmadd231pd(c, a, b, vector_len);
 2272   if (dst != c) {
 2273     vmovdqu(dst, c);
 2274   }
 2275 }
 2276 
 2277 // dst = c = a * b + c
 2278 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2279   Assembler::vfmadd231ps(c, a, b, vector_len);
 2280   if (dst != c) {
 2281     vmovdqu(dst, c);
 2282   }
 2283 }
 2284 
 2285 // dst = c = a * b + c
 2286 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2287   Assembler::vfmadd231pd(c, a, b, vector_len);
 2288   if (dst != c) {
 2289     vmovdqu(dst, c);
 2290   }
 2291 }
 2292 
 2293 // dst = c = a * b + c
 2294 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2295   Assembler::vfmadd231ps(c, a, b, vector_len);
 2296   if (dst != c) {
 2297     vmovdqu(dst, c);
 2298   }
 2299 }
 2300 
 2301 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2302   assert(rscratch != noreg || always_reachable(dst), "missing");
 2303 
 2304   if (reachable(dst)) {
 2305     incrementl(as_Address(dst));
 2306   } else {
 2307     lea(rscratch, dst);
 2308     incrementl(Address(rscratch, 0));
 2309   }
 2310 }
 2311 
 2312 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2313   incrementl(as_Address(dst, rscratch));
 2314 }
 2315 
 2316 void MacroAssembler::incrementl(Register reg, int value) {
 2317   if (value == min_jint) {addl(reg, value) ; return; }
 2318   if (value <  0) { decrementl(reg, -value); return; }
 2319   if (value == 0) {                        ; return; }
 2320   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2321   /* else */      { addl(reg, value)       ; return; }
 2322 }
 2323 
 2324 void MacroAssembler::incrementl(Address dst, int value) {
 2325   if (value == min_jint) {addl(dst, value) ; return; }
 2326   if (value <  0) { decrementl(dst, -value); return; }
 2327   if (value == 0) {                        ; return; }
 2328   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2329   /* else */      { addl(dst, value)       ; return; }
 2330 }
 2331 
 2332 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2333   assert(rscratch != noreg || always_reachable(dst), "missing");
 2334 
 2335   if (reachable(dst)) {
 2336     jmp_literal(dst.target(), dst.rspec());
 2337   } else {
 2338     lea(rscratch, dst);
 2339     jmp(rscratch);
 2340   }
 2341 }
 2342 
 2343 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2344   assert(rscratch != noreg || always_reachable(dst), "missing");
 2345 
 2346   if (reachable(dst)) {
 2347     InstructionMark im(this);
 2348     relocate(dst.reloc());
 2349     const int short_size = 2;
 2350     const int long_size = 6;
 2351     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2352     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2353       // 0111 tttn #8-bit disp
 2354       emit_int8(0x70 | cc);
 2355       emit_int8((offs - short_size) & 0xFF);
 2356     } else {
 2357       // 0000 1111 1000 tttn #32-bit disp
 2358       emit_int8(0x0F);
 2359       emit_int8((unsigned char)(0x80 | cc));
 2360       emit_int32(offs - long_size);
 2361     }
 2362   } else {
 2363 #ifdef ASSERT
 2364     warning("reversing conditional branch");
 2365 #endif /* ASSERT */
 2366     Label skip;
 2367     jccb(reverse[cc], skip);
 2368     lea(rscratch, dst);
 2369     Assembler::jmp(rscratch);
 2370     bind(skip);
 2371   }
 2372 }
 2373 
 2374 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2375   assert(rscratch != noreg || always_reachable(src), "missing");
 2376 
 2377   if (reachable(src)) {
 2378     Assembler::ldmxcsr(as_Address(src));
 2379   } else {
 2380     lea(rscratch, src);
 2381     Assembler::ldmxcsr(Address(rscratch, 0));
 2382   }
 2383 }
 2384 
 2385 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2386   int off;
 2387   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2388     off = offset();
 2389     movsbl(dst, src); // movsxb
 2390   } else {
 2391     off = load_unsigned_byte(dst, src);
 2392     shll(dst, 24);
 2393     sarl(dst, 24);
 2394   }
 2395   return off;
 2396 }
 2397 
 2398 // Note: load_signed_short used to be called load_signed_word.
 2399 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2400 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2401 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2402 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2403   int off;
 2404   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2405     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2406     // version but this is what 64bit has always done. This seems to imply
 2407     // that users are only using 32bits worth.
 2408     off = offset();
 2409     movswl(dst, src); // movsxw
 2410   } else {
 2411     off = load_unsigned_short(dst, src);
 2412     shll(dst, 16);
 2413     sarl(dst, 16);
 2414   }
 2415   return off;
 2416 }
 2417 
 2418 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2419   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2420   // and "3.9 Partial Register Penalties", p. 22).
 2421   int off;
 2422   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2423     off = offset();
 2424     movzbl(dst, src); // movzxb
 2425   } else {
 2426     xorl(dst, dst);
 2427     off = offset();
 2428     movb(dst, src);
 2429   }
 2430   return off;
 2431 }
 2432 
 2433 // Note: load_unsigned_short used to be called load_unsigned_word.
 2434 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2435   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2436   // and "3.9 Partial Register Penalties", p. 22).
 2437   int off;
 2438   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2439     off = offset();
 2440     movzwl(dst, src); // movzxw
 2441   } else {
 2442     xorl(dst, dst);
 2443     off = offset();
 2444     movw(dst, src);
 2445   }
 2446   return off;
 2447 }
 2448 
 2449 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2450   switch (size_in_bytes) {
 2451 #ifndef _LP64
 2452   case  8:
 2453     assert(dst2 != noreg, "second dest register required");
 2454     movl(dst,  src);
 2455     movl(dst2, src.plus_disp(BytesPerInt));
 2456     break;
 2457 #else
 2458   case  8:  movq(dst, src); break;
 2459 #endif
 2460   case  4:  movl(dst, src); break;
 2461   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2462   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2463   default:  ShouldNotReachHere();
 2464   }
 2465 }
 2466 
 2467 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2468   switch (size_in_bytes) {
 2469 #ifndef _LP64
 2470   case  8:
 2471     assert(src2 != noreg, "second source register required");
 2472     movl(dst,                        src);
 2473     movl(dst.plus_disp(BytesPerInt), src2);
 2474     break;
 2475 #else
 2476   case  8:  movq(dst, src); break;
 2477 #endif
 2478   case  4:  movl(dst, src); break;
 2479   case  2:  movw(dst, src); break;
 2480   case  1:  movb(dst, src); break;
 2481   default:  ShouldNotReachHere();
 2482   }
 2483 }
 2484 
 2485 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2486   assert(rscratch != noreg || always_reachable(dst), "missing");
 2487 
 2488   if (reachable(dst)) {
 2489     movl(as_Address(dst), src);
 2490   } else {
 2491     lea(rscratch, dst);
 2492     movl(Address(rscratch, 0), src);
 2493   }
 2494 }
 2495 
 2496 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2497   if (reachable(src)) {
 2498     movl(dst, as_Address(src));
 2499   } else {
 2500     lea(dst, src);
 2501     movl(dst, Address(dst, 0));
 2502   }
 2503 }
 2504 
 2505 // C++ bool manipulation
 2506 
 2507 void MacroAssembler::movbool(Register dst, Address src) {
 2508   if(sizeof(bool) == 1)
 2509     movb(dst, src);
 2510   else if(sizeof(bool) == 2)
 2511     movw(dst, src);
 2512   else if(sizeof(bool) == 4)
 2513     movl(dst, src);
 2514   else
 2515     // unsupported
 2516     ShouldNotReachHere();
 2517 }
 2518 
 2519 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2520   if(sizeof(bool) == 1)
 2521     movb(dst, (int) boolconst);
 2522   else if(sizeof(bool) == 2)
 2523     movw(dst, (int) boolconst);
 2524   else if(sizeof(bool) == 4)
 2525     movl(dst, (int) boolconst);
 2526   else
 2527     // unsupported
 2528     ShouldNotReachHere();
 2529 }
 2530 
 2531 void MacroAssembler::movbool(Address dst, Register src) {
 2532   if(sizeof(bool) == 1)
 2533     movb(dst, src);
 2534   else if(sizeof(bool) == 2)
 2535     movw(dst, src);
 2536   else if(sizeof(bool) == 4)
 2537     movl(dst, src);
 2538   else
 2539     // unsupported
 2540     ShouldNotReachHere();
 2541 }
 2542 
 2543 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2544   assert(rscratch != noreg || always_reachable(src), "missing");
 2545 
 2546   if (reachable(src)) {
 2547     movdl(dst, as_Address(src));
 2548   } else {
 2549     lea(rscratch, src);
 2550     movdl(dst, Address(rscratch, 0));
 2551   }
 2552 }
 2553 
 2554 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2555   assert(rscratch != noreg || always_reachable(src), "missing");
 2556 
 2557   if (reachable(src)) {
 2558     movq(dst, as_Address(src));
 2559   } else {
 2560     lea(rscratch, src);
 2561     movq(dst, Address(rscratch, 0));
 2562   }
 2563 }
 2564 
 2565 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2566   assert(rscratch != noreg || always_reachable(src), "missing");
 2567 
 2568   if (reachable(src)) {
 2569     if (UseXmmLoadAndClearUpper) {
 2570       movsd (dst, as_Address(src));
 2571     } else {
 2572       movlpd(dst, as_Address(src));
 2573     }
 2574   } else {
 2575     lea(rscratch, src);
 2576     if (UseXmmLoadAndClearUpper) {
 2577       movsd (dst, Address(rscratch, 0));
 2578     } else {
 2579       movlpd(dst, Address(rscratch, 0));
 2580     }
 2581   }
 2582 }
 2583 
 2584 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2585   assert(rscratch != noreg || always_reachable(src), "missing");
 2586 
 2587   if (reachable(src)) {
 2588     movss(dst, as_Address(src));
 2589   } else {
 2590     lea(rscratch, src);
 2591     movss(dst, Address(rscratch, 0));
 2592   }
 2593 }
 2594 
 2595 void MacroAssembler::movptr(Register dst, Register src) {
 2596   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2597 }
 2598 
 2599 void MacroAssembler::movptr(Register dst, Address src) {
 2600   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2601 }
 2602 
 2603 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2604 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2605 #ifdef _LP64
 2606   if (is_uimm32(src)) {
 2607     movl(dst, checked_cast<uint32_t>(src));
 2608   } else if (is_simm32(src)) {
 2609     movq(dst, checked_cast<int32_t>(src));
 2610   } else {
 2611     mov64(dst, src);
 2612   }
 2613 #else
 2614   movl(dst, src);
 2615 #endif
 2616 }
 2617 
 2618 void MacroAssembler::movptr(Address dst, Register src) {
 2619   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2620 }
 2621 
 2622 void MacroAssembler::movptr(Address dst, int32_t src) {
 2623   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2624 }
 2625 
 2626 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2627   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2628   Assembler::movdqu(dst, src);
 2629 }
 2630 
 2631 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2632   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2633   Assembler::movdqu(dst, src);
 2634 }
 2635 
 2636 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2637   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2638   Assembler::movdqu(dst, src);
 2639 }
 2640 
 2641 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2642   assert(rscratch != noreg || always_reachable(src), "missing");
 2643 
 2644   if (reachable(src)) {
 2645     movdqu(dst, as_Address(src));
 2646   } else {
 2647     lea(rscratch, src);
 2648     movdqu(dst, Address(rscratch, 0));
 2649   }
 2650 }
 2651 
 2652 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2653   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2654   Assembler::vmovdqu(dst, src);
 2655 }
 2656 
 2657 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2658   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2659   Assembler::vmovdqu(dst, src);
 2660 }
 2661 
 2662 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2663   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2664   Assembler::vmovdqu(dst, src);
 2665 }
 2666 
 2667 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2668   assert(rscratch != noreg || always_reachable(src), "missing");
 2669 
 2670   if (reachable(src)) {
 2671     vmovdqu(dst, as_Address(src));
 2672   }
 2673   else {
 2674     lea(rscratch, src);
 2675     vmovdqu(dst, Address(rscratch, 0));
 2676   }
 2677 }
 2678 
 2679 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2680   assert(rscratch != noreg || always_reachable(src), "missing");
 2681 
 2682   if (vector_len == AVX_512bit) {
 2683     evmovdquq(dst, src, AVX_512bit, rscratch);
 2684   } else if (vector_len == AVX_256bit) {
 2685     vmovdqu(dst, src, rscratch);
 2686   } else {
 2687     movdqu(dst, src, rscratch);
 2688   }
 2689 }
 2690 
 2691 void MacroAssembler::kmov(KRegister dst, Address src) {
 2692   if (VM_Version::supports_avx512bw()) {
 2693     kmovql(dst, src);
 2694   } else {
 2695     assert(VM_Version::supports_evex(), "");
 2696     kmovwl(dst, src);
 2697   }
 2698 }
 2699 
 2700 void MacroAssembler::kmov(Address dst, KRegister src) {
 2701   if (VM_Version::supports_avx512bw()) {
 2702     kmovql(dst, src);
 2703   } else {
 2704     assert(VM_Version::supports_evex(), "");
 2705     kmovwl(dst, src);
 2706   }
 2707 }
 2708 
 2709 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2710   if (VM_Version::supports_avx512bw()) {
 2711     kmovql(dst, src);
 2712   } else {
 2713     assert(VM_Version::supports_evex(), "");
 2714     kmovwl(dst, src);
 2715   }
 2716 }
 2717 
 2718 void MacroAssembler::kmov(Register dst, KRegister src) {
 2719   if (VM_Version::supports_avx512bw()) {
 2720     kmovql(dst, src);
 2721   } else {
 2722     assert(VM_Version::supports_evex(), "");
 2723     kmovwl(dst, src);
 2724   }
 2725 }
 2726 
 2727 void MacroAssembler::kmov(KRegister dst, Register src) {
 2728   if (VM_Version::supports_avx512bw()) {
 2729     kmovql(dst, src);
 2730   } else {
 2731     assert(VM_Version::supports_evex(), "");
 2732     kmovwl(dst, src);
 2733   }
 2734 }
 2735 
 2736 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2737   assert(rscratch != noreg || always_reachable(src), "missing");
 2738 
 2739   if (reachable(src)) {
 2740     kmovql(dst, as_Address(src));
 2741   } else {
 2742     lea(rscratch, src);
 2743     kmovql(dst, Address(rscratch, 0));
 2744   }
 2745 }
 2746 
 2747 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2748   assert(rscratch != noreg || always_reachable(src), "missing");
 2749 
 2750   if (reachable(src)) {
 2751     kmovwl(dst, as_Address(src));
 2752   } else {
 2753     lea(rscratch, src);
 2754     kmovwl(dst, Address(rscratch, 0));
 2755   }
 2756 }
 2757 
 2758 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2759                                int vector_len, Register rscratch) {
 2760   assert(rscratch != noreg || always_reachable(src), "missing");
 2761 
 2762   if (reachable(src)) {
 2763     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2764   } else {
 2765     lea(rscratch, src);
 2766     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2767   }
 2768 }
 2769 
 2770 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2771                                int vector_len, Register rscratch) {
 2772   assert(rscratch != noreg || always_reachable(src), "missing");
 2773 
 2774   if (reachable(src)) {
 2775     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2776   } else {
 2777     lea(rscratch, src);
 2778     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2779   }
 2780 }
 2781 
 2782 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2783   assert(rscratch != noreg || always_reachable(src), "missing");
 2784 
 2785   if (reachable(src)) {
 2786     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2787   } else {
 2788     lea(rscratch, src);
 2789     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2790   }
 2791 }
 2792 
 2793 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2794   assert(rscratch != noreg || always_reachable(src), "missing");
 2795 
 2796   if (reachable(src)) {
 2797     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2798   } else {
 2799     lea(rscratch, src);
 2800     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2801   }
 2802 }
 2803 
 2804 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2805   assert(rscratch != noreg || always_reachable(src), "missing");
 2806 
 2807   if (reachable(src)) {
 2808     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2809   } else {
 2810     lea(rscratch, src);
 2811     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2812   }
 2813 }
 2814 
 2815 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2816   assert(rscratch != noreg || always_reachable(src), "missing");
 2817 
 2818   if (reachable(src)) {
 2819     Assembler::movdqa(dst, as_Address(src));
 2820   } else {
 2821     lea(rscratch, src);
 2822     Assembler::movdqa(dst, Address(rscratch, 0));
 2823   }
 2824 }
 2825 
 2826 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2827   assert(rscratch != noreg || always_reachable(src), "missing");
 2828 
 2829   if (reachable(src)) {
 2830     Assembler::movsd(dst, as_Address(src));
 2831   } else {
 2832     lea(rscratch, src);
 2833     Assembler::movsd(dst, Address(rscratch, 0));
 2834   }
 2835 }
 2836 
 2837 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2838   assert(rscratch != noreg || always_reachable(src), "missing");
 2839 
 2840   if (reachable(src)) {
 2841     Assembler::movss(dst, as_Address(src));
 2842   } else {
 2843     lea(rscratch, src);
 2844     Assembler::movss(dst, Address(rscratch, 0));
 2845   }
 2846 }
 2847 
 2848 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2849   assert(rscratch != noreg || always_reachable(src), "missing");
 2850 
 2851   if (reachable(src)) {
 2852     Assembler::movddup(dst, as_Address(src));
 2853   } else {
 2854     lea(rscratch, src);
 2855     Assembler::movddup(dst, Address(rscratch, 0));
 2856   }
 2857 }
 2858 
 2859 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2860   assert(rscratch != noreg || always_reachable(src), "missing");
 2861 
 2862   if (reachable(src)) {
 2863     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2864   } else {
 2865     lea(rscratch, src);
 2866     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2867   }
 2868 }
 2869 
 2870 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2871   assert(rscratch != noreg || always_reachable(src), "missing");
 2872 
 2873   if (reachable(src)) {
 2874     Assembler::mulsd(dst, as_Address(src));
 2875   } else {
 2876     lea(rscratch, src);
 2877     Assembler::mulsd(dst, Address(rscratch, 0));
 2878   }
 2879 }
 2880 
 2881 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2882   assert(rscratch != noreg || always_reachable(src), "missing");
 2883 
 2884   if (reachable(src)) {
 2885     Assembler::mulss(dst, as_Address(src));
 2886   } else {
 2887     lea(rscratch, src);
 2888     Assembler::mulss(dst, Address(rscratch, 0));
 2889   }
 2890 }
 2891 
 2892 void MacroAssembler::null_check(Register reg, int offset) {
 2893   if (needs_explicit_null_check(offset)) {
 2894     // provoke OS null exception if reg is null by
 2895     // accessing M[reg] w/o changing any (non-CC) registers
 2896     // NOTE: cmpl is plenty here to provoke a segv
 2897     cmpptr(rax, Address(reg, 0));
 2898     // Note: should probably use testl(rax, Address(reg, 0));
 2899     //       may be shorter code (however, this version of
 2900     //       testl needs to be implemented first)
 2901   } else {
 2902     // nothing to do, (later) access of M[reg + offset]
 2903     // will provoke OS null exception if reg is null
 2904   }
 2905 }
 2906 
 2907 void MacroAssembler::os_breakpoint() {
 2908   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2909   // (e.g., MSVC can't call ps() otherwise)
 2910   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2911 }
 2912 
 2913 void MacroAssembler::unimplemented(const char* what) {
 2914   const char* buf = nullptr;
 2915   {
 2916     ResourceMark rm;
 2917     stringStream ss;
 2918     ss.print("unimplemented: %s", what);
 2919     buf = code_string(ss.as_string());
 2920   }
 2921   stop(buf);
 2922 }
 2923 
 2924 #ifdef _LP64
 2925 #define XSTATE_BV 0x200
 2926 #endif
 2927 
 2928 void MacroAssembler::pop_CPU_state() {
 2929   pop_FPU_state();
 2930   pop_IU_state();
 2931 }
 2932 
 2933 void MacroAssembler::pop_FPU_state() {
 2934 #ifndef _LP64
 2935   frstor(Address(rsp, 0));
 2936 #else
 2937   fxrstor(Address(rsp, 0));
 2938 #endif
 2939   addptr(rsp, FPUStateSizeInWords * wordSize);
 2940 }
 2941 
 2942 void MacroAssembler::pop_IU_state() {
 2943   popa();
 2944   LP64_ONLY(addq(rsp, 8));
 2945   popf();
 2946 }
 2947 
 2948 // Save Integer and Float state
 2949 // Warning: Stack must be 16 byte aligned (64bit)
 2950 void MacroAssembler::push_CPU_state() {
 2951   push_IU_state();
 2952   push_FPU_state();
 2953 }
 2954 
 2955 void MacroAssembler::push_FPU_state() {
 2956   subptr(rsp, FPUStateSizeInWords * wordSize);
 2957 #ifndef _LP64
 2958   fnsave(Address(rsp, 0));
 2959   fwait();
 2960 #else
 2961   fxsave(Address(rsp, 0));
 2962 #endif // LP64
 2963 }
 2964 
 2965 void MacroAssembler::push_IU_state() {
 2966   // Push flags first because pusha kills them
 2967   pushf();
 2968   // Make sure rsp stays 16-byte aligned
 2969   LP64_ONLY(subq(rsp, 8));
 2970   pusha();
 2971 }
 2972 
 2973 void MacroAssembler::push_cont_fastpath() {
 2974   if (!Continuations::enabled()) return;
 2975 
 2976 #ifndef _LP64
 2977   Register rthread = rax;
 2978   Register rrealsp = rbx;
 2979   push(rthread);
 2980   push(rrealsp);
 2981 
 2982   get_thread(rthread);
 2983 
 2984   // The code below wants the original RSP.
 2985   // Move it back after the pushes above.
 2986   movptr(rrealsp, rsp);
 2987   addptr(rrealsp, 2*wordSize);
 2988 #else
 2989   Register rthread = r15_thread;
 2990   Register rrealsp = rsp;
 2991 #endif
 2992 
 2993   Label done;
 2994   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 2995   jccb(Assembler::belowEqual, done);
 2996   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 2997   bind(done);
 2998 
 2999 #ifndef _LP64
 3000   pop(rrealsp);
 3001   pop(rthread);
 3002 #endif
 3003 }
 3004 
 3005 void MacroAssembler::pop_cont_fastpath() {
 3006   if (!Continuations::enabled()) return;
 3007 
 3008 #ifndef _LP64
 3009   Register rthread = rax;
 3010   Register rrealsp = rbx;
 3011   push(rthread);
 3012   push(rrealsp);
 3013 
 3014   get_thread(rthread);
 3015 
 3016   // The code below wants the original RSP.
 3017   // Move it back after the pushes above.
 3018   movptr(rrealsp, rsp);
 3019   addptr(rrealsp, 2*wordSize);
 3020 #else
 3021   Register rthread = r15_thread;
 3022   Register rrealsp = rsp;
 3023 #endif
 3024 
 3025   Label done;
 3026   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3027   jccb(Assembler::below, done);
 3028   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3029   bind(done);
 3030 
 3031 #ifndef _LP64
 3032   pop(rrealsp);
 3033   pop(rthread);
 3034 #endif
 3035 }
 3036 
 3037 void MacroAssembler::inc_held_monitor_count() {
 3038 #ifndef _LP64
 3039   Register thread = rax;
 3040   push(thread);
 3041   get_thread(thread);
 3042   incrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3043   pop(thread);
 3044 #else // LP64
 3045   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3046 #endif
 3047 }
 3048 
 3049 void MacroAssembler::dec_held_monitor_count() {
 3050 #ifndef _LP64
 3051   Register thread = rax;
 3052   push(thread);
 3053   get_thread(thread);
 3054   decrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3055   pop(thread);
 3056 #else // LP64
 3057   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3058 #endif
 3059 }
 3060 
 3061 #ifdef ASSERT
 3062 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3063 #ifdef _LP64
 3064   Label no_cont;
 3065   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3066   testl(cont, cont);
 3067   jcc(Assembler::zero, no_cont);
 3068   stop(name);
 3069   bind(no_cont);
 3070 #else
 3071   Unimplemented();
 3072 #endif
 3073 }
 3074 #endif
 3075 
 3076 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3077   if (!java_thread->is_valid()) {
 3078     java_thread = rdi;
 3079     get_thread(java_thread);
 3080   }
 3081   // we must set sp to zero to clear frame
 3082   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3083   // must clear fp, so that compiled frames are not confused; it is
 3084   // possible that we need it only for debugging
 3085   if (clear_fp) {
 3086     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3087   }
 3088   // Always clear the pc because it could have been set by make_walkable()
 3089   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3090   vzeroupper();
 3091 }
 3092 
 3093 void MacroAssembler::restore_rax(Register tmp) {
 3094   if (tmp == noreg) pop(rax);
 3095   else if (tmp != rax) mov(rax, tmp);
 3096 }
 3097 
 3098 void MacroAssembler::round_to(Register reg, int modulus) {
 3099   addptr(reg, modulus - 1);
 3100   andptr(reg, -modulus);
 3101 }
 3102 
 3103 void MacroAssembler::save_rax(Register tmp) {
 3104   if (tmp == noreg) push(rax);
 3105   else if (tmp != rax) mov(tmp, rax);
 3106 }
 3107 
 3108 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3109   if (at_return) {
 3110     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3111     // we may safely use rsp instead to perform the stack watermark check.
 3112     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3113     jcc(Assembler::above, slow_path);
 3114     return;
 3115   }
 3116   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3117   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3118 }
 3119 
 3120 // Calls to C land
 3121 //
 3122 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3123 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3124 // has to be reset to 0. This is required to allow proper stack traversal.
 3125 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3126                                          Register last_java_sp,
 3127                                          Register last_java_fp,
 3128                                          address  last_java_pc,
 3129                                          Register rscratch) {
 3130   vzeroupper();
 3131   // determine java_thread register
 3132   if (!java_thread->is_valid()) {
 3133     java_thread = rdi;
 3134     get_thread(java_thread);
 3135   }
 3136   // determine last_java_sp register
 3137   if (!last_java_sp->is_valid()) {
 3138     last_java_sp = rsp;
 3139   }
 3140   // last_java_fp is optional
 3141   if (last_java_fp->is_valid()) {
 3142     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3143   }
 3144   // last_java_pc is optional
 3145   if (last_java_pc != nullptr) {
 3146     Address java_pc(java_thread,
 3147                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3148     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3149   }
 3150   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3151 }
 3152 
 3153 void MacroAssembler::shlptr(Register dst, int imm8) {
 3154   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3155 }
 3156 
 3157 void MacroAssembler::shrptr(Register dst, int imm8) {
 3158   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3159 }
 3160 
 3161 void MacroAssembler::sign_extend_byte(Register reg) {
 3162   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3163     movsbl(reg, reg); // movsxb
 3164   } else {
 3165     shll(reg, 24);
 3166     sarl(reg, 24);
 3167   }
 3168 }
 3169 
 3170 void MacroAssembler::sign_extend_short(Register reg) {
 3171   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3172     movswl(reg, reg); // movsxw
 3173   } else {
 3174     shll(reg, 16);
 3175     sarl(reg, 16);
 3176   }
 3177 }
 3178 
 3179 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3180   if (imm32 >= 0 && is8bit(imm32)) {
 3181     testb(dst, imm32);
 3182   } else {
 3183     Assembler::testl(dst, imm32);
 3184   }
 3185 }
 3186 
 3187 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3188   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3189     testb(dst, imm32);
 3190   } else {
 3191     Assembler::testl(dst, imm32);
 3192   }
 3193 }
 3194 
 3195 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3196   assert(always_reachable(src), "Address should be reachable");
 3197   testl(dst, as_Address(src));
 3198 }
 3199 
 3200 #ifdef _LP64
 3201 
 3202 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3203   if (imm32 >= 0) {
 3204     testl(dst, imm32);
 3205   } else {
 3206     Assembler::testq(dst, imm32);
 3207   }
 3208 }
 3209 
 3210 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3211   if (imm32 >= 0) {
 3212     testl(dst, imm32);
 3213   } else {
 3214     Assembler::testq(dst, imm32);
 3215   }
 3216 }
 3217 
 3218 #endif
 3219 
 3220 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3221   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3222   Assembler::pcmpeqb(dst, src);
 3223 }
 3224 
 3225 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3226   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3227   Assembler::pcmpeqw(dst, src);
 3228 }
 3229 
 3230 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3231   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3232   Assembler::pcmpestri(dst, src, imm8);
 3233 }
 3234 
 3235 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3236   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3237   Assembler::pcmpestri(dst, src, imm8);
 3238 }
 3239 
 3240 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3241   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3242   Assembler::pmovzxbw(dst, src);
 3243 }
 3244 
 3245 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3246   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3247   Assembler::pmovzxbw(dst, src);
 3248 }
 3249 
 3250 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3251   assert((src->encoding() < 16),"XMM register should be 0-15");
 3252   Assembler::pmovmskb(dst, src);
 3253 }
 3254 
 3255 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3256   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3257   Assembler::ptest(dst, src);
 3258 }
 3259 
 3260 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3261   assert(rscratch != noreg || always_reachable(src), "missing");
 3262 
 3263   if (reachable(src)) {
 3264     Assembler::sqrtss(dst, as_Address(src));
 3265   } else {
 3266     lea(rscratch, src);
 3267     Assembler::sqrtss(dst, Address(rscratch, 0));
 3268   }
 3269 }
 3270 
 3271 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3272   assert(rscratch != noreg || always_reachable(src), "missing");
 3273 
 3274   if (reachable(src)) {
 3275     Assembler::subsd(dst, as_Address(src));
 3276   } else {
 3277     lea(rscratch, src);
 3278     Assembler::subsd(dst, Address(rscratch, 0));
 3279   }
 3280 }
 3281 
 3282 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3283   assert(rscratch != noreg || always_reachable(src), "missing");
 3284 
 3285   if (reachable(src)) {
 3286     Assembler::roundsd(dst, as_Address(src), rmode);
 3287   } else {
 3288     lea(rscratch, src);
 3289     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3290   }
 3291 }
 3292 
 3293 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3294   assert(rscratch != noreg || always_reachable(src), "missing");
 3295 
 3296   if (reachable(src)) {
 3297     Assembler::subss(dst, as_Address(src));
 3298   } else {
 3299     lea(rscratch, src);
 3300     Assembler::subss(dst, Address(rscratch, 0));
 3301   }
 3302 }
 3303 
 3304 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3305   assert(rscratch != noreg || always_reachable(src), "missing");
 3306 
 3307   if (reachable(src)) {
 3308     Assembler::ucomisd(dst, as_Address(src));
 3309   } else {
 3310     lea(rscratch, src);
 3311     Assembler::ucomisd(dst, Address(rscratch, 0));
 3312   }
 3313 }
 3314 
 3315 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3316   assert(rscratch != noreg || always_reachable(src), "missing");
 3317 
 3318   if (reachable(src)) {
 3319     Assembler::ucomiss(dst, as_Address(src));
 3320   } else {
 3321     lea(rscratch, src);
 3322     Assembler::ucomiss(dst, Address(rscratch, 0));
 3323   }
 3324 }
 3325 
 3326 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3327   assert(rscratch != noreg || always_reachable(src), "missing");
 3328 
 3329   // Used in sign-bit flipping with aligned address.
 3330   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3331   if (reachable(src)) {
 3332     Assembler::xorpd(dst, as_Address(src));
 3333   } else {
 3334     lea(rscratch, src);
 3335     Assembler::xorpd(dst, Address(rscratch, 0));
 3336   }
 3337 }
 3338 
 3339 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3340   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3341     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3342   }
 3343   else {
 3344     Assembler::xorpd(dst, src);
 3345   }
 3346 }
 3347 
 3348 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3349   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3350     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3351   } else {
 3352     Assembler::xorps(dst, src);
 3353   }
 3354 }
 3355 
 3356 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3357   assert(rscratch != noreg || always_reachable(src), "missing");
 3358 
 3359   // Used in sign-bit flipping with aligned address.
 3360   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3361   if (reachable(src)) {
 3362     Assembler::xorps(dst, as_Address(src));
 3363   } else {
 3364     lea(rscratch, src);
 3365     Assembler::xorps(dst, Address(rscratch, 0));
 3366   }
 3367 }
 3368 
 3369 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3370   assert(rscratch != noreg || always_reachable(src), "missing");
 3371 
 3372   // Used in sign-bit flipping with aligned address.
 3373   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3374   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3375   if (reachable(src)) {
 3376     Assembler::pshufb(dst, as_Address(src));
 3377   } else {
 3378     lea(rscratch, src);
 3379     Assembler::pshufb(dst, Address(rscratch, 0));
 3380   }
 3381 }
 3382 
 3383 // AVX 3-operands instructions
 3384 
 3385 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3386   assert(rscratch != noreg || always_reachable(src), "missing");
 3387 
 3388   if (reachable(src)) {
 3389     vaddsd(dst, nds, as_Address(src));
 3390   } else {
 3391     lea(rscratch, src);
 3392     vaddsd(dst, nds, Address(rscratch, 0));
 3393   }
 3394 }
 3395 
 3396 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3397   assert(rscratch != noreg || always_reachable(src), "missing");
 3398 
 3399   if (reachable(src)) {
 3400     vaddss(dst, nds, as_Address(src));
 3401   } else {
 3402     lea(rscratch, src);
 3403     vaddss(dst, nds, Address(rscratch, 0));
 3404   }
 3405 }
 3406 
 3407 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3408   assert(UseAVX > 0, "requires some form of AVX");
 3409   assert(rscratch != noreg || always_reachable(src), "missing");
 3410 
 3411   if (reachable(src)) {
 3412     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3413   } else {
 3414     lea(rscratch, src);
 3415     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3416   }
 3417 }
 3418 
 3419 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3420   assert(UseAVX > 0, "requires some form of AVX");
 3421   assert(rscratch != noreg || always_reachable(src), "missing");
 3422 
 3423   if (reachable(src)) {
 3424     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3425   } else {
 3426     lea(rscratch, src);
 3427     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3428   }
 3429 }
 3430 
 3431 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3432   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3433   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3434 
 3435   vandps(dst, nds, negate_field, vector_len, rscratch);
 3436 }
 3437 
 3438 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3439   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3440   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3441 
 3442   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3443 }
 3444 
 3445 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3446   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3447   Assembler::vpaddb(dst, nds, src, vector_len);
 3448 }
 3449 
 3450 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3451   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3452   Assembler::vpaddb(dst, nds, src, vector_len);
 3453 }
 3454 
 3455 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3456   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3457   Assembler::vpaddw(dst, nds, src, vector_len);
 3458 }
 3459 
 3460 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3461   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3462   Assembler::vpaddw(dst, nds, src, vector_len);
 3463 }
 3464 
 3465 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3466   assert(rscratch != noreg || always_reachable(src), "missing");
 3467 
 3468   if (reachable(src)) {
 3469     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3470   } else {
 3471     lea(rscratch, src);
 3472     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3473   }
 3474 }
 3475 
 3476 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3477   assert(rscratch != noreg || always_reachable(src), "missing");
 3478 
 3479   if (reachable(src)) {
 3480     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3481   } else {
 3482     lea(rscratch, src);
 3483     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3484   }
 3485 }
 3486 
 3487 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3488   assert(rscratch != noreg || always_reachable(src), "missing");
 3489 
 3490   if (reachable(src)) {
 3491     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3492   } else {
 3493     lea(rscratch, src);
 3494     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3495   }
 3496 }
 3497 
 3498 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3499   assert(rscratch != noreg || always_reachable(src), "missing");
 3500 
 3501   if (reachable(src)) {
 3502     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3503   } else {
 3504     lea(rscratch, src);
 3505     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3506   }
 3507 }
 3508 
 3509 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3510   assert(rscratch != noreg || always_reachable(src), "missing");
 3511 
 3512   if (reachable(src)) {
 3513     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3514   } else {
 3515     lea(rscratch, src);
 3516     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3517   }
 3518 }
 3519 
 3520 // Vector float blend
 3521 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3522 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3523   // WARN: Allow dst == (src1|src2), mask == scratch
 3524   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3525   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3526   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3527   if (blend_emulation && scratch_available && dst_available) {
 3528     if (compute_mask) {
 3529       vpsrad(scratch, mask, 32, vector_len);
 3530       mask = scratch;
 3531     }
 3532     if (dst == src1) {
 3533       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3534       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3535     } else {
 3536       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3537       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3538     }
 3539     vpor(dst, dst, scratch, vector_len);
 3540   } else {
 3541     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3542   }
 3543 }
 3544 
 3545 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3546 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3547   // WARN: Allow dst == (src1|src2), mask == scratch
 3548   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3549   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3550   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3551   if (blend_emulation && scratch_available && dst_available) {
 3552     if (compute_mask) {
 3553       vpxor(scratch, scratch, scratch, vector_len);
 3554       vpcmpgtq(scratch, scratch, mask, vector_len);
 3555       mask = scratch;
 3556     }
 3557     if (dst == src1) {
 3558       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3559       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3560     } else {
 3561       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3562       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3563     }
 3564     vpor(dst, dst, scratch, vector_len);
 3565   } else {
 3566     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3567   }
 3568 }
 3569 
 3570 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3571   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3572   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3573 }
 3574 
 3575 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3576   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3577   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3578 }
 3579 
 3580 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3581   assert(rscratch != noreg || always_reachable(src), "missing");
 3582 
 3583   if (reachable(src)) {
 3584     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3585   } else {
 3586     lea(rscratch, src);
 3587     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3588   }
 3589 }
 3590 
 3591 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3592                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3593   assert(rscratch != noreg || always_reachable(src), "missing");
 3594 
 3595   if (reachable(src)) {
 3596     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3597   } else {
 3598     lea(rscratch, src);
 3599     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3600   }
 3601 }
 3602 
 3603 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3604                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3605   assert(rscratch != noreg || always_reachable(src), "missing");
 3606 
 3607   if (reachable(src)) {
 3608     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3609   } else {
 3610     lea(rscratch, src);
 3611     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3612   }
 3613 }
 3614 
 3615 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3616                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3617   assert(rscratch != noreg || always_reachable(src), "missing");
 3618 
 3619   if (reachable(src)) {
 3620     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3621   } else {
 3622     lea(rscratch, src);
 3623     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3624   }
 3625 }
 3626 
 3627 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3628                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3629   assert(rscratch != noreg || always_reachable(src), "missing");
 3630 
 3631   if (reachable(src)) {
 3632     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3633   } else {
 3634     lea(rscratch, src);
 3635     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3636   }
 3637 }
 3638 
 3639 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3640   if (width == Assembler::Q) {
 3641     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3642   } else {
 3643     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3644   }
 3645 }
 3646 
 3647 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3648   int eq_cond_enc = 0x29;
 3649   int gt_cond_enc = 0x37;
 3650   if (width != Assembler::Q) {
 3651     eq_cond_enc = 0x74 + width;
 3652     gt_cond_enc = 0x64 + width;
 3653   }
 3654   switch (cond) {
 3655   case eq:
 3656     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3657     break;
 3658   case neq:
 3659     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3660     vallones(xtmp, vector_len);
 3661     vpxor(dst, xtmp, dst, vector_len);
 3662     break;
 3663   case le:
 3664     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3665     vallones(xtmp, vector_len);
 3666     vpxor(dst, xtmp, dst, vector_len);
 3667     break;
 3668   case nlt:
 3669     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3670     vallones(xtmp, vector_len);
 3671     vpxor(dst, xtmp, dst, vector_len);
 3672     break;
 3673   case lt:
 3674     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3675     break;
 3676   case nle:
 3677     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3678     break;
 3679   default:
 3680     assert(false, "Should not reach here");
 3681   }
 3682 }
 3683 
 3684 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3685   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3686   Assembler::vpmovzxbw(dst, src, vector_len);
 3687 }
 3688 
 3689 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3690   assert((src->encoding() < 16),"XMM register should be 0-15");
 3691   Assembler::vpmovmskb(dst, src, vector_len);
 3692 }
 3693 
 3694 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3695   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3696   Assembler::vpmullw(dst, nds, src, vector_len);
 3697 }
 3698 
 3699 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3700   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3701   Assembler::vpmullw(dst, nds, src, vector_len);
 3702 }
 3703 
 3704 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3705   assert((UseAVX > 0), "AVX support is needed");
 3706   assert(rscratch != noreg || always_reachable(src), "missing");
 3707 
 3708   if (reachable(src)) {
 3709     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3710   } else {
 3711     lea(rscratch, src);
 3712     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3713   }
 3714 }
 3715 
 3716 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3717   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3718   Assembler::vpsubb(dst, nds, src, vector_len);
 3719 }
 3720 
 3721 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3722   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3723   Assembler::vpsubb(dst, nds, src, vector_len);
 3724 }
 3725 
 3726 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3727   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3728   Assembler::vpsubw(dst, nds, src, vector_len);
 3729 }
 3730 
 3731 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3732   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3733   Assembler::vpsubw(dst, nds, src, vector_len);
 3734 }
 3735 
 3736 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3737   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3738   Assembler::vpsraw(dst, nds, shift, vector_len);
 3739 }
 3740 
 3741 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3742   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3743   Assembler::vpsraw(dst, nds, shift, vector_len);
 3744 }
 3745 
 3746 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3747   assert(UseAVX > 2,"");
 3748   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3749      vector_len = 2;
 3750   }
 3751   Assembler::evpsraq(dst, nds, shift, vector_len);
 3752 }
 3753 
 3754 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3755   assert(UseAVX > 2,"");
 3756   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3757      vector_len = 2;
 3758   }
 3759   Assembler::evpsraq(dst, nds, shift, vector_len);
 3760 }
 3761 
 3762 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3763   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3764   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3765 }
 3766 
 3767 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3768   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3769   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3770 }
 3771 
 3772 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3773   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3774   Assembler::vpsllw(dst, nds, shift, vector_len);
 3775 }
 3776 
 3777 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3778   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3779   Assembler::vpsllw(dst, nds, shift, vector_len);
 3780 }
 3781 
 3782 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3783   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3784   Assembler::vptest(dst, src);
 3785 }
 3786 
 3787 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3788   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3789   Assembler::punpcklbw(dst, src);
 3790 }
 3791 
 3792 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3793   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3794   Assembler::pshufd(dst, src, mode);
 3795 }
 3796 
 3797 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3798   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3799   Assembler::pshuflw(dst, src, mode);
 3800 }
 3801 
 3802 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3803   assert(rscratch != noreg || always_reachable(src), "missing");
 3804 
 3805   if (reachable(src)) {
 3806     vandpd(dst, nds, as_Address(src), vector_len);
 3807   } else {
 3808     lea(rscratch, src);
 3809     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3810   }
 3811 }
 3812 
 3813 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3814   assert(rscratch != noreg || always_reachable(src), "missing");
 3815 
 3816   if (reachable(src)) {
 3817     vandps(dst, nds, as_Address(src), vector_len);
 3818   } else {
 3819     lea(rscratch, src);
 3820     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3821   }
 3822 }
 3823 
 3824 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3825                             bool merge, int vector_len, Register rscratch) {
 3826   assert(rscratch != noreg || always_reachable(src), "missing");
 3827 
 3828   if (reachable(src)) {
 3829     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3830   } else {
 3831     lea(rscratch, src);
 3832     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3833   }
 3834 }
 3835 
 3836 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3837   assert(rscratch != noreg || always_reachable(src), "missing");
 3838 
 3839   if (reachable(src)) {
 3840     vdivsd(dst, nds, as_Address(src));
 3841   } else {
 3842     lea(rscratch, src);
 3843     vdivsd(dst, nds, Address(rscratch, 0));
 3844   }
 3845 }
 3846 
 3847 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3848   assert(rscratch != noreg || always_reachable(src), "missing");
 3849 
 3850   if (reachable(src)) {
 3851     vdivss(dst, nds, as_Address(src));
 3852   } else {
 3853     lea(rscratch, src);
 3854     vdivss(dst, nds, Address(rscratch, 0));
 3855   }
 3856 }
 3857 
 3858 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3859   assert(rscratch != noreg || always_reachable(src), "missing");
 3860 
 3861   if (reachable(src)) {
 3862     vmulsd(dst, nds, as_Address(src));
 3863   } else {
 3864     lea(rscratch, src);
 3865     vmulsd(dst, nds, Address(rscratch, 0));
 3866   }
 3867 }
 3868 
 3869 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3870   assert(rscratch != noreg || always_reachable(src), "missing");
 3871 
 3872   if (reachable(src)) {
 3873     vmulss(dst, nds, as_Address(src));
 3874   } else {
 3875     lea(rscratch, src);
 3876     vmulss(dst, nds, Address(rscratch, 0));
 3877   }
 3878 }
 3879 
 3880 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3881   assert(rscratch != noreg || always_reachable(src), "missing");
 3882 
 3883   if (reachable(src)) {
 3884     vsubsd(dst, nds, as_Address(src));
 3885   } else {
 3886     lea(rscratch, src);
 3887     vsubsd(dst, nds, Address(rscratch, 0));
 3888   }
 3889 }
 3890 
 3891 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3892   assert(rscratch != noreg || always_reachable(src), "missing");
 3893 
 3894   if (reachable(src)) {
 3895     vsubss(dst, nds, as_Address(src));
 3896   } else {
 3897     lea(rscratch, src);
 3898     vsubss(dst, nds, Address(rscratch, 0));
 3899   }
 3900 }
 3901 
 3902 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3903   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3904   assert(rscratch != noreg || always_reachable(src), "missing");
 3905 
 3906   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3907 }
 3908 
 3909 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3910   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3911   assert(rscratch != noreg || always_reachable(src), "missing");
 3912 
 3913   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3914 }
 3915 
 3916 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3917   assert(rscratch != noreg || always_reachable(src), "missing");
 3918 
 3919   if (reachable(src)) {
 3920     vxorpd(dst, nds, as_Address(src), vector_len);
 3921   } else {
 3922     lea(rscratch, src);
 3923     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 3924   }
 3925 }
 3926 
 3927 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3928   assert(rscratch != noreg || always_reachable(src), "missing");
 3929 
 3930   if (reachable(src)) {
 3931     vxorps(dst, nds, as_Address(src), vector_len);
 3932   } else {
 3933     lea(rscratch, src);
 3934     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 3935   }
 3936 }
 3937 
 3938 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3939   assert(rscratch != noreg || always_reachable(src), "missing");
 3940 
 3941   if (UseAVX > 1 || (vector_len < 1)) {
 3942     if (reachable(src)) {
 3943       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 3944     } else {
 3945       lea(rscratch, src);
 3946       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 3947     }
 3948   } else {
 3949     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 3950   }
 3951 }
 3952 
 3953 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3954   assert(rscratch != noreg || always_reachable(src), "missing");
 3955 
 3956   if (reachable(src)) {
 3957     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 3958   } else {
 3959     lea(rscratch, src);
 3960     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 3961   }
 3962 }
 3963 
 3964 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 3965   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 3966   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 3967   // The inverted mask is sign-extended
 3968   andptr(possibly_non_local, inverted_mask);
 3969 }
 3970 
 3971 void MacroAssembler::resolve_jobject(Register value,
 3972                                      Register thread,
 3973                                      Register tmp) {
 3974   assert_different_registers(value, thread, tmp);
 3975   Label done, tagged, weak_tagged;
 3976   testptr(value, value);
 3977   jcc(Assembler::zero, done);           // Use null as-is.
 3978   testptr(value, JNIHandles::tag_mask); // Test for tag.
 3979   jcc(Assembler::notZero, tagged);
 3980 
 3981   // Resolve local handle
 3982   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 3983   verify_oop(value);
 3984   jmp(done);
 3985 
 3986   bind(tagged);
 3987   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 3988   jcc(Assembler::notZero, weak_tagged);
 3989 
 3990   // Resolve global handle
 3991   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 3992   verify_oop(value);
 3993   jmp(done);
 3994 
 3995   bind(weak_tagged);
 3996   // Resolve jweak.
 3997   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 3998                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 3999   verify_oop(value);
 4000 
 4001   bind(done);
 4002 }
 4003 
 4004 void MacroAssembler::resolve_global_jobject(Register value,
 4005                                             Register thread,
 4006                                             Register tmp) {
 4007   assert_different_registers(value, thread, tmp);
 4008   Label done;
 4009 
 4010   testptr(value, value);
 4011   jcc(Assembler::zero, done);           // Use null as-is.
 4012 
 4013 #ifdef ASSERT
 4014   {
 4015     Label valid_global_tag;
 4016     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4017     jcc(Assembler::notZero, valid_global_tag);
 4018     stop("non global jobject using resolve_global_jobject");
 4019     bind(valid_global_tag);
 4020   }
 4021 #endif
 4022 
 4023   // Resolve global handle
 4024   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4025   verify_oop(value);
 4026 
 4027   bind(done);
 4028 }
 4029 
 4030 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4031   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4032 }
 4033 
 4034 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4035 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4036   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4037 }
 4038 
 4039 void MacroAssembler::subptr(Register dst, Register src) {
 4040   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4041 }
 4042 
 4043 // C++ bool manipulation
 4044 void MacroAssembler::testbool(Register dst) {
 4045   if(sizeof(bool) == 1)
 4046     testb(dst, 0xff);
 4047   else if(sizeof(bool) == 2) {
 4048     // testw implementation needed for two byte bools
 4049     ShouldNotReachHere();
 4050   } else if(sizeof(bool) == 4)
 4051     testl(dst, dst);
 4052   else
 4053     // unsupported
 4054     ShouldNotReachHere();
 4055 }
 4056 
 4057 void MacroAssembler::testptr(Register dst, Register src) {
 4058   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4059 }
 4060 
 4061 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4062 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4063                                    Register var_size_in_bytes,
 4064                                    int con_size_in_bytes,
 4065                                    Register t1,
 4066                                    Register t2,
 4067                                    Label& slow_case) {
 4068   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4069   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4070 }
 4071 
 4072 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4073   RegSet regs;
 4074 #ifdef _LP64
 4075   regs += RegSet::of(rax, rcx, rdx);
 4076 #ifndef WINDOWS
 4077   regs += RegSet::of(rsi, rdi);
 4078 #endif
 4079   regs += RegSet::range(r8, r11);
 4080 #else
 4081   regs += RegSet::of(rax, rcx, rdx);
 4082 #endif
 4083   return regs;
 4084 }
 4085 
 4086 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4087   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4088 #if defined(WINDOWS) && defined(_LP64)
 4089   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4090   if (num_xmm_registers > 16) {
 4091      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4092   }
 4093   return result;
 4094 #else
 4095   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4096 #endif
 4097 }
 4098 
 4099 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4100 
 4101 #ifndef _LP64
 4102 static bool use_x87_registers() { return UseSSE < 2; }
 4103 #endif
 4104 static bool use_xmm_registers() { return UseSSE >= 1; }
 4105 
 4106 // C1 only ever uses the first double/float of the XMM register.
 4107 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4108 
 4109 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4110   if (UseSSE == 1) {
 4111     masm->movflt(Address(rsp, offset), reg);
 4112   } else {
 4113     masm->movdbl(Address(rsp, offset), reg);
 4114   }
 4115 }
 4116 
 4117 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4118   if (UseSSE == 1) {
 4119     masm->movflt(reg, Address(rsp, offset));
 4120   } else {
 4121     masm->movdbl(reg, Address(rsp, offset));
 4122   }
 4123 }
 4124 
 4125 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4126                                   bool save_fpu, int& gp_area_size,
 4127                                   int& fp_area_size, int& xmm_area_size) {
 4128 
 4129   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4130                          StackAlignmentInBytes);
 4131 #ifdef _LP64
 4132   fp_area_size = 0;
 4133 #else
 4134   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4135 #endif
 4136   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4137 
 4138   return gp_area_size + fp_area_size + xmm_area_size;
 4139 }
 4140 
 4141 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4142   block_comment("push_call_clobbered_registers start");
 4143   // Regular registers
 4144   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4145 
 4146   int gp_area_size;
 4147   int fp_area_size;
 4148   int xmm_area_size;
 4149   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4150                                                gp_area_size, fp_area_size, xmm_area_size);
 4151   subptr(rsp, total_save_size);
 4152 
 4153   push_set(gp_registers_to_push, 0);
 4154 
 4155 #ifndef _LP64
 4156   if (save_fpu && use_x87_registers()) {
 4157     fnsave(Address(rsp, gp_area_size));
 4158     fwait();
 4159   }
 4160 #endif
 4161   if (save_fpu && use_xmm_registers()) {
 4162     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4163   }
 4164 
 4165   block_comment("push_call_clobbered_registers end");
 4166 }
 4167 
 4168 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4169   block_comment("pop_call_clobbered_registers start");
 4170 
 4171   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4172 
 4173   int gp_area_size;
 4174   int fp_area_size;
 4175   int xmm_area_size;
 4176   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4177                                                gp_area_size, fp_area_size, xmm_area_size);
 4178 
 4179   if (restore_fpu && use_xmm_registers()) {
 4180     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4181   }
 4182 #ifndef _LP64
 4183   if (restore_fpu && use_x87_registers()) {
 4184     frstor(Address(rsp, gp_area_size));
 4185   }
 4186 #endif
 4187 
 4188   pop_set(gp_registers_to_pop, 0);
 4189 
 4190   addptr(rsp, total_save_size);
 4191 
 4192   vzeroupper();
 4193 
 4194   block_comment("pop_call_clobbered_registers end");
 4195 }
 4196 
 4197 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4198   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4199   int spill_offset = offset;
 4200 
 4201   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4202     save_xmm_register(this, spill_offset, *it);
 4203     spill_offset += xmm_save_size();
 4204   }
 4205 }
 4206 
 4207 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4208   int restore_size = set.size() * xmm_save_size();
 4209   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4210 
 4211   int restore_offset = offset + restore_size - xmm_save_size();
 4212 
 4213   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4214     restore_xmm_register(this, restore_offset, *it);
 4215     restore_offset -= xmm_save_size();
 4216   }
 4217 }
 4218 
 4219 void MacroAssembler::push_set(RegSet set, int offset) {
 4220   int spill_offset;
 4221   if (offset == -1) {
 4222     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4223     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4224     subptr(rsp, aligned_size);
 4225     spill_offset = 0;
 4226   } else {
 4227     spill_offset = offset;
 4228   }
 4229 
 4230   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4231     movptr(Address(rsp, spill_offset), *it);
 4232     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4233   }
 4234 }
 4235 
 4236 void MacroAssembler::pop_set(RegSet set, int offset) {
 4237 
 4238   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4239   int restore_size = set.size() * gp_reg_size;
 4240   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4241 
 4242   int restore_offset;
 4243   if (offset == -1) {
 4244     restore_offset = restore_size - gp_reg_size;
 4245   } else {
 4246     restore_offset = offset + restore_size - gp_reg_size;
 4247   }
 4248   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4249     movptr(*it, Address(rsp, restore_offset));
 4250     restore_offset -= gp_reg_size;
 4251   }
 4252 
 4253   if (offset == -1) {
 4254     addptr(rsp, aligned_size);
 4255   }
 4256 }
 4257 
 4258 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4259 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4260   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4261   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4262   Label done;
 4263 
 4264   testptr(length_in_bytes, length_in_bytes);
 4265   jcc(Assembler::zero, done);
 4266 
 4267   // initialize topmost word, divide index by 2, check if odd and test if zero
 4268   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4269 #ifdef ASSERT
 4270   {
 4271     Label L;
 4272     testptr(length_in_bytes, BytesPerWord - 1);
 4273     jcc(Assembler::zero, L);
 4274     stop("length must be a multiple of BytesPerWord");
 4275     bind(L);
 4276   }
 4277 #endif
 4278   Register index = length_in_bytes;
 4279   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4280   if (UseIncDec) {
 4281     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4282   } else {
 4283     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4284     shrptr(index, 1);
 4285   }
 4286 #ifndef _LP64
 4287   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4288   {
 4289     Label even;
 4290     // note: if index was a multiple of 8, then it cannot
 4291     //       be 0 now otherwise it must have been 0 before
 4292     //       => if it is even, we don't need to check for 0 again
 4293     jcc(Assembler::carryClear, even);
 4294     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4295     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4296     // index could be 0 now, must check again
 4297     jcc(Assembler::zero, done);
 4298     bind(even);
 4299   }
 4300 #endif // !_LP64
 4301   // initialize remaining object fields: index is a multiple of 2 now
 4302   {
 4303     Label loop;
 4304     bind(loop);
 4305     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4306     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4307     decrement(index);
 4308     jcc(Assembler::notZero, loop);
 4309   }
 4310 
 4311   bind(done);
 4312 }
 4313 
 4314 // Look up the method for a megamorphic invokeinterface call.
 4315 // The target method is determined by <intf_klass, itable_index>.
 4316 // The receiver klass is in recv_klass.
 4317 // On success, the result will be in method_result, and execution falls through.
 4318 // On failure, execution transfers to the given label.
 4319 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4320                                              Register intf_klass,
 4321                                              RegisterOrConstant itable_index,
 4322                                              Register method_result,
 4323                                              Register scan_temp,
 4324                                              Label& L_no_such_interface,
 4325                                              bool return_method) {
 4326   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4327   assert_different_registers(method_result, intf_klass, scan_temp);
 4328   assert(recv_klass != method_result || !return_method,
 4329          "recv_klass can be destroyed when method isn't needed");
 4330 
 4331   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4332          "caller must use same register for non-constant itable index as for method");
 4333 
 4334   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4335   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4336   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4337   int scan_step   = itableOffsetEntry::size() * wordSize;
 4338   int vte_size    = vtableEntry::size_in_bytes();
 4339   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4340   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4341 
 4342   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4343 
 4344   // %%% Could store the aligned, prescaled offset in the klassoop.
 4345   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4346 
 4347   if (return_method) {
 4348     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4349     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4350     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4351   }
 4352 
 4353   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4354   //   if (scan->interface() == intf) {
 4355   //     result = (klass + scan->offset() + itable_index);
 4356   //   }
 4357   // }
 4358   Label search, found_method;
 4359 
 4360   for (int peel = 1; peel >= 0; peel--) {
 4361     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4362     cmpptr(intf_klass, method_result);
 4363 
 4364     if (peel) {
 4365       jccb(Assembler::equal, found_method);
 4366     } else {
 4367       jccb(Assembler::notEqual, search);
 4368       // (invert the test to fall through to found_method...)
 4369     }
 4370 
 4371     if (!peel)  break;
 4372 
 4373     bind(search);
 4374 
 4375     // Check that the previous entry is non-null.  A null entry means that
 4376     // the receiver class doesn't implement the interface, and wasn't the
 4377     // same as when the caller was compiled.
 4378     testptr(method_result, method_result);
 4379     jcc(Assembler::zero, L_no_such_interface);
 4380     addptr(scan_temp, scan_step);
 4381   }
 4382 
 4383   bind(found_method);
 4384 
 4385   if (return_method) {
 4386     // Got a hit.
 4387     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4388     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4389   }
 4390 }
 4391 
 4392 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4393 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4394 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4395 // The target method is determined by <holder_klass, itable_index>.
 4396 // The receiver klass is in recv_klass.
 4397 // On success, the result will be in method_result, and execution falls through.
 4398 // On failure, execution transfers to the given label.
 4399 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4400                                                   Register holder_klass,
 4401                                                   Register resolved_klass,
 4402                                                   Register method_result,
 4403                                                   Register scan_temp,
 4404                                                   Register temp_reg2,
 4405                                                   Register receiver,
 4406                                                   int itable_index,
 4407                                                   Label& L_no_such_interface) {
 4408   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4409   Register temp_itbl_klass = method_result;
 4410   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4411 
 4412   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4413   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4414   int scan_step = itableOffsetEntry::size() * wordSize;
 4415   int vte_size = vtableEntry::size_in_bytes();
 4416   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4417   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4418   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4419   assert(vte_size == wordSize, "adjust times_vte_scale");
 4420 
 4421   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4422 
 4423   // temp_itbl_klass = recv_klass.itable[0]
 4424   // scan_temp = &recv_klass.itable[0] + step
 4425   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4426   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4427   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4428   xorptr(temp_reg, temp_reg);
 4429 
 4430   // Initial checks:
 4431   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4432   //   - if (itable[0] == 0), no such interface
 4433   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4434   cmpptr(holder_klass, resolved_klass);
 4435   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4436   testptr(temp_itbl_klass, temp_itbl_klass);
 4437   jccb(Assembler::zero, L_no_such_interface);
 4438   cmpptr(holder_klass, temp_itbl_klass);
 4439   jccb(Assembler::equal, L_holder_found);
 4440 
 4441   // Loop: Look for holder_klass record in itable
 4442   //   do {
 4443   //     tmp = itable[index];
 4444   //     index += step;
 4445   //     if (tmp == holder_klass) {
 4446   //       goto L_holder_found; // Found!
 4447   //     }
 4448   //   } while (tmp != 0);
 4449   //   goto L_no_such_interface // Not found.
 4450   Label L_scan_holder;
 4451   bind(L_scan_holder);
 4452     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4453     addptr(scan_temp, scan_step);
 4454     cmpptr(holder_klass, temp_itbl_klass);
 4455     jccb(Assembler::equal, L_holder_found);
 4456     testptr(temp_itbl_klass, temp_itbl_klass);
 4457     jccb(Assembler::notZero, L_scan_holder);
 4458 
 4459   jmpb(L_no_such_interface);
 4460 
 4461   // Loop: Look for resolved_class record in itable
 4462   //   do {
 4463   //     tmp = itable[index];
 4464   //     index += step;
 4465   //     if (tmp == holder_klass) {
 4466   //        // Also check if we have met a holder klass
 4467   //        holder_tmp = itable[index-step-ioffset];
 4468   //     }
 4469   //     if (tmp == resolved_klass) {
 4470   //        goto L_resolved_found;  // Found!
 4471   //     }
 4472   //   } while (tmp != 0);
 4473   //   goto L_no_such_interface // Not found.
 4474   //
 4475   Label L_loop_scan_resolved;
 4476   bind(L_loop_scan_resolved);
 4477     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4478     addptr(scan_temp, scan_step);
 4479     bind(L_loop_scan_resolved_entry);
 4480     cmpptr(holder_klass, temp_itbl_klass);
 4481     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4482     cmpptr(resolved_klass, temp_itbl_klass);
 4483     jccb(Assembler::equal, L_resolved_found);
 4484     testptr(temp_itbl_klass, temp_itbl_klass);
 4485     jccb(Assembler::notZero, L_loop_scan_resolved);
 4486 
 4487   jmpb(L_no_such_interface);
 4488 
 4489   Label L_ready;
 4490 
 4491   // See if we already have a holder klass. If not, go and scan for it.
 4492   bind(L_resolved_found);
 4493   testptr(temp_reg, temp_reg);
 4494   jccb(Assembler::zero, L_scan_holder);
 4495   jmpb(L_ready);
 4496 
 4497   bind(L_holder_found);
 4498   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4499 
 4500   // Finally, temp_reg contains holder_klass vtable offset
 4501   bind(L_ready);
 4502   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4503   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4504     load_klass(scan_temp, receiver, noreg);
 4505     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4506   } else {
 4507     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4508   }
 4509 }
 4510 
 4511 
 4512 // virtual method calling
 4513 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4514                                            RegisterOrConstant vtable_index,
 4515                                            Register method_result) {
 4516   const ByteSize base = Klass::vtable_start_offset();
 4517   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4518   Address vtable_entry_addr(recv_klass,
 4519                             vtable_index, Address::times_ptr,
 4520                             base + vtableEntry::method_offset());
 4521   movptr(method_result, vtable_entry_addr);
 4522 }
 4523 
 4524 
 4525 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4526                            Register super_klass,
 4527                            Register temp_reg,
 4528                            Label& L_success) {
 4529   Label L_failure;
 4530   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4531   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4532   bind(L_failure);
 4533 }
 4534 
 4535 
 4536 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4537                                                    Register super_klass,
 4538                                                    Register temp_reg,
 4539                                                    Label* L_success,
 4540                                                    Label* L_failure,
 4541                                                    Label* L_slow_path,
 4542                                         RegisterOrConstant super_check_offset) {
 4543   assert_different_registers(sub_klass, super_klass, temp_reg);
 4544   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4545   if (super_check_offset.is_register()) {
 4546     assert_different_registers(sub_klass, super_klass,
 4547                                super_check_offset.as_register());
 4548   } else if (must_load_sco) {
 4549     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4550   }
 4551 
 4552   Label L_fallthrough;
 4553   int label_nulls = 0;
 4554   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4555   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4556   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4557   assert(label_nulls <= 1, "at most one null in the batch");
 4558 
 4559   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4560   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4561   Address super_check_offset_addr(super_klass, sco_offset);
 4562 
 4563   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4564   // range of a jccb.  If this routine grows larger, reconsider at
 4565   // least some of these.
 4566 #define local_jcc(assembler_cond, label)                                \
 4567   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4568   else                             jcc( assembler_cond, label) /*omit semi*/
 4569 
 4570   // Hacked jmp, which may only be used just before L_fallthrough.
 4571 #define final_jmp(label)                                                \
 4572   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4573   else                            jmp(label)                /*omit semi*/
 4574 
 4575   // If the pointers are equal, we are done (e.g., String[] elements).
 4576   // This self-check enables sharing of secondary supertype arrays among
 4577   // non-primary types such as array-of-interface.  Otherwise, each such
 4578   // type would need its own customized SSA.
 4579   // We move this check to the front of the fast path because many
 4580   // type checks are in fact trivially successful in this manner,
 4581   // so we get a nicely predicted branch right at the start of the check.
 4582   cmpptr(sub_klass, super_klass);
 4583   local_jcc(Assembler::equal, *L_success);
 4584 
 4585   // Check the supertype display:
 4586   if (must_load_sco) {
 4587     // Positive movl does right thing on LP64.
 4588     movl(temp_reg, super_check_offset_addr);
 4589     super_check_offset = RegisterOrConstant(temp_reg);
 4590   }
 4591   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4592   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4593 
 4594   // This check has worked decisively for primary supers.
 4595   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4596   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4597   // This works in the same check above because of a tricky aliasing
 4598   // between the super_cache and the primary super display elements.
 4599   // (The 'super_check_addr' can address either, as the case requires.)
 4600   // Note that the cache is updated below if it does not help us find
 4601   // what we need immediately.
 4602   // So if it was a primary super, we can just fail immediately.
 4603   // Otherwise, it's the slow path for us (no success at this point).
 4604 
 4605   if (super_check_offset.is_register()) {
 4606     local_jcc(Assembler::equal, *L_success);
 4607     cmpl(super_check_offset.as_register(), sc_offset);
 4608     if (L_failure == &L_fallthrough) {
 4609       local_jcc(Assembler::equal, *L_slow_path);
 4610     } else {
 4611       local_jcc(Assembler::notEqual, *L_failure);
 4612       final_jmp(*L_slow_path);
 4613     }
 4614   } else if (super_check_offset.as_constant() == sc_offset) {
 4615     // Need a slow path; fast failure is impossible.
 4616     if (L_slow_path == &L_fallthrough) {
 4617       local_jcc(Assembler::equal, *L_success);
 4618     } else {
 4619       local_jcc(Assembler::notEqual, *L_slow_path);
 4620       final_jmp(*L_success);
 4621     }
 4622   } else {
 4623     // No slow path; it's a fast decision.
 4624     if (L_failure == &L_fallthrough) {
 4625       local_jcc(Assembler::equal, *L_success);
 4626     } else {
 4627       local_jcc(Assembler::notEqual, *L_failure);
 4628       final_jmp(*L_success);
 4629     }
 4630   }
 4631 
 4632   bind(L_fallthrough);
 4633 
 4634 #undef local_jcc
 4635 #undef final_jmp
 4636 }
 4637 
 4638 
 4639 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4640                                                    Register super_klass,
 4641                                                    Register temp_reg,
 4642                                                    Register temp2_reg,
 4643                                                    Label* L_success,
 4644                                                    Label* L_failure,
 4645                                                    bool set_cond_codes) {
 4646   assert_different_registers(sub_klass, super_klass, temp_reg);
 4647   if (temp2_reg != noreg)
 4648     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4649 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4650 
 4651   Label L_fallthrough;
 4652   int label_nulls = 0;
 4653   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4654   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4655   assert(label_nulls <= 1, "at most one null in the batch");
 4656 
 4657   // a couple of useful fields in sub_klass:
 4658   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4659   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4660   Address secondary_supers_addr(sub_klass, ss_offset);
 4661   Address super_cache_addr(     sub_klass, sc_offset);
 4662 
 4663   // Do a linear scan of the secondary super-klass chain.
 4664   // This code is rarely used, so simplicity is a virtue here.
 4665   // The repne_scan instruction uses fixed registers, which we must spill.
 4666   // Don't worry too much about pre-existing connections with the input regs.
 4667 
 4668   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4669   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4670 
 4671   // Get super_klass value into rax (even if it was in rdi or rcx).
 4672   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4673   if (super_klass != rax) {
 4674     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4675     mov(rax, super_klass);
 4676   }
 4677   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4678   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4679 
 4680 #ifndef PRODUCT
 4681   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4682   ExternalAddress pst_counter_addr((address) pst_counter);
 4683   NOT_LP64(  incrementl(pst_counter_addr) );
 4684   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4685   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4686 #endif //PRODUCT
 4687 
 4688   // We will consult the secondary-super array.
 4689   movptr(rdi, secondary_supers_addr);
 4690   // Load the array length.  (Positive movl does right thing on LP64.)
 4691   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4692   // Skip to start of data.
 4693   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4694 
 4695   // Scan RCX words at [RDI] for an occurrence of RAX.
 4696   // Set NZ/Z based on last compare.
 4697   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4698   // not change flags (only scas instruction which is repeated sets flags).
 4699   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4700 
 4701     testptr(rax,rax); // Set Z = 0
 4702     repne_scan();
 4703 
 4704   // Unspill the temp. registers:
 4705   if (pushed_rdi)  pop(rdi);
 4706   if (pushed_rcx)  pop(rcx);
 4707   if (pushed_rax)  pop(rax);
 4708 
 4709   if (set_cond_codes) {
 4710     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4711     assert(!pushed_rdi, "rdi must be left non-null");
 4712     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4713   }
 4714 
 4715   if (L_failure == &L_fallthrough)
 4716         jccb(Assembler::notEqual, *L_failure);
 4717   else  jcc(Assembler::notEqual, *L_failure);
 4718 
 4719   // Success.  Cache the super we found and proceed in triumph.
 4720   movptr(super_cache_addr, super_klass);
 4721 
 4722   if (L_success != &L_fallthrough) {
 4723     jmp(*L_success);
 4724   }
 4725 
 4726 #undef IS_A_TEMP
 4727 
 4728   bind(L_fallthrough);
 4729 }
 4730 
 4731 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 4732   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 4733 
 4734   Label L_fallthrough;
 4735   if (L_fast_path == nullptr) {
 4736     L_fast_path = &L_fallthrough;
 4737   } else if (L_slow_path == nullptr) {
 4738     L_slow_path = &L_fallthrough;
 4739   }
 4740 
 4741   // Fast path check: class is fully initialized
 4742   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 4743   jcc(Assembler::equal, *L_fast_path);
 4744 
 4745   // Fast path check: current thread is initializer thread
 4746   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 4747   if (L_slow_path == &L_fallthrough) {
 4748     jcc(Assembler::equal, *L_fast_path);
 4749     bind(*L_slow_path);
 4750   } else if (L_fast_path == &L_fallthrough) {
 4751     jcc(Assembler::notEqual, *L_slow_path);
 4752     bind(*L_fast_path);
 4753   } else {
 4754     Unimplemented();
 4755   }
 4756 }
 4757 
 4758 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 4759   if (VM_Version::supports_cmov()) {
 4760     cmovl(cc, dst, src);
 4761   } else {
 4762     Label L;
 4763     jccb(negate_condition(cc), L);
 4764     movl(dst, src);
 4765     bind(L);
 4766   }
 4767 }
 4768 
 4769 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 4770   if (VM_Version::supports_cmov()) {
 4771     cmovl(cc, dst, src);
 4772   } else {
 4773     Label L;
 4774     jccb(negate_condition(cc), L);
 4775     movl(dst, src);
 4776     bind(L);
 4777   }
 4778 }
 4779 
 4780 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 4781   if (!VerifyOops) return;
 4782 
 4783   BLOCK_COMMENT("verify_oop {");
 4784 #ifdef _LP64
 4785   push(rscratch1);
 4786 #endif
 4787   push(rax);                          // save rax
 4788   push(reg);                          // pass register argument
 4789 
 4790   // Pass register number to verify_oop_subroutine
 4791   const char* b = nullptr;
 4792   {
 4793     ResourceMark rm;
 4794     stringStream ss;
 4795     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 4796     b = code_string(ss.as_string());
 4797   }
 4798   ExternalAddress buffer((address) b);
 4799   pushptr(buffer.addr(), rscratch1);
 4800 
 4801   // call indirectly to solve generation ordering problem
 4802   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 4803   call(rax);
 4804   // Caller pops the arguments (oop, message) and restores rax, r10
 4805   BLOCK_COMMENT("} verify_oop");
 4806 }
 4807 
 4808 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 4809   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 4810     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 4811     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 4812     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 4813   } else if (VM_Version::supports_avx()) {
 4814     vpcmpeqd(dst, dst, dst, vector_len);
 4815   } else {
 4816     assert(VM_Version::supports_sse2(), "");
 4817     pcmpeqd(dst, dst);
 4818   }
 4819 }
 4820 
 4821 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 4822                                          int extra_slot_offset) {
 4823   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 4824   int stackElementSize = Interpreter::stackElementSize;
 4825   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 4826 #ifdef ASSERT
 4827   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 4828   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 4829 #endif
 4830   Register             scale_reg    = noreg;
 4831   Address::ScaleFactor scale_factor = Address::no_scale;
 4832   if (arg_slot.is_constant()) {
 4833     offset += arg_slot.as_constant() * stackElementSize;
 4834   } else {
 4835     scale_reg    = arg_slot.as_register();
 4836     scale_factor = Address::times(stackElementSize);
 4837   }
 4838   offset += wordSize;           // return PC is on stack
 4839   return Address(rsp, scale_reg, scale_factor, offset);
 4840 }
 4841 
 4842 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 4843   if (!VerifyOops) return;
 4844 
 4845 #ifdef _LP64
 4846   push(rscratch1);
 4847 #endif
 4848   push(rax); // save rax,
 4849   // addr may contain rsp so we will have to adjust it based on the push
 4850   // we just did (and on 64 bit we do two pushes)
 4851   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 4852   // stores rax into addr which is backwards of what was intended.
 4853   if (addr.uses(rsp)) {
 4854     lea(rax, addr);
 4855     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 4856   } else {
 4857     pushptr(addr);
 4858   }
 4859 
 4860   // Pass register number to verify_oop_subroutine
 4861   const char* b = nullptr;
 4862   {
 4863     ResourceMark rm;
 4864     stringStream ss;
 4865     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 4866     b = code_string(ss.as_string());
 4867   }
 4868   ExternalAddress buffer((address) b);
 4869   pushptr(buffer.addr(), rscratch1);
 4870 
 4871   // call indirectly to solve generation ordering problem
 4872   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 4873   call(rax);
 4874   // Caller pops the arguments (addr, message) and restores rax, r10.
 4875 }
 4876 
 4877 void MacroAssembler::verify_tlab() {
 4878 #ifdef ASSERT
 4879   if (UseTLAB && VerifyOops) {
 4880     Label next, ok;
 4881     Register t1 = rsi;
 4882     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 4883 
 4884     push(t1);
 4885     NOT_LP64(push(thread_reg));
 4886     NOT_LP64(get_thread(thread_reg));
 4887 
 4888     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 4889     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 4890     jcc(Assembler::aboveEqual, next);
 4891     STOP("assert(top >= start)");
 4892     should_not_reach_here();
 4893 
 4894     bind(next);
 4895     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 4896     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 4897     jcc(Assembler::aboveEqual, ok);
 4898     STOP("assert(top <= end)");
 4899     should_not_reach_here();
 4900 
 4901     bind(ok);
 4902     NOT_LP64(pop(thread_reg));
 4903     pop(t1);
 4904   }
 4905 #endif
 4906 }
 4907 
 4908 class ControlWord {
 4909  public:
 4910   int32_t _value;
 4911 
 4912   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 4913   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 4914   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 4915   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 4916   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 4917   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 4918   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 4919   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 4920 
 4921   void print() const {
 4922     // rounding control
 4923     const char* rc;
 4924     switch (rounding_control()) {
 4925       case 0: rc = "round near"; break;
 4926       case 1: rc = "round down"; break;
 4927       case 2: rc = "round up  "; break;
 4928       case 3: rc = "chop      "; break;
 4929       default:
 4930         rc = nullptr; // silence compiler warnings
 4931         fatal("Unknown rounding control: %d", rounding_control());
 4932     };
 4933     // precision control
 4934     const char* pc;
 4935     switch (precision_control()) {
 4936       case 0: pc = "24 bits "; break;
 4937       case 1: pc = "reserved"; break;
 4938       case 2: pc = "53 bits "; break;
 4939       case 3: pc = "64 bits "; break;
 4940       default:
 4941         pc = nullptr; // silence compiler warnings
 4942         fatal("Unknown precision control: %d", precision_control());
 4943     };
 4944     // flags
 4945     char f[9];
 4946     f[0] = ' ';
 4947     f[1] = ' ';
 4948     f[2] = (precision   ()) ? 'P' : 'p';
 4949     f[3] = (underflow   ()) ? 'U' : 'u';
 4950     f[4] = (overflow    ()) ? 'O' : 'o';
 4951     f[5] = (zero_divide ()) ? 'Z' : 'z';
 4952     f[6] = (denormalized()) ? 'D' : 'd';
 4953     f[7] = (invalid     ()) ? 'I' : 'i';
 4954     f[8] = '\x0';
 4955     // output
 4956     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 4957   }
 4958 
 4959 };
 4960 
 4961 class StatusWord {
 4962  public:
 4963   int32_t _value;
 4964 
 4965   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 4966   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 4967   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 4968   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 4969   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 4970   int  top() const                     { return  (_value >> 11) & 7      ; }
 4971   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 4972   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 4973   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 4974   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 4975   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 4976   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 4977   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 4978   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 4979 
 4980   void print() const {
 4981     // condition codes
 4982     char c[5];
 4983     c[0] = (C3()) ? '3' : '-';
 4984     c[1] = (C2()) ? '2' : '-';
 4985     c[2] = (C1()) ? '1' : '-';
 4986     c[3] = (C0()) ? '0' : '-';
 4987     c[4] = '\x0';
 4988     // flags
 4989     char f[9];
 4990     f[0] = (error_status()) ? 'E' : '-';
 4991     f[1] = (stack_fault ()) ? 'S' : '-';
 4992     f[2] = (precision   ()) ? 'P' : '-';
 4993     f[3] = (underflow   ()) ? 'U' : '-';
 4994     f[4] = (overflow    ()) ? 'O' : '-';
 4995     f[5] = (zero_divide ()) ? 'Z' : '-';
 4996     f[6] = (denormalized()) ? 'D' : '-';
 4997     f[7] = (invalid     ()) ? 'I' : '-';
 4998     f[8] = '\x0';
 4999     // output
 5000     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5001   }
 5002 
 5003 };
 5004 
 5005 class TagWord {
 5006  public:
 5007   int32_t _value;
 5008 
 5009   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5010 
 5011   void print() const {
 5012     printf("%04x", _value & 0xFFFF);
 5013   }
 5014 
 5015 };
 5016 
 5017 class FPU_Register {
 5018  public:
 5019   int32_t _m0;
 5020   int32_t _m1;
 5021   int16_t _ex;
 5022 
 5023   bool is_indefinite() const           {
 5024     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5025   }
 5026 
 5027   void print() const {
 5028     char  sign = (_ex < 0) ? '-' : '+';
 5029     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5030     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5031   };
 5032 
 5033 };
 5034 
 5035 class FPU_State {
 5036  public:
 5037   enum {
 5038     register_size       = 10,
 5039     number_of_registers =  8,
 5040     register_mask       =  7
 5041   };
 5042 
 5043   ControlWord  _control_word;
 5044   StatusWord   _status_word;
 5045   TagWord      _tag_word;
 5046   int32_t      _error_offset;
 5047   int32_t      _error_selector;
 5048   int32_t      _data_offset;
 5049   int32_t      _data_selector;
 5050   int8_t       _register[register_size * number_of_registers];
 5051 
 5052   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5053   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5054 
 5055   const char* tag_as_string(int tag) const {
 5056     switch (tag) {
 5057       case 0: return "valid";
 5058       case 1: return "zero";
 5059       case 2: return "special";
 5060       case 3: return "empty";
 5061     }
 5062     ShouldNotReachHere();
 5063     return nullptr;
 5064   }
 5065 
 5066   void print() const {
 5067     // print computation registers
 5068     { int t = _status_word.top();
 5069       for (int i = 0; i < number_of_registers; i++) {
 5070         int j = (i - t) & register_mask;
 5071         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5072         st(j)->print();
 5073         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5074       }
 5075     }
 5076     printf("\n");
 5077     // print control registers
 5078     printf("ctrl = "); _control_word.print(); printf("\n");
 5079     printf("stat = "); _status_word .print(); printf("\n");
 5080     printf("tags = "); _tag_word    .print(); printf("\n");
 5081   }
 5082 
 5083 };
 5084 
 5085 class Flag_Register {
 5086  public:
 5087   int32_t _value;
 5088 
 5089   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5090   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5091   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5092   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5093   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5094   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5095   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5096 
 5097   void print() const {
 5098     // flags
 5099     char f[8];
 5100     f[0] = (overflow       ()) ? 'O' : '-';
 5101     f[1] = (direction      ()) ? 'D' : '-';
 5102     f[2] = (sign           ()) ? 'S' : '-';
 5103     f[3] = (zero           ()) ? 'Z' : '-';
 5104     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5105     f[5] = (parity         ()) ? 'P' : '-';
 5106     f[6] = (carry          ()) ? 'C' : '-';
 5107     f[7] = '\x0';
 5108     // output
 5109     printf("%08x  flags = %s", _value, f);
 5110   }
 5111 
 5112 };
 5113 
 5114 class IU_Register {
 5115  public:
 5116   int32_t _value;
 5117 
 5118   void print() const {
 5119     printf("%08x  %11d", _value, _value);
 5120   }
 5121 
 5122 };
 5123 
 5124 class IU_State {
 5125  public:
 5126   Flag_Register _eflags;
 5127   IU_Register   _rdi;
 5128   IU_Register   _rsi;
 5129   IU_Register   _rbp;
 5130   IU_Register   _rsp;
 5131   IU_Register   _rbx;
 5132   IU_Register   _rdx;
 5133   IU_Register   _rcx;
 5134   IU_Register   _rax;
 5135 
 5136   void print() const {
 5137     // computation registers
 5138     printf("rax,  = "); _rax.print(); printf("\n");
 5139     printf("rbx,  = "); _rbx.print(); printf("\n");
 5140     printf("rcx  = "); _rcx.print(); printf("\n");
 5141     printf("rdx  = "); _rdx.print(); printf("\n");
 5142     printf("rdi  = "); _rdi.print(); printf("\n");
 5143     printf("rsi  = "); _rsi.print(); printf("\n");
 5144     printf("rbp,  = "); _rbp.print(); printf("\n");
 5145     printf("rsp  = "); _rsp.print(); printf("\n");
 5146     printf("\n");
 5147     // control registers
 5148     printf("flgs = "); _eflags.print(); printf("\n");
 5149   }
 5150 };
 5151 
 5152 
 5153 class CPU_State {
 5154  public:
 5155   FPU_State _fpu_state;
 5156   IU_State  _iu_state;
 5157 
 5158   void print() const {
 5159     printf("--------------------------------------------------\n");
 5160     _iu_state .print();
 5161     printf("\n");
 5162     _fpu_state.print();
 5163     printf("--------------------------------------------------\n");
 5164   }
 5165 
 5166 };
 5167 
 5168 
 5169 static void _print_CPU_state(CPU_State* state) {
 5170   state->print();
 5171 };
 5172 
 5173 
 5174 void MacroAssembler::print_CPU_state() {
 5175   push_CPU_state();
 5176   push(rsp);                // pass CPU state
 5177   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5178   addptr(rsp, wordSize);       // discard argument
 5179   pop_CPU_state();
 5180 }
 5181 
 5182 
 5183 #ifndef _LP64
 5184 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5185   static int counter = 0;
 5186   FPU_State* fs = &state->_fpu_state;
 5187   counter++;
 5188   // For leaf calls, only verify that the top few elements remain empty.
 5189   // We only need 1 empty at the top for C2 code.
 5190   if( stack_depth < 0 ) {
 5191     if( fs->tag_for_st(7) != 3 ) {
 5192       printf("FPR7 not empty\n");
 5193       state->print();
 5194       assert(false, "error");
 5195       return false;
 5196     }
 5197     return true;                // All other stack states do not matter
 5198   }
 5199 
 5200   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5201          "bad FPU control word");
 5202 
 5203   // compute stack depth
 5204   int i = 0;
 5205   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5206   int d = i;
 5207   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5208   // verify findings
 5209   if (i != FPU_State::number_of_registers) {
 5210     // stack not contiguous
 5211     printf("%s: stack not contiguous at ST%d\n", s, i);
 5212     state->print();
 5213     assert(false, "error");
 5214     return false;
 5215   }
 5216   // check if computed stack depth corresponds to expected stack depth
 5217   if (stack_depth < 0) {
 5218     // expected stack depth is -stack_depth or less
 5219     if (d > -stack_depth) {
 5220       // too many elements on the stack
 5221       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5222       state->print();
 5223       assert(false, "error");
 5224       return false;
 5225     }
 5226   } else {
 5227     // expected stack depth is stack_depth
 5228     if (d != stack_depth) {
 5229       // wrong stack depth
 5230       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5231       state->print();
 5232       assert(false, "error");
 5233       return false;
 5234     }
 5235   }
 5236   // everything is cool
 5237   return true;
 5238 }
 5239 
 5240 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5241   if (!VerifyFPU) return;
 5242   push_CPU_state();
 5243   push(rsp);                // pass CPU state
 5244   ExternalAddress msg((address) s);
 5245   // pass message string s
 5246   pushptr(msg.addr(), noreg);
 5247   push(stack_depth);        // pass stack depth
 5248   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5249   addptr(rsp, 3 * wordSize);   // discard arguments
 5250   // check for error
 5251   { Label L;
 5252     testl(rax, rax);
 5253     jcc(Assembler::notZero, L);
 5254     int3();                  // break if error condition
 5255     bind(L);
 5256   }
 5257   pop_CPU_state();
 5258 }
 5259 #endif // _LP64
 5260 
 5261 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5262   // Either restore the MXCSR register after returning from the JNI Call
 5263   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5264   if (VM_Version::supports_sse()) {
 5265     if (RestoreMXCSROnJNICalls) {
 5266       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5267     } else if (CheckJNICalls) {
 5268       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5269     }
 5270   }
 5271   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5272   vzeroupper();
 5273 
 5274 #ifndef _LP64
 5275   // Either restore the x87 floating pointer control word after returning
 5276   // from the JNI call or verify that it wasn't changed.
 5277   if (CheckJNICalls) {
 5278     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5279   }
 5280 #endif // _LP64
 5281 }
 5282 
 5283 // ((OopHandle)result).resolve();
 5284 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5285   assert_different_registers(result, tmp);
 5286 
 5287   // Only 64 bit platforms support GCs that require a tmp register
 5288   // Only IN_HEAP loads require a thread_tmp register
 5289   // OopHandle::resolve is an indirection like jobject.
 5290   access_load_at(T_OBJECT, IN_NATIVE,
 5291                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5292 }
 5293 
 5294 // ((WeakHandle)result).resolve();
 5295 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5296   assert_different_registers(rresult, rtmp);
 5297   Label resolved;
 5298 
 5299   // A null weak handle resolves to null.
 5300   cmpptr(rresult, 0);
 5301   jcc(Assembler::equal, resolved);
 5302 
 5303   // Only 64 bit platforms support GCs that require a tmp register
 5304   // Only IN_HEAP loads require a thread_tmp register
 5305   // WeakHandle::resolve is an indirection like jweak.
 5306   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5307                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5308   bind(resolved);
 5309 }
 5310 
 5311 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5312   // get mirror
 5313   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5314   load_method_holder(mirror, method);
 5315   movptr(mirror, Address(mirror, mirror_offset));
 5316   resolve_oop_handle(mirror, tmp);
 5317 }
 5318 
 5319 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5320   load_method_holder(rresult, rmethod);
 5321   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5322 }
 5323 
 5324 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5325   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5326   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5327   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5328 }
 5329 
 5330 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5331   assert_different_registers(src, tmp);
 5332   assert_different_registers(dst, tmp);
 5333 #ifdef _LP64
 5334   if (UseCompressedClassPointers) {
 5335     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5336     decode_klass_not_null(dst, tmp);
 5337   } else
 5338 #endif
 5339     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5340 }
 5341 
 5342 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5343   assert_different_registers(src, tmp);
 5344   assert_different_registers(dst, tmp);
 5345 #ifdef _LP64
 5346   if (UseCompressedClassPointers) {
 5347     encode_klass_not_null(src, tmp);
 5348     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5349   } else
 5350 #endif
 5351     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5352 }
 5353 
 5354 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5355                                     Register tmp1, Register thread_tmp) {
 5356   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5357   decorators = AccessInternal::decorator_fixup(decorators, type);
 5358   bool as_raw = (decorators & AS_RAW) != 0;
 5359   if (as_raw) {
 5360     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5361   } else {
 5362     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5363   }
 5364 }
 5365 
 5366 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5367                                      Register tmp1, Register tmp2, Register tmp3) {
 5368   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5369   decorators = AccessInternal::decorator_fixup(decorators, type);
 5370   bool as_raw = (decorators & AS_RAW) != 0;
 5371   if (as_raw) {
 5372     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5373   } else {
 5374     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5375   }
 5376 }
 5377 
 5378 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5379                                    Register thread_tmp, DecoratorSet decorators) {
 5380   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5381 }
 5382 
 5383 // Doesn't do verification, generates fixed size code
 5384 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5385                                             Register thread_tmp, DecoratorSet decorators) {
 5386   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5387 }
 5388 
 5389 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5390                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5391   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5392 }
 5393 
 5394 // Used for storing nulls.
 5395 void MacroAssembler::store_heap_oop_null(Address dst) {
 5396   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5397 }
 5398 
 5399 #ifdef _LP64
 5400 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 5401   if (UseCompressedClassPointers) {
 5402     // Store to klass gap in destination
 5403     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 5404   }
 5405 }
 5406 
 5407 #ifdef ASSERT
 5408 void MacroAssembler::verify_heapbase(const char* msg) {
 5409   assert (UseCompressedOops, "should be compressed");
 5410   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5411   if (CheckCompressedOops) {
 5412     Label ok;
 5413     ExternalAddress src2(CompressedOops::ptrs_base_addr());
 5414     const bool is_src2_reachable = reachable(src2);
 5415     if (!is_src2_reachable) {
 5416       push(rscratch1);  // cmpptr trashes rscratch1
 5417     }
 5418     cmpptr(r12_heapbase, src2, rscratch1);
 5419     jcc(Assembler::equal, ok);
 5420     STOP(msg);
 5421     bind(ok);
 5422     if (!is_src2_reachable) {
 5423       pop(rscratch1);
 5424     }
 5425   }
 5426 }
 5427 #endif
 5428 
 5429 // Algorithm must match oop.inline.hpp encode_heap_oop.
 5430 void MacroAssembler::encode_heap_oop(Register r) {
 5431 #ifdef ASSERT
 5432   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 5433 #endif
 5434   verify_oop_msg(r, "broken oop in encode_heap_oop");
 5435   if (CompressedOops::base() == nullptr) {
 5436     if (CompressedOops::shift() != 0) {
 5437       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5438       shrq(r, LogMinObjAlignmentInBytes);
 5439     }
 5440     return;
 5441   }
 5442   testq(r, r);
 5443   cmovq(Assembler::equal, r, r12_heapbase);
 5444   subq(r, r12_heapbase);
 5445   shrq(r, LogMinObjAlignmentInBytes);
 5446 }
 5447 
 5448 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 5449 #ifdef ASSERT
 5450   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 5451   if (CheckCompressedOops) {
 5452     Label ok;
 5453     testq(r, r);
 5454     jcc(Assembler::notEqual, ok);
 5455     STOP("null oop passed to encode_heap_oop_not_null");
 5456     bind(ok);
 5457   }
 5458 #endif
 5459   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 5460   if (CompressedOops::base() != nullptr) {
 5461     subq(r, r12_heapbase);
 5462   }
 5463   if (CompressedOops::shift() != 0) {
 5464     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5465     shrq(r, LogMinObjAlignmentInBytes);
 5466   }
 5467 }
 5468 
 5469 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 5470 #ifdef ASSERT
 5471   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 5472   if (CheckCompressedOops) {
 5473     Label ok;
 5474     testq(src, src);
 5475     jcc(Assembler::notEqual, ok);
 5476     STOP("null oop passed to encode_heap_oop_not_null2");
 5477     bind(ok);
 5478   }
 5479 #endif
 5480   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 5481   if (dst != src) {
 5482     movq(dst, src);
 5483   }
 5484   if (CompressedOops::base() != nullptr) {
 5485     subq(dst, r12_heapbase);
 5486   }
 5487   if (CompressedOops::shift() != 0) {
 5488     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5489     shrq(dst, LogMinObjAlignmentInBytes);
 5490   }
 5491 }
 5492 
 5493 void  MacroAssembler::decode_heap_oop(Register r) {
 5494 #ifdef ASSERT
 5495   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 5496 #endif
 5497   if (CompressedOops::base() == nullptr) {
 5498     if (CompressedOops::shift() != 0) {
 5499       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5500       shlq(r, LogMinObjAlignmentInBytes);
 5501     }
 5502   } else {
 5503     Label done;
 5504     shlq(r, LogMinObjAlignmentInBytes);
 5505     jccb(Assembler::equal, done);
 5506     addq(r, r12_heapbase);
 5507     bind(done);
 5508   }
 5509   verify_oop_msg(r, "broken oop in decode_heap_oop");
 5510 }
 5511 
 5512 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 5513   // Note: it will change flags
 5514   assert (UseCompressedOops, "should only be used for compressed headers");
 5515   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5516   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5517   // vtableStubs also counts instructions in pd_code_size_limit.
 5518   // Also do not verify_oop as this is called by verify_oop.
 5519   if (CompressedOops::shift() != 0) {
 5520     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5521     shlq(r, LogMinObjAlignmentInBytes);
 5522     if (CompressedOops::base() != nullptr) {
 5523       addq(r, r12_heapbase);
 5524     }
 5525   } else {
 5526     assert (CompressedOops::base() == nullptr, "sanity");
 5527   }
 5528 }
 5529 
 5530 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 5531   // Note: it will change flags
 5532   assert (UseCompressedOops, "should only be used for compressed headers");
 5533   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5534   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5535   // vtableStubs also counts instructions in pd_code_size_limit.
 5536   // Also do not verify_oop as this is called by verify_oop.
 5537   if (CompressedOops::shift() != 0) {
 5538     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5539     if (LogMinObjAlignmentInBytes == Address::times_8) {
 5540       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 5541     } else {
 5542       if (dst != src) {
 5543         movq(dst, src);
 5544       }
 5545       shlq(dst, LogMinObjAlignmentInBytes);
 5546       if (CompressedOops::base() != nullptr) {
 5547         addq(dst, r12_heapbase);
 5548       }
 5549     }
 5550   } else {
 5551     assert (CompressedOops::base() == nullptr, "sanity");
 5552     if (dst != src) {
 5553       movq(dst, src);
 5554     }
 5555   }
 5556 }
 5557 
 5558 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 5559   assert_different_registers(r, tmp);
 5560   if (CompressedKlassPointers::base() != nullptr) {
 5561     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5562     subq(r, tmp);
 5563   }
 5564   if (CompressedKlassPointers::shift() != 0) {
 5565     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5566     shrq(r, LogKlassAlignmentInBytes);
 5567   }
 5568 }
 5569 
 5570 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 5571   assert_different_registers(src, dst);
 5572   if (CompressedKlassPointers::base() != nullptr) {
 5573     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 5574     addq(dst, src);
 5575   } else {
 5576     movptr(dst, src);
 5577   }
 5578   if (CompressedKlassPointers::shift() != 0) {
 5579     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5580     shrq(dst, LogKlassAlignmentInBytes);
 5581   }
 5582 }
 5583 
 5584 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 5585   assert_different_registers(r, tmp);
 5586   // Note: it will change flags
 5587   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 5588   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5589   // vtableStubs also counts instructions in pd_code_size_limit.
 5590   // Also do not verify_oop as this is called by verify_oop.
 5591   if (CompressedKlassPointers::shift() != 0) {
 5592     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5593     shlq(r, LogKlassAlignmentInBytes);
 5594   }
 5595   if (CompressedKlassPointers::base() != nullptr) {
 5596     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5597     addq(r, tmp);
 5598   }
 5599 }
 5600 
 5601 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 5602   assert_different_registers(src, dst);
 5603   // Note: it will change flags
 5604   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5605   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5606   // vtableStubs also counts instructions in pd_code_size_limit.
 5607   // Also do not verify_oop as this is called by verify_oop.
 5608 
 5609   if (CompressedKlassPointers::base() == nullptr &&
 5610       CompressedKlassPointers::shift() == 0) {
 5611     // The best case scenario is that there is no base or shift. Then it is already
 5612     // a pointer that needs nothing but a register rename.
 5613     movl(dst, src);
 5614   } else {
 5615     if (CompressedKlassPointers::base() != nullptr) {
 5616       mov64(dst, (int64_t)CompressedKlassPointers::base());
 5617     } else {
 5618       xorq(dst, dst);
 5619     }
 5620     if (CompressedKlassPointers::shift() != 0) {
 5621       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5622       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 5623       leaq(dst, Address(dst, src, Address::times_8, 0));
 5624     } else {
 5625       addq(dst, src);
 5626     }
 5627   }
 5628 }
 5629 
 5630 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 5631   assert (UseCompressedOops, "should only be used for compressed headers");
 5632   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5633   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5634   int oop_index = oop_recorder()->find_index(obj);
 5635   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5636   mov_narrow_oop(dst, oop_index, rspec);
 5637 }
 5638 
 5639 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 5640   assert (UseCompressedOops, "should only be used for compressed headers");
 5641   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5642   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5643   int oop_index = oop_recorder()->find_index(obj);
 5644   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5645   mov_narrow_oop(dst, oop_index, rspec);
 5646 }
 5647 
 5648 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 5649   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5650   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5651   int klass_index = oop_recorder()->find_index(k);
 5652   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5653   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5654 }
 5655 
 5656 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 5657   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5658   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5659   int klass_index = oop_recorder()->find_index(k);
 5660   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5661   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5662 }
 5663 
 5664 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 5665   assert (UseCompressedOops, "should only be used for compressed headers");
 5666   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5667   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5668   int oop_index = oop_recorder()->find_index(obj);
 5669   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5670   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 5671 }
 5672 
 5673 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 5674   assert (UseCompressedOops, "should only be used for compressed headers");
 5675   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5676   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5677   int oop_index = oop_recorder()->find_index(obj);
 5678   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 5679   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 5680 }
 5681 
 5682 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 5683   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5684   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5685   int klass_index = oop_recorder()->find_index(k);
 5686   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5687   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5688 }
 5689 
 5690 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 5691   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5692   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 5693   int klass_index = oop_recorder()->find_index(k);
 5694   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5695   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5696 }
 5697 
 5698 void MacroAssembler::reinit_heapbase() {
 5699   if (UseCompressedOops) {
 5700     if (Universe::heap() != nullptr) {
 5701       if (CompressedOops::base() == nullptr) {
 5702         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 5703       } else {
 5704         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 5705       }
 5706     } else {
 5707       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 5708     }
 5709   }
 5710 }
 5711 
 5712 #endif // _LP64
 5713 
 5714 #if COMPILER2_OR_JVMCI
 5715 
 5716 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 5717 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 5718   // cnt - number of qwords (8-byte words).
 5719   // base - start address, qword aligned.
 5720   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 5721   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 5722   if (use64byteVector) {
 5723     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 5724   } else if (MaxVectorSize >= 32) {
 5725     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
 5726   } else {
 5727     pxor(xtmp, xtmp);
 5728   }
 5729   jmp(L_zero_64_bytes);
 5730 
 5731   BIND(L_loop);
 5732   if (MaxVectorSize >= 32) {
 5733     fill64(base, 0, xtmp, use64byteVector);
 5734   } else {
 5735     movdqu(Address(base,  0), xtmp);
 5736     movdqu(Address(base, 16), xtmp);
 5737     movdqu(Address(base, 32), xtmp);
 5738     movdqu(Address(base, 48), xtmp);
 5739   }
 5740   addptr(base, 64);
 5741 
 5742   BIND(L_zero_64_bytes);
 5743   subptr(cnt, 8);
 5744   jccb(Assembler::greaterEqual, L_loop);
 5745 
 5746   // Copy trailing 64 bytes
 5747   if (use64byteVector) {
 5748     addptr(cnt, 8);
 5749     jccb(Assembler::equal, L_end);
 5750     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 5751     jmp(L_end);
 5752   } else {
 5753     addptr(cnt, 4);
 5754     jccb(Assembler::less, L_tail);
 5755     if (MaxVectorSize >= 32) {
 5756       vmovdqu(Address(base, 0), xtmp);
 5757     } else {
 5758       movdqu(Address(base,  0), xtmp);
 5759       movdqu(Address(base, 16), xtmp);
 5760     }
 5761   }
 5762   addptr(base, 32);
 5763   subptr(cnt, 4);
 5764 
 5765   BIND(L_tail);
 5766   addptr(cnt, 4);
 5767   jccb(Assembler::lessEqual, L_end);
 5768   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 5769     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 5770   } else {
 5771     decrement(cnt);
 5772 
 5773     BIND(L_sloop);
 5774     movq(Address(base, 0), xtmp);
 5775     addptr(base, 8);
 5776     decrement(cnt);
 5777     jccb(Assembler::greaterEqual, L_sloop);
 5778   }
 5779   BIND(L_end);
 5780 }
 5781 
 5782 // Clearing constant sized memory using YMM/ZMM registers.
 5783 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 5784   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
 5785   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 5786 
 5787   int vector64_count = (cnt & (~0x7)) >> 3;
 5788   cnt = cnt & 0x7;
 5789   const int fill64_per_loop = 4;
 5790   const int max_unrolled_fill64 = 8;
 5791 
 5792   // 64 byte initialization loop.
 5793   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 5794   int start64 = 0;
 5795   if (vector64_count > max_unrolled_fill64) {
 5796     Label LOOP;
 5797     Register index = rtmp;
 5798 
 5799     start64 = vector64_count - (vector64_count % fill64_per_loop);
 5800 
 5801     movl(index, 0);
 5802     BIND(LOOP);
 5803     for (int i = 0; i < fill64_per_loop; i++) {
 5804       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 5805     }
 5806     addl(index, fill64_per_loop * 64);
 5807     cmpl(index, start64 * 64);
 5808     jccb(Assembler::less, LOOP);
 5809   }
 5810   for (int i = start64; i < vector64_count; i++) {
 5811     fill64(base, i * 64, xtmp, use64byteVector);
 5812   }
 5813 
 5814   // Clear remaining 64 byte tail.
 5815   int disp = vector64_count * 64;
 5816   if (cnt) {
 5817     switch (cnt) {
 5818       case 1:
 5819         movq(Address(base, disp), xtmp);
 5820         break;
 5821       case 2:
 5822         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 5823         break;
 5824       case 3:
 5825         movl(rtmp, 0x7);
 5826         kmovwl(mask, rtmp);
 5827         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 5828         break;
 5829       case 4:
 5830         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 5831         break;
 5832       case 5:
 5833         if (use64byteVector) {
 5834           movl(rtmp, 0x1F);
 5835           kmovwl(mask, rtmp);
 5836           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 5837         } else {
 5838           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 5839           movq(Address(base, disp + 32), xtmp);
 5840         }
 5841         break;
 5842       case 6:
 5843         if (use64byteVector) {
 5844           movl(rtmp, 0x3F);
 5845           kmovwl(mask, rtmp);
 5846           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 5847         } else {
 5848           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 5849           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 5850         }
 5851         break;
 5852       case 7:
 5853         if (use64byteVector) {
 5854           movl(rtmp, 0x7F);
 5855           kmovwl(mask, rtmp);
 5856           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 5857         } else {
 5858           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 5859           movl(rtmp, 0x7);
 5860           kmovwl(mask, rtmp);
 5861           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 5862         }
 5863         break;
 5864       default:
 5865         fatal("Unexpected length : %d\n",cnt);
 5866         break;
 5867     }
 5868   }
 5869 }
 5870 
 5871 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 5872                                bool is_large, KRegister mask) {
 5873   // cnt      - number of qwords (8-byte words).
 5874   // base     - start address, qword aligned.
 5875   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 5876   assert(base==rdi, "base register must be edi for rep stos");
 5877   assert(tmp==rax,   "tmp register must be eax for rep stos");
 5878   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 5879   assert(InitArrayShortSize % BytesPerLong == 0,
 5880     "InitArrayShortSize should be the multiple of BytesPerLong");
 5881 
 5882   Label DONE;
 5883   if (!is_large || !UseXMMForObjInit) {
 5884     xorptr(tmp, tmp);
 5885   }
 5886 
 5887   if (!is_large) {
 5888     Label LOOP, LONG;
 5889     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 5890     jccb(Assembler::greater, LONG);
 5891 
 5892     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 5893 
 5894     decrement(cnt);
 5895     jccb(Assembler::negative, DONE); // Zero length
 5896 
 5897     // Use individual pointer-sized stores for small counts:
 5898     BIND(LOOP);
 5899     movptr(Address(base, cnt, Address::times_ptr), tmp);
 5900     decrement(cnt);
 5901     jccb(Assembler::greaterEqual, LOOP);
 5902     jmpb(DONE);
 5903 
 5904     BIND(LONG);
 5905   }
 5906 
 5907   // Use longer rep-prefixed ops for non-small counts:
 5908   if (UseFastStosb) {
 5909     shlptr(cnt, 3); // convert to number of bytes
 5910     rep_stosb();
 5911   } else if (UseXMMForObjInit) {
 5912     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 5913   } else {
 5914     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 5915     rep_stos();
 5916   }
 5917 
 5918   BIND(DONE);
 5919 }
 5920 
 5921 #endif //COMPILER2_OR_JVMCI
 5922 
 5923 
 5924 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 5925                                    Register to, Register value, Register count,
 5926                                    Register rtmp, XMMRegister xtmp) {
 5927   ShortBranchVerifier sbv(this);
 5928   assert_different_registers(to, value, count, rtmp);
 5929   Label L_exit;
 5930   Label L_fill_2_bytes, L_fill_4_bytes;
 5931 
 5932 #if defined(COMPILER2) && defined(_LP64)
 5933   if(MaxVectorSize >=32 &&
 5934      VM_Version::supports_avx512vlbw() &&
 5935      VM_Version::supports_bmi2()) {
 5936     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 5937     return;
 5938   }
 5939 #endif
 5940 
 5941   int shift = -1;
 5942   switch (t) {
 5943     case T_BYTE:
 5944       shift = 2;
 5945       break;
 5946     case T_SHORT:
 5947       shift = 1;
 5948       break;
 5949     case T_INT:
 5950       shift = 0;
 5951       break;
 5952     default: ShouldNotReachHere();
 5953   }
 5954 
 5955   if (t == T_BYTE) {
 5956     andl(value, 0xff);
 5957     movl(rtmp, value);
 5958     shll(rtmp, 8);
 5959     orl(value, rtmp);
 5960   }
 5961   if (t == T_SHORT) {
 5962     andl(value, 0xffff);
 5963   }
 5964   if (t == T_BYTE || t == T_SHORT) {
 5965     movl(rtmp, value);
 5966     shll(rtmp, 16);
 5967     orl(value, rtmp);
 5968   }
 5969 
 5970   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 5971   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 5972   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 5973     Label L_skip_align2;
 5974     // align source address at 4 bytes address boundary
 5975     if (t == T_BYTE) {
 5976       Label L_skip_align1;
 5977       // One byte misalignment happens only for byte arrays
 5978       testptr(to, 1);
 5979       jccb(Assembler::zero, L_skip_align1);
 5980       movb(Address(to, 0), value);
 5981       increment(to);
 5982       decrement(count);
 5983       BIND(L_skip_align1);
 5984     }
 5985     // Two bytes misalignment happens only for byte and short (char) arrays
 5986     testptr(to, 2);
 5987     jccb(Assembler::zero, L_skip_align2);
 5988     movw(Address(to, 0), value);
 5989     addptr(to, 2);
 5990     subl(count, 1<<(shift-1));
 5991     BIND(L_skip_align2);
 5992   }
 5993   if (UseSSE < 2) {
 5994     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 5995     // Fill 32-byte chunks
 5996     subl(count, 8 << shift);
 5997     jcc(Assembler::less, L_check_fill_8_bytes);
 5998     align(16);
 5999 
 6000     BIND(L_fill_32_bytes_loop);
 6001 
 6002     for (int i = 0; i < 32; i += 4) {
 6003       movl(Address(to, i), value);
 6004     }
 6005 
 6006     addptr(to, 32);
 6007     subl(count, 8 << shift);
 6008     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6009     BIND(L_check_fill_8_bytes);
 6010     addl(count, 8 << shift);
 6011     jccb(Assembler::zero, L_exit);
 6012     jmpb(L_fill_8_bytes);
 6013 
 6014     //
 6015     // length is too short, just fill qwords
 6016     //
 6017     BIND(L_fill_8_bytes_loop);
 6018     movl(Address(to, 0), value);
 6019     movl(Address(to, 4), value);
 6020     addptr(to, 8);
 6021     BIND(L_fill_8_bytes);
 6022     subl(count, 1 << (shift + 1));
 6023     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6024     // fall through to fill 4 bytes
 6025   } else {
 6026     Label L_fill_32_bytes;
 6027     if (!UseUnalignedLoadStores) {
 6028       // align to 8 bytes, we know we are 4 byte aligned to start
 6029       testptr(to, 4);
 6030       jccb(Assembler::zero, L_fill_32_bytes);
 6031       movl(Address(to, 0), value);
 6032       addptr(to, 4);
 6033       subl(count, 1<<shift);
 6034     }
 6035     BIND(L_fill_32_bytes);
 6036     {
 6037       assert( UseSSE >= 2, "supported cpu only" );
 6038       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6039       movdl(xtmp, value);
 6040       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6041         Label L_check_fill_32_bytes;
 6042         if (UseAVX > 2) {
 6043           // Fill 64-byte chunks
 6044           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6045 
 6046           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6047           cmpl(count, VM_Version::avx3_threshold());
 6048           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6049 
 6050           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6051 
 6052           subl(count, 16 << shift);
 6053           jccb(Assembler::less, L_check_fill_32_bytes);
 6054           align(16);
 6055 
 6056           BIND(L_fill_64_bytes_loop_avx3);
 6057           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6058           addptr(to, 64);
 6059           subl(count, 16 << shift);
 6060           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6061           jmpb(L_check_fill_32_bytes);
 6062 
 6063           BIND(L_check_fill_64_bytes_avx2);
 6064         }
 6065         // Fill 64-byte chunks
 6066         Label L_fill_64_bytes_loop;
 6067         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6068 
 6069         subl(count, 16 << shift);
 6070         jcc(Assembler::less, L_check_fill_32_bytes);
 6071         align(16);
 6072 
 6073         BIND(L_fill_64_bytes_loop);
 6074         vmovdqu(Address(to, 0), xtmp);
 6075         vmovdqu(Address(to, 32), xtmp);
 6076         addptr(to, 64);
 6077         subl(count, 16 << shift);
 6078         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6079 
 6080         BIND(L_check_fill_32_bytes);
 6081         addl(count, 8 << shift);
 6082         jccb(Assembler::less, L_check_fill_8_bytes);
 6083         vmovdqu(Address(to, 0), xtmp);
 6084         addptr(to, 32);
 6085         subl(count, 8 << shift);
 6086 
 6087         BIND(L_check_fill_8_bytes);
 6088         // clean upper bits of YMM registers
 6089         movdl(xtmp, value);
 6090         pshufd(xtmp, xtmp, 0);
 6091       } else {
 6092         // Fill 32-byte chunks
 6093         pshufd(xtmp, xtmp, 0);
 6094 
 6095         subl(count, 8 << shift);
 6096         jcc(Assembler::less, L_check_fill_8_bytes);
 6097         align(16);
 6098 
 6099         BIND(L_fill_32_bytes_loop);
 6100 
 6101         if (UseUnalignedLoadStores) {
 6102           movdqu(Address(to, 0), xtmp);
 6103           movdqu(Address(to, 16), xtmp);
 6104         } else {
 6105           movq(Address(to, 0), xtmp);
 6106           movq(Address(to, 8), xtmp);
 6107           movq(Address(to, 16), xtmp);
 6108           movq(Address(to, 24), xtmp);
 6109         }
 6110 
 6111         addptr(to, 32);
 6112         subl(count, 8 << shift);
 6113         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6114 
 6115         BIND(L_check_fill_8_bytes);
 6116       }
 6117       addl(count, 8 << shift);
 6118       jccb(Assembler::zero, L_exit);
 6119       jmpb(L_fill_8_bytes);
 6120 
 6121       //
 6122       // length is too short, just fill qwords
 6123       //
 6124       BIND(L_fill_8_bytes_loop);
 6125       movq(Address(to, 0), xtmp);
 6126       addptr(to, 8);
 6127       BIND(L_fill_8_bytes);
 6128       subl(count, 1 << (shift + 1));
 6129       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6130     }
 6131   }
 6132   // fill trailing 4 bytes
 6133   BIND(L_fill_4_bytes);
 6134   testl(count, 1<<shift);
 6135   jccb(Assembler::zero, L_fill_2_bytes);
 6136   movl(Address(to, 0), value);
 6137   if (t == T_BYTE || t == T_SHORT) {
 6138     Label L_fill_byte;
 6139     addptr(to, 4);
 6140     BIND(L_fill_2_bytes);
 6141     // fill trailing 2 bytes
 6142     testl(count, 1<<(shift-1));
 6143     jccb(Assembler::zero, L_fill_byte);
 6144     movw(Address(to, 0), value);
 6145     if (t == T_BYTE) {
 6146       addptr(to, 2);
 6147       BIND(L_fill_byte);
 6148       // fill trailing byte
 6149       testl(count, 1);
 6150       jccb(Assembler::zero, L_exit);
 6151       movb(Address(to, 0), value);
 6152     } else {
 6153       BIND(L_fill_byte);
 6154     }
 6155   } else {
 6156     BIND(L_fill_2_bytes);
 6157   }
 6158   BIND(L_exit);
 6159 }
 6160 
 6161 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6162   switch(type) {
 6163     case T_BYTE:
 6164     case T_BOOLEAN:
 6165       evpbroadcastb(dst, src, vector_len);
 6166       break;
 6167     case T_SHORT:
 6168     case T_CHAR:
 6169       evpbroadcastw(dst, src, vector_len);
 6170       break;
 6171     case T_INT:
 6172     case T_FLOAT:
 6173       evpbroadcastd(dst, src, vector_len);
 6174       break;
 6175     case T_LONG:
 6176     case T_DOUBLE:
 6177       evpbroadcastq(dst, src, vector_len);
 6178       break;
 6179     default:
 6180       fatal("Unhandled type : %s", type2name(type));
 6181       break;
 6182   }
 6183 }
 6184 
 6185 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6186    //@IntrinsicCandidate
 6187    //private static int implEncodeISOArray(byte[] sa, int sp,
 6188    //byte[] da, int dp, int len) {
 6189    //  int i = 0;
 6190    //  for (; i < len; i++) {
 6191    //    char c = StringUTF16.getChar(sa, sp++);
 6192    //    if (c > '\u00FF')
 6193    //      break;
 6194    //    da[dp++] = (byte)c;
 6195    //  }
 6196    //  return i;
 6197    //}
 6198    //
 6199    //@IntrinsicCandidate
 6200    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6201    //    byte[] da, int dp, int len) {
 6202    //  int i = 0;
 6203    //  for (; i < len; i++) {
 6204    //    char c = sa[sp++];
 6205    //    if (c >= '\u0080')
 6206    //      break;
 6207    //    da[dp++] = (byte)c;
 6208    //  }
 6209    //  return i;
 6210    //}
 6211 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6212   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6213   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6214   Register tmp5, Register result, bool ascii) {
 6215 
 6216   // rsi: src
 6217   // rdi: dst
 6218   // rdx: len
 6219   // rcx: tmp5
 6220   // rax: result
 6221   ShortBranchVerifier sbv(this);
 6222   assert_different_registers(src, dst, len, tmp5, result);
 6223   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6224 
 6225   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6226   int short_mask = ascii ? 0xff80 : 0xff00;
 6227 
 6228   // set result
 6229   xorl(result, result);
 6230   // check for zero length
 6231   testl(len, len);
 6232   jcc(Assembler::zero, L_done);
 6233 
 6234   movl(result, len);
 6235 
 6236   // Setup pointers
 6237   lea(src, Address(src, len, Address::times_2)); // char[]
 6238   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6239   negptr(len);
 6240 
 6241   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6242     Label L_copy_8_chars, L_copy_8_chars_exit;
 6243     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6244 
 6245     if (UseAVX >= 2) {
 6246       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6247       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6248       movdl(tmp1Reg, tmp5);
 6249       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 6250       jmp(L_chars_32_check);
 6251 
 6252       bind(L_copy_32_chars);
 6253       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 6254       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 6255       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6256       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6257       jccb(Assembler::notZero, L_copy_32_chars_exit);
 6258       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 6259       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 6260       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 6261 
 6262       bind(L_chars_32_check);
 6263       addptr(len, 32);
 6264       jcc(Assembler::lessEqual, L_copy_32_chars);
 6265 
 6266       bind(L_copy_32_chars_exit);
 6267       subptr(len, 16);
 6268       jccb(Assembler::greater, L_copy_16_chars_exit);
 6269 
 6270     } else if (UseSSE42Intrinsics) {
 6271       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6272       movdl(tmp1Reg, tmp5);
 6273       pshufd(tmp1Reg, tmp1Reg, 0);
 6274       jmpb(L_chars_16_check);
 6275     }
 6276 
 6277     bind(L_copy_16_chars);
 6278     if (UseAVX >= 2) {
 6279       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 6280       vptest(tmp2Reg, tmp1Reg);
 6281       jcc(Assembler::notZero, L_copy_16_chars_exit);
 6282       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 6283       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 6284     } else {
 6285       if (UseAVX > 0) {
 6286         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6287         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6288         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 6289       } else {
 6290         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 6291         por(tmp2Reg, tmp3Reg);
 6292         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 6293         por(tmp2Reg, tmp4Reg);
 6294       }
 6295       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 6296       jccb(Assembler::notZero, L_copy_16_chars_exit);
 6297       packuswb(tmp3Reg, tmp4Reg);
 6298     }
 6299     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 6300 
 6301     bind(L_chars_16_check);
 6302     addptr(len, 16);
 6303     jcc(Assembler::lessEqual, L_copy_16_chars);
 6304 
 6305     bind(L_copy_16_chars_exit);
 6306     if (UseAVX >= 2) {
 6307       // clean upper bits of YMM registers
 6308       vpxor(tmp2Reg, tmp2Reg);
 6309       vpxor(tmp3Reg, tmp3Reg);
 6310       vpxor(tmp4Reg, tmp4Reg);
 6311       movdl(tmp1Reg, tmp5);
 6312       pshufd(tmp1Reg, tmp1Reg, 0);
 6313     }
 6314     subptr(len, 8);
 6315     jccb(Assembler::greater, L_copy_8_chars_exit);
 6316 
 6317     bind(L_copy_8_chars);
 6318     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 6319     ptest(tmp3Reg, tmp1Reg);
 6320     jccb(Assembler::notZero, L_copy_8_chars_exit);
 6321     packuswb(tmp3Reg, tmp1Reg);
 6322     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 6323     addptr(len, 8);
 6324     jccb(Assembler::lessEqual, L_copy_8_chars);
 6325 
 6326     bind(L_copy_8_chars_exit);
 6327     subptr(len, 8);
 6328     jccb(Assembler::zero, L_done);
 6329   }
 6330 
 6331   bind(L_copy_1_char);
 6332   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 6333   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 6334   jccb(Assembler::notZero, L_copy_1_char_exit);
 6335   movb(Address(dst, len, Address::times_1, 0), tmp5);
 6336   addptr(len, 1);
 6337   jccb(Assembler::less, L_copy_1_char);
 6338 
 6339   bind(L_copy_1_char_exit);
 6340   addptr(result, len); // len is negative count of not processed elements
 6341 
 6342   bind(L_done);
 6343 }
 6344 
 6345 #ifdef _LP64
 6346 /**
 6347  * Helper for multiply_to_len().
 6348  */
 6349 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 6350   addq(dest_lo, src1);
 6351   adcq(dest_hi, 0);
 6352   addq(dest_lo, src2);
 6353   adcq(dest_hi, 0);
 6354 }
 6355 
 6356 /**
 6357  * Multiply 64 bit by 64 bit first loop.
 6358  */
 6359 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 6360                                            Register y, Register y_idx, Register z,
 6361                                            Register carry, Register product,
 6362                                            Register idx, Register kdx) {
 6363   //
 6364   //  jlong carry, x[], y[], z[];
 6365   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 6366   //    huge_128 product = y[idx] * x[xstart] + carry;
 6367   //    z[kdx] = (jlong)product;
 6368   //    carry  = (jlong)(product >>> 64);
 6369   //  }
 6370   //  z[xstart] = carry;
 6371   //
 6372 
 6373   Label L_first_loop, L_first_loop_exit;
 6374   Label L_one_x, L_one_y, L_multiply;
 6375 
 6376   decrementl(xstart);
 6377   jcc(Assembler::negative, L_one_x);
 6378 
 6379   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 6380   rorq(x_xstart, 32); // convert big-endian to little-endian
 6381 
 6382   bind(L_first_loop);
 6383   decrementl(idx);
 6384   jcc(Assembler::negative, L_first_loop_exit);
 6385   decrementl(idx);
 6386   jcc(Assembler::negative, L_one_y);
 6387   movq(y_idx, Address(y, idx, Address::times_4,  0));
 6388   rorq(y_idx, 32); // convert big-endian to little-endian
 6389   bind(L_multiply);
 6390   movq(product, x_xstart);
 6391   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 6392   addq(product, carry);
 6393   adcq(rdx, 0);
 6394   subl(kdx, 2);
 6395   movl(Address(z, kdx, Address::times_4,  4), product);
 6396   shrq(product, 32);
 6397   movl(Address(z, kdx, Address::times_4,  0), product);
 6398   movq(carry, rdx);
 6399   jmp(L_first_loop);
 6400 
 6401   bind(L_one_y);
 6402   movl(y_idx, Address(y,  0));
 6403   jmp(L_multiply);
 6404 
 6405   bind(L_one_x);
 6406   movl(x_xstart, Address(x,  0));
 6407   jmp(L_first_loop);
 6408 
 6409   bind(L_first_loop_exit);
 6410 }
 6411 
 6412 /**
 6413  * Multiply 64 bit by 64 bit and add 128 bit.
 6414  */
 6415 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 6416                                             Register yz_idx, Register idx,
 6417                                             Register carry, Register product, int offset) {
 6418   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 6419   //     z[kdx] = (jlong)product;
 6420 
 6421   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 6422   rorq(yz_idx, 32); // convert big-endian to little-endian
 6423   movq(product, x_xstart);
 6424   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 6425   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 6426   rorq(yz_idx, 32); // convert big-endian to little-endian
 6427 
 6428   add2_with_carry(rdx, product, carry, yz_idx);
 6429 
 6430   movl(Address(z, idx, Address::times_4,  offset+4), product);
 6431   shrq(product, 32);
 6432   movl(Address(z, idx, Address::times_4,  offset), product);
 6433 
 6434 }
 6435 
 6436 /**
 6437  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 6438  */
 6439 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 6440                                              Register yz_idx, Register idx, Register jdx,
 6441                                              Register carry, Register product,
 6442                                              Register carry2) {
 6443   //   jlong carry, x[], y[], z[];
 6444   //   int kdx = ystart+1;
 6445   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6446   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 6447   //     z[kdx+idx+1] = (jlong)product;
 6448   //     jlong carry2  = (jlong)(product >>> 64);
 6449   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 6450   //     z[kdx+idx] = (jlong)product;
 6451   //     carry  = (jlong)(product >>> 64);
 6452   //   }
 6453   //   idx += 2;
 6454   //   if (idx > 0) {
 6455   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 6456   //     z[kdx+idx] = (jlong)product;
 6457   //     carry  = (jlong)(product >>> 64);
 6458   //   }
 6459   //
 6460 
 6461   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6462 
 6463   movl(jdx, idx);
 6464   andl(jdx, 0xFFFFFFFC);
 6465   shrl(jdx, 2);
 6466 
 6467   bind(L_third_loop);
 6468   subl(jdx, 1);
 6469   jcc(Assembler::negative, L_third_loop_exit);
 6470   subl(idx, 4);
 6471 
 6472   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 6473   movq(carry2, rdx);
 6474 
 6475   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 6476   movq(carry, rdx);
 6477   jmp(L_third_loop);
 6478 
 6479   bind (L_third_loop_exit);
 6480 
 6481   andl (idx, 0x3);
 6482   jcc(Assembler::zero, L_post_third_loop_done);
 6483 
 6484   Label L_check_1;
 6485   subl(idx, 2);
 6486   jcc(Assembler::negative, L_check_1);
 6487 
 6488   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 6489   movq(carry, rdx);
 6490 
 6491   bind (L_check_1);
 6492   addl (idx, 0x2);
 6493   andl (idx, 0x1);
 6494   subl(idx, 1);
 6495   jcc(Assembler::negative, L_post_third_loop_done);
 6496 
 6497   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 6498   movq(product, x_xstart);
 6499   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 6500   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 6501 
 6502   add2_with_carry(rdx, product, yz_idx, carry);
 6503 
 6504   movl(Address(z, idx, Address::times_4,  0), product);
 6505   shrq(product, 32);
 6506 
 6507   shlq(rdx, 32);
 6508   orq(product, rdx);
 6509   movq(carry, product);
 6510 
 6511   bind(L_post_third_loop_done);
 6512 }
 6513 
 6514 /**
 6515  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 6516  *
 6517  */
 6518 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 6519                                                   Register carry, Register carry2,
 6520                                                   Register idx, Register jdx,
 6521                                                   Register yz_idx1, Register yz_idx2,
 6522                                                   Register tmp, Register tmp3, Register tmp4) {
 6523   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 6524 
 6525   //   jlong carry, x[], y[], z[];
 6526   //   int kdx = ystart+1;
 6527   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 6528   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 6529   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 6530   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 6531   //     carry  = (jlong)(tmp4 >>> 64);
 6532   //     z[kdx+idx+1] = (jlong)tmp3;
 6533   //     z[kdx+idx] = (jlong)tmp4;
 6534   //   }
 6535   //   idx += 2;
 6536   //   if (idx > 0) {
 6537   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 6538   //     z[kdx+idx] = (jlong)yz_idx1;
 6539   //     carry  = (jlong)(yz_idx1 >>> 64);
 6540   //   }
 6541   //
 6542 
 6543   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 6544 
 6545   movl(jdx, idx);
 6546   andl(jdx, 0xFFFFFFFC);
 6547   shrl(jdx, 2);
 6548 
 6549   bind(L_third_loop);
 6550   subl(jdx, 1);
 6551   jcc(Assembler::negative, L_third_loop_exit);
 6552   subl(idx, 4);
 6553 
 6554   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 6555   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 6556   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 6557   rorxq(yz_idx2, yz_idx2, 32);
 6558 
 6559   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 6560   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 6561 
 6562   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 6563   rorxq(yz_idx1, yz_idx1, 32);
 6564   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6565   rorxq(yz_idx2, yz_idx2, 32);
 6566 
 6567   if (VM_Version::supports_adx()) {
 6568     adcxq(tmp3, carry);
 6569     adoxq(tmp3, yz_idx1);
 6570 
 6571     adcxq(tmp4, tmp);
 6572     adoxq(tmp4, yz_idx2);
 6573 
 6574     movl(carry, 0); // does not affect flags
 6575     adcxq(carry2, carry);
 6576     adoxq(carry2, carry);
 6577   } else {
 6578     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 6579     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 6580   }
 6581   movq(carry, carry2);
 6582 
 6583   movl(Address(z, idx, Address::times_4, 12), tmp3);
 6584   shrq(tmp3, 32);
 6585   movl(Address(z, idx, Address::times_4,  8), tmp3);
 6586 
 6587   movl(Address(z, idx, Address::times_4,  4), tmp4);
 6588   shrq(tmp4, 32);
 6589   movl(Address(z, idx, Address::times_4,  0), tmp4);
 6590 
 6591   jmp(L_third_loop);
 6592 
 6593   bind (L_third_loop_exit);
 6594 
 6595   andl (idx, 0x3);
 6596   jcc(Assembler::zero, L_post_third_loop_done);
 6597 
 6598   Label L_check_1;
 6599   subl(idx, 2);
 6600   jcc(Assembler::negative, L_check_1);
 6601 
 6602   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 6603   rorxq(yz_idx1, yz_idx1, 32);
 6604   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 6605   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 6606   rorxq(yz_idx2, yz_idx2, 32);
 6607 
 6608   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 6609 
 6610   movl(Address(z, idx, Address::times_4,  4), tmp3);
 6611   shrq(tmp3, 32);
 6612   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6613   movq(carry, tmp4);
 6614 
 6615   bind (L_check_1);
 6616   addl (idx, 0x2);
 6617   andl (idx, 0x1);
 6618   subl(idx, 1);
 6619   jcc(Assembler::negative, L_post_third_loop_done);
 6620   movl(tmp4, Address(y, idx, Address::times_4,  0));
 6621   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 6622   movl(tmp4, Address(z, idx, Address::times_4,  0));
 6623 
 6624   add2_with_carry(carry2, tmp3, tmp4, carry);
 6625 
 6626   movl(Address(z, idx, Address::times_4,  0), tmp3);
 6627   shrq(tmp3, 32);
 6628 
 6629   shlq(carry2, 32);
 6630   orq(tmp3, carry2);
 6631   movq(carry, tmp3);
 6632 
 6633   bind(L_post_third_loop_done);
 6634 }
 6635 
 6636 /**
 6637  * Code for BigInteger::multiplyToLen() intrinsic.
 6638  *
 6639  * rdi: x
 6640  * rax: xlen
 6641  * rsi: y
 6642  * rcx: ylen
 6643  * r8:  z
 6644  * r11: zlen
 6645  * r12: tmp1
 6646  * r13: tmp2
 6647  * r14: tmp3
 6648  * r15: tmp4
 6649  * rbx: tmp5
 6650  *
 6651  */
 6652 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
 6653                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 6654   ShortBranchVerifier sbv(this);
 6655   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 6656 
 6657   push(tmp1);
 6658   push(tmp2);
 6659   push(tmp3);
 6660   push(tmp4);
 6661   push(tmp5);
 6662 
 6663   push(xlen);
 6664   push(zlen);
 6665 
 6666   const Register idx = tmp1;
 6667   const Register kdx = tmp2;
 6668   const Register xstart = tmp3;
 6669 
 6670   const Register y_idx = tmp4;
 6671   const Register carry = tmp5;
 6672   const Register product  = xlen;
 6673   const Register x_xstart = zlen;  // reuse register
 6674 
 6675   // First Loop.
 6676   //
 6677   //  final static long LONG_MASK = 0xffffffffL;
 6678   //  int xstart = xlen - 1;
 6679   //  int ystart = ylen - 1;
 6680   //  long carry = 0;
 6681   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 6682   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 6683   //    z[kdx] = (int)product;
 6684   //    carry = product >>> 32;
 6685   //  }
 6686   //  z[xstart] = (int)carry;
 6687   //
 6688 
 6689   movl(idx, ylen);      // idx = ylen;
 6690   movl(kdx, zlen);      // kdx = xlen+ylen;
 6691   xorq(carry, carry);   // carry = 0;
 6692 
 6693   Label L_done;
 6694 
 6695   movl(xstart, xlen);
 6696   decrementl(xstart);
 6697   jcc(Assembler::negative, L_done);
 6698 
 6699   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 6700 
 6701   Label L_second_loop;
 6702   testl(kdx, kdx);
 6703   jcc(Assembler::zero, L_second_loop);
 6704 
 6705   Label L_carry;
 6706   subl(kdx, 1);
 6707   jcc(Assembler::zero, L_carry);
 6708 
 6709   movl(Address(z, kdx, Address::times_4,  0), carry);
 6710   shrq(carry, 32);
 6711   subl(kdx, 1);
 6712 
 6713   bind(L_carry);
 6714   movl(Address(z, kdx, Address::times_4,  0), carry);
 6715 
 6716   // Second and third (nested) loops.
 6717   //
 6718   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 6719   //   carry = 0;
 6720   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 6721   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 6722   //                    (z[k] & LONG_MASK) + carry;
 6723   //     z[k] = (int)product;
 6724   //     carry = product >>> 32;
 6725   //   }
 6726   //   z[i] = (int)carry;
 6727   // }
 6728   //
 6729   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 6730 
 6731   const Register jdx = tmp1;
 6732 
 6733   bind(L_second_loop);
 6734   xorl(carry, carry);    // carry = 0;
 6735   movl(jdx, ylen);       // j = ystart+1
 6736 
 6737   subl(xstart, 1);       // i = xstart-1;
 6738   jcc(Assembler::negative, L_done);
 6739 
 6740   push (z);
 6741 
 6742   Label L_last_x;
 6743   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 6744   subl(xstart, 1);       // i = xstart-1;
 6745   jcc(Assembler::negative, L_last_x);
 6746 
 6747   if (UseBMI2Instructions) {
 6748     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 6749     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 6750   } else {
 6751     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 6752     rorq(x_xstart, 32);  // convert big-endian to little-endian
 6753   }
 6754 
 6755   Label L_third_loop_prologue;
 6756   bind(L_third_loop_prologue);
 6757 
 6758   push (x);
 6759   push (xstart);
 6760   push (ylen);
 6761 
 6762 
 6763   if (UseBMI2Instructions) {
 6764     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 6765   } else { // !UseBMI2Instructions
 6766     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 6767   }
 6768 
 6769   pop(ylen);
 6770   pop(xlen);
 6771   pop(x);
 6772   pop(z);
 6773 
 6774   movl(tmp3, xlen);
 6775   addl(tmp3, 1);
 6776   movl(Address(z, tmp3, Address::times_4,  0), carry);
 6777   subl(tmp3, 1);
 6778   jccb(Assembler::negative, L_done);
 6779 
 6780   shrq(carry, 32);
 6781   movl(Address(z, tmp3, Address::times_4,  0), carry);
 6782   jmp(L_second_loop);
 6783 
 6784   // Next infrequent code is moved outside loops.
 6785   bind(L_last_x);
 6786   if (UseBMI2Instructions) {
 6787     movl(rdx, Address(x,  0));
 6788   } else {
 6789     movl(x_xstart, Address(x,  0));
 6790   }
 6791   jmp(L_third_loop_prologue);
 6792 
 6793   bind(L_done);
 6794 
 6795   pop(zlen);
 6796   pop(xlen);
 6797 
 6798   pop(tmp5);
 6799   pop(tmp4);
 6800   pop(tmp3);
 6801   pop(tmp2);
 6802   pop(tmp1);
 6803 }
 6804 
 6805 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 6806   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 6807   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 6808   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 6809   Label VECTOR8_TAIL, VECTOR4_TAIL;
 6810   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 6811   Label SAME_TILL_END, DONE;
 6812   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 6813 
 6814   //scale is in rcx in both Win64 and Unix
 6815   ShortBranchVerifier sbv(this);
 6816 
 6817   shlq(length);
 6818   xorq(result, result);
 6819 
 6820   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 6821       VM_Version::supports_avx512vlbw()) {
 6822     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 6823 
 6824     cmpq(length, 64);
 6825     jcc(Assembler::less, VECTOR32_TAIL);
 6826 
 6827     movq(tmp1, length);
 6828     andq(tmp1, 0x3F);      // tail count
 6829     andq(length, ~(0x3F)); //vector count
 6830 
 6831     bind(VECTOR64_LOOP);
 6832     // AVX512 code to compare 64 byte vectors.
 6833     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 6834     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 6835     kortestql(k7, k7);
 6836     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 6837     addq(result, 64);
 6838     subq(length, 64);
 6839     jccb(Assembler::notZero, VECTOR64_LOOP);
 6840 
 6841     //bind(VECTOR64_TAIL);
 6842     testq(tmp1, tmp1);
 6843     jcc(Assembler::zero, SAME_TILL_END);
 6844 
 6845     //bind(VECTOR64_TAIL);
 6846     // AVX512 code to compare up to 63 byte vectors.
 6847     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 6848     shlxq(tmp2, tmp2, tmp1);
 6849     notq(tmp2);
 6850     kmovql(k3, tmp2);
 6851 
 6852     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 6853     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 6854 
 6855     ktestql(k7, k3);
 6856     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 6857 
 6858     bind(VECTOR64_NOT_EQUAL);
 6859     kmovql(tmp1, k7);
 6860     notq(tmp1);
 6861     tzcntq(tmp1, tmp1);
 6862     addq(result, tmp1);
 6863     shrq(result);
 6864     jmp(DONE);
 6865     bind(VECTOR32_TAIL);
 6866   }
 6867 
 6868   cmpq(length, 8);
 6869   jcc(Assembler::equal, VECTOR8_LOOP);
 6870   jcc(Assembler::less, VECTOR4_TAIL);
 6871 
 6872   if (UseAVX >= 2) {
 6873     Label VECTOR16_TAIL, VECTOR32_LOOP;
 6874 
 6875     cmpq(length, 16);
 6876     jcc(Assembler::equal, VECTOR16_LOOP);
 6877     jcc(Assembler::less, VECTOR8_LOOP);
 6878 
 6879     cmpq(length, 32);
 6880     jccb(Assembler::less, VECTOR16_TAIL);
 6881 
 6882     subq(length, 32);
 6883     bind(VECTOR32_LOOP);
 6884     vmovdqu(rymm0, Address(obja, result));
 6885     vmovdqu(rymm1, Address(objb, result));
 6886     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 6887     vptest(rymm2, rymm2);
 6888     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 6889     addq(result, 32);
 6890     subq(length, 32);
 6891     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 6892     addq(length, 32);
 6893     jcc(Assembler::equal, SAME_TILL_END);
 6894     //falling through if less than 32 bytes left //close the branch here.
 6895 
 6896     bind(VECTOR16_TAIL);
 6897     cmpq(length, 16);
 6898     jccb(Assembler::less, VECTOR8_TAIL);
 6899     bind(VECTOR16_LOOP);
 6900     movdqu(rymm0, Address(obja, result));
 6901     movdqu(rymm1, Address(objb, result));
 6902     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 6903     ptest(rymm2, rymm2);
 6904     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 6905     addq(result, 16);
 6906     subq(length, 16);
 6907     jcc(Assembler::equal, SAME_TILL_END);
 6908     //falling through if less than 16 bytes left
 6909   } else {//regular intrinsics
 6910 
 6911     cmpq(length, 16);
 6912     jccb(Assembler::less, VECTOR8_TAIL);
 6913 
 6914     subq(length, 16);
 6915     bind(VECTOR16_LOOP);
 6916     movdqu(rymm0, Address(obja, result));
 6917     movdqu(rymm1, Address(objb, result));
 6918     pxor(rymm0, rymm1);
 6919     ptest(rymm0, rymm0);
 6920     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 6921     addq(result, 16);
 6922     subq(length, 16);
 6923     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 6924     addq(length, 16);
 6925     jcc(Assembler::equal, SAME_TILL_END);
 6926     //falling through if less than 16 bytes left
 6927   }
 6928 
 6929   bind(VECTOR8_TAIL);
 6930   cmpq(length, 8);
 6931   jccb(Assembler::less, VECTOR4_TAIL);
 6932   bind(VECTOR8_LOOP);
 6933   movq(tmp1, Address(obja, result));
 6934   movq(tmp2, Address(objb, result));
 6935   xorq(tmp1, tmp2);
 6936   testq(tmp1, tmp1);
 6937   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 6938   addq(result, 8);
 6939   subq(length, 8);
 6940   jcc(Assembler::equal, SAME_TILL_END);
 6941   //falling through if less than 8 bytes left
 6942 
 6943   bind(VECTOR4_TAIL);
 6944   cmpq(length, 4);
 6945   jccb(Assembler::less, BYTES_TAIL);
 6946   bind(VECTOR4_LOOP);
 6947   movl(tmp1, Address(obja, result));
 6948   xorl(tmp1, Address(objb, result));
 6949   testl(tmp1, tmp1);
 6950   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 6951   addq(result, 4);
 6952   subq(length, 4);
 6953   jcc(Assembler::equal, SAME_TILL_END);
 6954   //falling through if less than 4 bytes left
 6955 
 6956   bind(BYTES_TAIL);
 6957   bind(BYTES_LOOP);
 6958   load_unsigned_byte(tmp1, Address(obja, result));
 6959   load_unsigned_byte(tmp2, Address(objb, result));
 6960   xorl(tmp1, tmp2);
 6961   testl(tmp1, tmp1);
 6962   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 6963   decq(length);
 6964   jcc(Assembler::zero, SAME_TILL_END);
 6965   incq(result);
 6966   load_unsigned_byte(tmp1, Address(obja, result));
 6967   load_unsigned_byte(tmp2, Address(objb, result));
 6968   xorl(tmp1, tmp2);
 6969   testl(tmp1, tmp1);
 6970   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 6971   decq(length);
 6972   jcc(Assembler::zero, SAME_TILL_END);
 6973   incq(result);
 6974   load_unsigned_byte(tmp1, Address(obja, result));
 6975   load_unsigned_byte(tmp2, Address(objb, result));
 6976   xorl(tmp1, tmp2);
 6977   testl(tmp1, tmp1);
 6978   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 6979   jmp(SAME_TILL_END);
 6980 
 6981   if (UseAVX >= 2) {
 6982     bind(VECTOR32_NOT_EQUAL);
 6983     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 6984     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 6985     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 6986     vpmovmskb(tmp1, rymm0);
 6987     bsfq(tmp1, tmp1);
 6988     addq(result, tmp1);
 6989     shrq(result);
 6990     jmp(DONE);
 6991   }
 6992 
 6993   bind(VECTOR16_NOT_EQUAL);
 6994   if (UseAVX >= 2) {
 6995     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 6996     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 6997     pxor(rymm0, rymm2);
 6998   } else {
 6999     pcmpeqb(rymm2, rymm2);
 7000     pxor(rymm0, rymm1);
 7001     pcmpeqb(rymm0, rymm1);
 7002     pxor(rymm0, rymm2);
 7003   }
 7004   pmovmskb(tmp1, rymm0);
 7005   bsfq(tmp1, tmp1);
 7006   addq(result, tmp1);
 7007   shrq(result);
 7008   jmpb(DONE);
 7009 
 7010   bind(VECTOR8_NOT_EQUAL);
 7011   bind(VECTOR4_NOT_EQUAL);
 7012   bsfq(tmp1, tmp1);
 7013   shrq(tmp1, 3);
 7014   addq(result, tmp1);
 7015   bind(BYTES_NOT_EQUAL);
 7016   shrq(result);
 7017   jmpb(DONE);
 7018 
 7019   bind(SAME_TILL_END);
 7020   mov64(result, -1);
 7021 
 7022   bind(DONE);
 7023 }
 7024 
 7025 //Helper functions for square_to_len()
 7026 
 7027 /**
 7028  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7029  * Preserves x and z and modifies rest of the registers.
 7030  */
 7031 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7032   // Perform square and right shift by 1
 7033   // Handle odd xlen case first, then for even xlen do the following
 7034   // jlong carry = 0;
 7035   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7036   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7037   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7038   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7039   //     carry = (jlong)product;
 7040   // }
 7041 
 7042   xorq(tmp5, tmp5);     // carry
 7043   xorq(rdxReg, rdxReg);
 7044   xorl(tmp1, tmp1);     // index for x
 7045   xorl(tmp4, tmp4);     // index for z
 7046 
 7047   Label L_first_loop, L_first_loop_exit;
 7048 
 7049   testl(xlen, 1);
 7050   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7051 
 7052   // Square and right shift by 1 the odd element using 32 bit multiply
 7053   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7054   imulq(raxReg, raxReg);
 7055   shrq(raxReg, 1);
 7056   adcq(tmp5, 0);
 7057   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7058   incrementl(tmp1);
 7059   addl(tmp4, 2);
 7060 
 7061   // Square and  right shift by 1 the rest using 64 bit multiply
 7062   bind(L_first_loop);
 7063   cmpptr(tmp1, xlen);
 7064   jccb(Assembler::equal, L_first_loop_exit);
 7065 
 7066   // Square
 7067   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7068   rorq(raxReg, 32);    // convert big-endian to little-endian
 7069   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7070 
 7071   // Right shift by 1 and save carry
 7072   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7073   rcrq(rdxReg, 1);
 7074   rcrq(raxReg, 1);
 7075   adcq(tmp5, 0);
 7076 
 7077   // Store result in z
 7078   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7079   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7080 
 7081   // Update indices for x and z
 7082   addl(tmp1, 2);
 7083   addl(tmp4, 4);
 7084   jmp(L_first_loop);
 7085 
 7086   bind(L_first_loop_exit);
 7087 }
 7088 
 7089 
 7090 /**
 7091  * Perform the following multiply add operation using BMI2 instructions
 7092  * carry:sum = sum + op1*op2 + carry
 7093  * op2 should be in rdx
 7094  * op2 is preserved, all other registers are modified
 7095  */
 7096 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7097   // assert op2 is rdx
 7098   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7099   addq(sum, carry);
 7100   adcq(tmp2, 0);
 7101   addq(sum, op1);
 7102   adcq(tmp2, 0);
 7103   movq(carry, tmp2);
 7104 }
 7105 
 7106 /**
 7107  * Perform the following multiply add operation:
 7108  * carry:sum = sum + op1*op2 + carry
 7109  * Preserves op1, op2 and modifies rest of registers
 7110  */
 7111 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7112   // rdx:rax = op1 * op2
 7113   movq(raxReg, op2);
 7114   mulq(op1);
 7115 
 7116   //  rdx:rax = sum + carry + rdx:rax
 7117   addq(sum, carry);
 7118   adcq(rdxReg, 0);
 7119   addq(sum, raxReg);
 7120   adcq(rdxReg, 0);
 7121 
 7122   // carry:sum = rdx:sum
 7123   movq(carry, rdxReg);
 7124 }
 7125 
 7126 /**
 7127  * Add 64 bit long carry into z[] with carry propagation.
 7128  * Preserves z and carry register values and modifies rest of registers.
 7129  *
 7130  */
 7131 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7132   Label L_fourth_loop, L_fourth_loop_exit;
 7133 
 7134   movl(tmp1, 1);
 7135   subl(zlen, 2);
 7136   addq(Address(z, zlen, Address::times_4, 0), carry);
 7137 
 7138   bind(L_fourth_loop);
 7139   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7140   subl(zlen, 2);
 7141   jccb(Assembler::negative, L_fourth_loop_exit);
 7142   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7143   jmp(L_fourth_loop);
 7144   bind(L_fourth_loop_exit);
 7145 }
 7146 
 7147 /**
 7148  * Shift z[] left by 1 bit.
 7149  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7150  *
 7151  */
 7152 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7153 
 7154   Label L_fifth_loop, L_fifth_loop_exit;
 7155 
 7156   // Fifth loop
 7157   // Perform primitiveLeftShift(z, zlen, 1)
 7158 
 7159   const Register prev_carry = tmp1;
 7160   const Register new_carry = tmp4;
 7161   const Register value = tmp2;
 7162   const Register zidx = tmp3;
 7163 
 7164   // int zidx, carry;
 7165   // long value;
 7166   // carry = 0;
 7167   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7168   //    (carry:value)  = (z[i] << 1) | carry ;
 7169   //    z[i] = value;
 7170   // }
 7171 
 7172   movl(zidx, zlen);
 7173   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7174 
 7175   bind(L_fifth_loop);
 7176   decl(zidx);  // Use decl to preserve carry flag
 7177   decl(zidx);
 7178   jccb(Assembler::negative, L_fifth_loop_exit);
 7179 
 7180   if (UseBMI2Instructions) {
 7181      movq(value, Address(z, zidx, Address::times_4, 0));
 7182      rclq(value, 1);
 7183      rorxq(value, value, 32);
 7184      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7185   }
 7186   else {
 7187     // clear new_carry
 7188     xorl(new_carry, new_carry);
 7189 
 7190     // Shift z[i] by 1, or in previous carry and save new carry
 7191     movq(value, Address(z, zidx, Address::times_4, 0));
 7192     shlq(value, 1);
 7193     adcl(new_carry, 0);
 7194 
 7195     orq(value, prev_carry);
 7196     rorq(value, 0x20);
 7197     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7198 
 7199     // Set previous carry = new carry
 7200     movl(prev_carry, new_carry);
 7201   }
 7202   jmp(L_fifth_loop);
 7203 
 7204   bind(L_fifth_loop_exit);
 7205 }
 7206 
 7207 
 7208 /**
 7209  * Code for BigInteger::squareToLen() intrinsic
 7210  *
 7211  * rdi: x
 7212  * rsi: len
 7213  * r8:  z
 7214  * rcx: zlen
 7215  * r12: tmp1
 7216  * r13: tmp2
 7217  * r14: tmp3
 7218  * r15: tmp4
 7219  * rbx: tmp5
 7220  *
 7221  */
 7222 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7223 
 7224   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7225   push(tmp1);
 7226   push(tmp2);
 7227   push(tmp3);
 7228   push(tmp4);
 7229   push(tmp5);
 7230 
 7231   // First loop
 7232   // Store the squares, right shifted one bit (i.e., divided by 2).
 7233   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7234 
 7235   // Add in off-diagonal sums.
 7236   //
 7237   // Second, third (nested) and fourth loops.
 7238   // zlen +=2;
 7239   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7240   //    carry = 0;
 7241   //    long op2 = x[xidx:xidx+1];
 7242   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7243   //       k -= 2;
 7244   //       long op1 = x[j:j+1];
 7245   //       long sum = z[k:k+1];
 7246   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7247   //       z[k:k+1] = sum;
 7248   //    }
 7249   //    add_one_64(z, k, carry, tmp_regs);
 7250   // }
 7251 
 7252   const Register carry = tmp5;
 7253   const Register sum = tmp3;
 7254   const Register op1 = tmp4;
 7255   Register op2 = tmp2;
 7256 
 7257   push(zlen);
 7258   push(len);
 7259   addl(zlen,2);
 7260   bind(L_second_loop);
 7261   xorq(carry, carry);
 7262   subl(zlen, 4);
 7263   subl(len, 2);
 7264   push(zlen);
 7265   push(len);
 7266   cmpl(len, 0);
 7267   jccb(Assembler::lessEqual, L_second_loop_exit);
 7268 
 7269   // Multiply an array by one 64 bit long.
 7270   if (UseBMI2Instructions) {
 7271     op2 = rdxReg;
 7272     movq(op2, Address(x, len, Address::times_4,  0));
 7273     rorxq(op2, op2, 32);
 7274   }
 7275   else {
 7276     movq(op2, Address(x, len, Address::times_4,  0));
 7277     rorq(op2, 32);
 7278   }
 7279 
 7280   bind(L_third_loop);
 7281   decrementl(len);
 7282   jccb(Assembler::negative, L_third_loop_exit);
 7283   decrementl(len);
 7284   jccb(Assembler::negative, L_last_x);
 7285 
 7286   movq(op1, Address(x, len, Address::times_4,  0));
 7287   rorq(op1, 32);
 7288 
 7289   bind(L_multiply);
 7290   subl(zlen, 2);
 7291   movq(sum, Address(z, zlen, Address::times_4,  0));
 7292 
 7293   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 7294   if (UseBMI2Instructions) {
 7295     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 7296   }
 7297   else {
 7298     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7299   }
 7300 
 7301   movq(Address(z, zlen, Address::times_4, 0), sum);
 7302 
 7303   jmp(L_third_loop);
 7304   bind(L_third_loop_exit);
 7305 
 7306   // Fourth loop
 7307   // Add 64 bit long carry into z with carry propagation.
 7308   // Uses offsetted zlen.
 7309   add_one_64(z, zlen, carry, tmp1);
 7310 
 7311   pop(len);
 7312   pop(zlen);
 7313   jmp(L_second_loop);
 7314 
 7315   // Next infrequent code is moved outside loops.
 7316   bind(L_last_x);
 7317   movl(op1, Address(x, 0));
 7318   jmp(L_multiply);
 7319 
 7320   bind(L_second_loop_exit);
 7321   pop(len);
 7322   pop(zlen);
 7323   pop(len);
 7324   pop(zlen);
 7325 
 7326   // Fifth loop
 7327   // Shift z left 1 bit.
 7328   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 7329 
 7330   // z[zlen-1] |= x[len-1] & 1;
 7331   movl(tmp3, Address(x, len, Address::times_4, -4));
 7332   andl(tmp3, 1);
 7333   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 7334 
 7335   pop(tmp5);
 7336   pop(tmp4);
 7337   pop(tmp3);
 7338   pop(tmp2);
 7339   pop(tmp1);
 7340 }
 7341 
 7342 /**
 7343  * Helper function for mul_add()
 7344  * Multiply the in[] by int k and add to out[] starting at offset offs using
 7345  * 128 bit by 32 bit multiply and return the carry in tmp5.
 7346  * Only quad int aligned length of in[] is operated on in this function.
 7347  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 7348  * This function preserves out, in and k registers.
 7349  * len and offset point to the appropriate index in "in" & "out" correspondingly
 7350  * tmp5 has the carry.
 7351  * other registers are temporary and are modified.
 7352  *
 7353  */
 7354 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 7355   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 7356   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7357 
 7358   Label L_first_loop, L_first_loop_exit;
 7359 
 7360   movl(tmp1, len);
 7361   shrl(tmp1, 2);
 7362 
 7363   bind(L_first_loop);
 7364   subl(tmp1, 1);
 7365   jccb(Assembler::negative, L_first_loop_exit);
 7366 
 7367   subl(len, 4);
 7368   subl(offset, 4);
 7369 
 7370   Register op2 = tmp2;
 7371   const Register sum = tmp3;
 7372   const Register op1 = tmp4;
 7373   const Register carry = tmp5;
 7374 
 7375   if (UseBMI2Instructions) {
 7376     op2 = rdxReg;
 7377   }
 7378 
 7379   movq(op1, Address(in, len, Address::times_4,  8));
 7380   rorq(op1, 32);
 7381   movq(sum, Address(out, offset, Address::times_4,  8));
 7382   rorq(sum, 32);
 7383   if (UseBMI2Instructions) {
 7384     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7385   }
 7386   else {
 7387     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7388   }
 7389   // Store back in big endian from little endian
 7390   rorq(sum, 0x20);
 7391   movq(Address(out, offset, Address::times_4,  8), sum);
 7392 
 7393   movq(op1, Address(in, len, Address::times_4,  0));
 7394   rorq(op1, 32);
 7395   movq(sum, Address(out, offset, Address::times_4,  0));
 7396   rorq(sum, 32);
 7397   if (UseBMI2Instructions) {
 7398     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7399   }
 7400   else {
 7401     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7402   }
 7403   // Store back in big endian from little endian
 7404   rorq(sum, 0x20);
 7405   movq(Address(out, offset, Address::times_4,  0), sum);
 7406 
 7407   jmp(L_first_loop);
 7408   bind(L_first_loop_exit);
 7409 }
 7410 
 7411 /**
 7412  * Code for BigInteger::mulAdd() intrinsic
 7413  *
 7414  * rdi: out
 7415  * rsi: in
 7416  * r11: offs (out.length - offset)
 7417  * rcx: len
 7418  * r8:  k
 7419  * r12: tmp1
 7420  * r13: tmp2
 7421  * r14: tmp3
 7422  * r15: tmp4
 7423  * rbx: tmp5
 7424  * Multiply the in[] by word k and add to out[], return the carry in rax
 7425  */
 7426 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 7427    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 7428    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7429 
 7430   Label L_carry, L_last_in, L_done;
 7431 
 7432 // carry = 0;
 7433 // for (int j=len-1; j >= 0; j--) {
 7434 //    long product = (in[j] & LONG_MASK) * kLong +
 7435 //                   (out[offs] & LONG_MASK) + carry;
 7436 //    out[offs--] = (int)product;
 7437 //    carry = product >>> 32;
 7438 // }
 7439 //
 7440   push(tmp1);
 7441   push(tmp2);
 7442   push(tmp3);
 7443   push(tmp4);
 7444   push(tmp5);
 7445 
 7446   Register op2 = tmp2;
 7447   const Register sum = tmp3;
 7448   const Register op1 = tmp4;
 7449   const Register carry =  tmp5;
 7450 
 7451   if (UseBMI2Instructions) {
 7452     op2 = rdxReg;
 7453     movl(op2, k);
 7454   }
 7455   else {
 7456     movl(op2, k);
 7457   }
 7458 
 7459   xorq(carry, carry);
 7460 
 7461   //First loop
 7462 
 7463   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 7464   //The carry is in tmp5
 7465   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7466 
 7467   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 7468   decrementl(len);
 7469   jccb(Assembler::negative, L_carry);
 7470   decrementl(len);
 7471   jccb(Assembler::negative, L_last_in);
 7472 
 7473   movq(op1, Address(in, len, Address::times_4,  0));
 7474   rorq(op1, 32);
 7475 
 7476   subl(offs, 2);
 7477   movq(sum, Address(out, offs, Address::times_4,  0));
 7478   rorq(sum, 32);
 7479 
 7480   if (UseBMI2Instructions) {
 7481     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 7482   }
 7483   else {
 7484     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 7485   }
 7486 
 7487   // Store back in big endian from little endian
 7488   rorq(sum, 0x20);
 7489   movq(Address(out, offs, Address::times_4,  0), sum);
 7490 
 7491   testl(len, len);
 7492   jccb(Assembler::zero, L_carry);
 7493 
 7494   //Multiply the last in[] entry, if any
 7495   bind(L_last_in);
 7496   movl(op1, Address(in, 0));
 7497   movl(sum, Address(out, offs, Address::times_4,  -4));
 7498 
 7499   movl(raxReg, k);
 7500   mull(op1); //tmp4 * eax -> edx:eax
 7501   addl(sum, carry);
 7502   adcl(rdxReg, 0);
 7503   addl(sum, raxReg);
 7504   adcl(rdxReg, 0);
 7505   movl(carry, rdxReg);
 7506 
 7507   movl(Address(out, offs, Address::times_4,  -4), sum);
 7508 
 7509   bind(L_carry);
 7510   //return tmp5/carry as carry in rax
 7511   movl(rax, carry);
 7512 
 7513   bind(L_done);
 7514   pop(tmp5);
 7515   pop(tmp4);
 7516   pop(tmp3);
 7517   pop(tmp2);
 7518   pop(tmp1);
 7519 }
 7520 #endif
 7521 
 7522 /**
 7523  * Emits code to update CRC-32 with a byte value according to constants in table
 7524  *
 7525  * @param [in,out]crc   Register containing the crc.
 7526  * @param [in]val       Register containing the byte to fold into the CRC.
 7527  * @param [in]table     Register containing the table of crc constants.
 7528  *
 7529  * uint32_t crc;
 7530  * val = crc_table[(val ^ crc) & 0xFF];
 7531  * crc = val ^ (crc >> 8);
 7532  *
 7533  */
 7534 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 7535   xorl(val, crc);
 7536   andl(val, 0xFF);
 7537   shrl(crc, 8); // unsigned shift
 7538   xorl(crc, Address(table, val, Address::times_4, 0));
 7539 }
 7540 
 7541 /**
 7542  * Fold 128-bit data chunk
 7543  */
 7544 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 7545   if (UseAVX > 0) {
 7546     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 7547     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 7548     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 7549     pxor(xcrc, xtmp);
 7550   } else {
 7551     movdqa(xtmp, xcrc);
 7552     pclmulhdq(xtmp, xK);   // [123:64]
 7553     pclmulldq(xcrc, xK);   // [63:0]
 7554     pxor(xcrc, xtmp);
 7555     movdqu(xtmp, Address(buf, offset));
 7556     pxor(xcrc, xtmp);
 7557   }
 7558 }
 7559 
 7560 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 7561   if (UseAVX > 0) {
 7562     vpclmulhdq(xtmp, xK, xcrc);
 7563     vpclmulldq(xcrc, xK, xcrc);
 7564     pxor(xcrc, xbuf);
 7565     pxor(xcrc, xtmp);
 7566   } else {
 7567     movdqa(xtmp, xcrc);
 7568     pclmulhdq(xtmp, xK);
 7569     pclmulldq(xcrc, xK);
 7570     pxor(xcrc, xbuf);
 7571     pxor(xcrc, xtmp);
 7572   }
 7573 }
 7574 
 7575 /**
 7576  * 8-bit folds to compute 32-bit CRC
 7577  *
 7578  * uint64_t xcrc;
 7579  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 7580  */
 7581 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 7582   movdl(tmp, xcrc);
 7583   andl(tmp, 0xFF);
 7584   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 7585   psrldq(xcrc, 1); // unsigned shift one byte
 7586   pxor(xcrc, xtmp);
 7587 }
 7588 
 7589 /**
 7590  * uint32_t crc;
 7591  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 7592  */
 7593 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 7594   movl(tmp, crc);
 7595   andl(tmp, 0xFF);
 7596   shrl(crc, 8);
 7597   xorl(crc, Address(table, tmp, Address::times_4, 0));
 7598 }
 7599 
 7600 /**
 7601  * @param crc   register containing existing CRC (32-bit)
 7602  * @param buf   register pointing to input byte buffer (byte*)
 7603  * @param len   register containing number of bytes
 7604  * @param table register that will contain address of CRC table
 7605  * @param tmp   scratch register
 7606  */
 7607 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 7608   assert_different_registers(crc, buf, len, table, tmp, rax);
 7609 
 7610   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 7611   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 7612 
 7613   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 7614   // context for the registers used, where all instructions below are using 128-bit mode
 7615   // On EVEX without VL and BW, these instructions will all be AVX.
 7616   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 7617   notl(crc); // ~crc
 7618   cmpl(len, 16);
 7619   jcc(Assembler::less, L_tail);
 7620 
 7621   // Align buffer to 16 bytes
 7622   movl(tmp, buf);
 7623   andl(tmp, 0xF);
 7624   jccb(Assembler::zero, L_aligned);
 7625   subl(tmp,  16);
 7626   addl(len, tmp);
 7627 
 7628   align(4);
 7629   BIND(L_align_loop);
 7630   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 7631   update_byte_crc32(crc, rax, table);
 7632   increment(buf);
 7633   incrementl(tmp);
 7634   jccb(Assembler::less, L_align_loop);
 7635 
 7636   BIND(L_aligned);
 7637   movl(tmp, len); // save
 7638   shrl(len, 4);
 7639   jcc(Assembler::zero, L_tail_restore);
 7640 
 7641   // Fold crc into first bytes of vector
 7642   movdqa(xmm1, Address(buf, 0));
 7643   movdl(rax, xmm1);
 7644   xorl(crc, rax);
 7645   if (VM_Version::supports_sse4_1()) {
 7646     pinsrd(xmm1, crc, 0);
 7647   } else {
 7648     pinsrw(xmm1, crc, 0);
 7649     shrl(crc, 16);
 7650     pinsrw(xmm1, crc, 1);
 7651   }
 7652   addptr(buf, 16);
 7653   subl(len, 4); // len > 0
 7654   jcc(Assembler::less, L_fold_tail);
 7655 
 7656   movdqa(xmm2, Address(buf,  0));
 7657   movdqa(xmm3, Address(buf, 16));
 7658   movdqa(xmm4, Address(buf, 32));
 7659   addptr(buf, 48);
 7660   subl(len, 3);
 7661   jcc(Assembler::lessEqual, L_fold_512b);
 7662 
 7663   // Fold total 512 bits of polynomial on each iteration,
 7664   // 128 bits per each of 4 parallel streams.
 7665   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 7666 
 7667   align32();
 7668   BIND(L_fold_512b_loop);
 7669   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 7670   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 7671   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 7672   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 7673   addptr(buf, 64);
 7674   subl(len, 4);
 7675   jcc(Assembler::greater, L_fold_512b_loop);
 7676 
 7677   // Fold 512 bits to 128 bits.
 7678   BIND(L_fold_512b);
 7679   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 7680   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 7681   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 7682   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 7683 
 7684   // Fold the rest of 128 bits data chunks
 7685   BIND(L_fold_tail);
 7686   addl(len, 3);
 7687   jccb(Assembler::lessEqual, L_fold_128b);
 7688   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 7689 
 7690   BIND(L_fold_tail_loop);
 7691   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 7692   addptr(buf, 16);
 7693   decrementl(len);
 7694   jccb(Assembler::greater, L_fold_tail_loop);
 7695 
 7696   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 7697   BIND(L_fold_128b);
 7698   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 7699   if (UseAVX > 0) {
 7700     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 7701     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 7702     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 7703   } else {
 7704     movdqa(xmm2, xmm0);
 7705     pclmulqdq(xmm2, xmm1, 0x1);
 7706     movdqa(xmm3, xmm0);
 7707     pand(xmm3, xmm2);
 7708     pclmulqdq(xmm0, xmm3, 0x1);
 7709   }
 7710   psrldq(xmm1, 8);
 7711   psrldq(xmm2, 4);
 7712   pxor(xmm0, xmm1);
 7713   pxor(xmm0, xmm2);
 7714 
 7715   // 8 8-bit folds to compute 32-bit CRC.
 7716   for (int j = 0; j < 4; j++) {
 7717     fold_8bit_crc32(xmm0, table, xmm1, rax);
 7718   }
 7719   movdl(crc, xmm0); // mov 32 bits to general register
 7720   for (int j = 0; j < 4; j++) {
 7721     fold_8bit_crc32(crc, table, rax);
 7722   }
 7723 
 7724   BIND(L_tail_restore);
 7725   movl(len, tmp); // restore
 7726   BIND(L_tail);
 7727   andl(len, 0xf);
 7728   jccb(Assembler::zero, L_exit);
 7729 
 7730   // Fold the rest of bytes
 7731   align(4);
 7732   BIND(L_tail_loop);
 7733   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 7734   update_byte_crc32(crc, rax, table);
 7735   increment(buf);
 7736   decrementl(len);
 7737   jccb(Assembler::greater, L_tail_loop);
 7738 
 7739   BIND(L_exit);
 7740   notl(crc); // ~c
 7741 }
 7742 
 7743 #ifdef _LP64
 7744 // Helper function for AVX 512 CRC32
 7745 // Fold 512-bit data chunks
 7746 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 7747                                              Register pos, int offset) {
 7748   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 7749   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 7750   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 7751   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 7752   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 7753 }
 7754 
 7755 // Helper function for AVX 512 CRC32
 7756 // Compute CRC32 for < 256B buffers
 7757 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 7758                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 7759                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 7760 
 7761   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 7762   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 7763   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 7764 
 7765   // check if there is enough buffer to be able to fold 16B at a time
 7766   cmpl(len, 32);
 7767   jcc(Assembler::less, L_less_than_32);
 7768 
 7769   // if there is, load the constants
 7770   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 7771   movdl(xmm0, crc);                        // get the initial crc value
 7772   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 7773   pxor(xmm7, xmm0);
 7774 
 7775   // update the buffer pointer
 7776   addl(pos, 16);
 7777   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 7778   subl(len, 32);
 7779   jmp(L_16B_reduction_loop);
 7780 
 7781   bind(L_less_than_32);
 7782   //mov initial crc to the return value. this is necessary for zero - length buffers.
 7783   movl(rax, crc);
 7784   testl(len, len);
 7785   jcc(Assembler::equal, L_cleanup);
 7786 
 7787   movdl(xmm0, crc);                        //get the initial crc value
 7788 
 7789   cmpl(len, 16);
 7790   jcc(Assembler::equal, L_exact_16_left);
 7791   jcc(Assembler::less, L_less_than_16_left);
 7792 
 7793   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 7794   pxor(xmm7, xmm0);                       //xor the initial crc value
 7795   addl(pos, 16);
 7796   subl(len, 16);
 7797   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 7798   jmp(L_get_last_two_xmms);
 7799 
 7800   bind(L_less_than_16_left);
 7801   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 7802   pxor(xmm1, xmm1);
 7803   movptr(tmp1, rsp);
 7804   movdqu(Address(tmp1, 0 * 16), xmm1);
 7805 
 7806   cmpl(len, 4);
 7807   jcc(Assembler::less, L_only_less_than_4);
 7808 
 7809   //backup the counter value
 7810   movl(tmp2, len);
 7811   cmpl(len, 8);
 7812   jcc(Assembler::less, L_less_than_8_left);
 7813 
 7814   //load 8 Bytes
 7815   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 7816   movq(Address(tmp1, 0 * 16), rax);
 7817   addptr(tmp1, 8);
 7818   subl(len, 8);
 7819   addl(pos, 8);
 7820 
 7821   bind(L_less_than_8_left);
 7822   cmpl(len, 4);
 7823   jcc(Assembler::less, L_less_than_4_left);
 7824 
 7825   //load 4 Bytes
 7826   movl(rax, Address(buf, pos, Address::times_1, 0));
 7827   movl(Address(tmp1, 0 * 16), rax);
 7828   addptr(tmp1, 4);
 7829   subl(len, 4);
 7830   addl(pos, 4);
 7831 
 7832   bind(L_less_than_4_left);
 7833   cmpl(len, 2);
 7834   jcc(Assembler::less, L_less_than_2_left);
 7835 
 7836   // load 2 Bytes
 7837   movw(rax, Address(buf, pos, Address::times_1, 0));
 7838   movl(Address(tmp1, 0 * 16), rax);
 7839   addptr(tmp1, 2);
 7840   subl(len, 2);
 7841   addl(pos, 2);
 7842 
 7843   bind(L_less_than_2_left);
 7844   cmpl(len, 1);
 7845   jcc(Assembler::less, L_zero_left);
 7846 
 7847   // load 1 Byte
 7848   movb(rax, Address(buf, pos, Address::times_1, 0));
 7849   movb(Address(tmp1, 0 * 16), rax);
 7850 
 7851   bind(L_zero_left);
 7852   movdqu(xmm7, Address(rsp, 0));
 7853   pxor(xmm7, xmm0);                       //xor the initial crc value
 7854 
 7855   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 7856   movdqu(xmm0, Address(rax, tmp2));
 7857   pshufb(xmm7, xmm0);
 7858   jmp(L_128_done);
 7859 
 7860   bind(L_exact_16_left);
 7861   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 7862   pxor(xmm7, xmm0);                       //xor the initial crc value
 7863   jmp(L_128_done);
 7864 
 7865   bind(L_only_less_than_4);
 7866   cmpl(len, 3);
 7867   jcc(Assembler::less, L_only_less_than_3);
 7868 
 7869   // load 3 Bytes
 7870   movb(rax, Address(buf, pos, Address::times_1, 0));
 7871   movb(Address(tmp1, 0), rax);
 7872 
 7873   movb(rax, Address(buf, pos, Address::times_1, 1));
 7874   movb(Address(tmp1, 1), rax);
 7875 
 7876   movb(rax, Address(buf, pos, Address::times_1, 2));
 7877   movb(Address(tmp1, 2), rax);
 7878 
 7879   movdqu(xmm7, Address(rsp, 0));
 7880   pxor(xmm7, xmm0);                     //xor the initial crc value
 7881 
 7882   pslldq(xmm7, 0x5);
 7883   jmp(L_barrett);
 7884   bind(L_only_less_than_3);
 7885   cmpl(len, 2);
 7886   jcc(Assembler::less, L_only_less_than_2);
 7887 
 7888   // load 2 Bytes
 7889   movb(rax, Address(buf, pos, Address::times_1, 0));
 7890   movb(Address(tmp1, 0), rax);
 7891 
 7892   movb(rax, Address(buf, pos, Address::times_1, 1));
 7893   movb(Address(tmp1, 1), rax);
 7894 
 7895   movdqu(xmm7, Address(rsp, 0));
 7896   pxor(xmm7, xmm0);                     //xor the initial crc value
 7897 
 7898   pslldq(xmm7, 0x6);
 7899   jmp(L_barrett);
 7900 
 7901   bind(L_only_less_than_2);
 7902   //load 1 Byte
 7903   movb(rax, Address(buf, pos, Address::times_1, 0));
 7904   movb(Address(tmp1, 0), rax);
 7905 
 7906   movdqu(xmm7, Address(rsp, 0));
 7907   pxor(xmm7, xmm0);                     //xor the initial crc value
 7908 
 7909   pslldq(xmm7, 0x7);
 7910 }
 7911 
 7912 /**
 7913 * Compute CRC32 using AVX512 instructions
 7914 * param crc   register containing existing CRC (32-bit)
 7915 * param buf   register pointing to input byte buffer (byte*)
 7916 * param len   register containing number of bytes
 7917 * param table address of crc or crc32c table
 7918 * param tmp1  scratch register
 7919 * param tmp2  scratch register
 7920 * return rax  result register
 7921 *
 7922 * This routine is identical for crc32c with the exception of the precomputed constant
 7923 * table which will be passed as the table argument.  The calculation steps are
 7924 * the same for both variants.
 7925 */
 7926 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 7927   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 7928 
 7929   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 7930   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 7931   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 7932   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 7933   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 7934 
 7935   const Register pos = r12;
 7936   push(r12);
 7937   subptr(rsp, 16 * 2 + 8);
 7938 
 7939   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 7940   // context for the registers used, where all instructions below are using 128-bit mode
 7941   // On EVEX without VL and BW, these instructions will all be AVX.
 7942   movl(pos, 0);
 7943 
 7944   // check if smaller than 256B
 7945   cmpl(len, 256);
 7946   jcc(Assembler::less, L_less_than_256);
 7947 
 7948   // load the initial crc value
 7949   movdl(xmm10, crc);
 7950 
 7951   // receive the initial 64B data, xor the initial crc value
 7952   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 7953   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 7954   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 7955   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 7956 
 7957   subl(len, 256);
 7958   cmpl(len, 256);
 7959   jcc(Assembler::less, L_fold_128_B_loop);
 7960 
 7961   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 7962   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 7963   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 7964   subl(len, 256);
 7965 
 7966   bind(L_fold_256_B_loop);
 7967   addl(pos, 256);
 7968   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 7969   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 7970   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 7971   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 7972 
 7973   subl(len, 256);
 7974   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 7975 
 7976   // Fold 256 into 128
 7977   addl(pos, 256);
 7978   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 7979   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 7980   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 7981 
 7982   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 7983   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 7984   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 7985 
 7986   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 7987   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 7988 
 7989   addl(len, 128);
 7990   jmp(L_fold_128_B_register);
 7991 
 7992   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 7993   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 7994 
 7995   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 7996   bind(L_fold_128_B_loop);
 7997   addl(pos, 128);
 7998   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 7999   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8000 
 8001   subl(len, 128);
 8002   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8003 
 8004   addl(pos, 128);
 8005 
 8006   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8007   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8008   bind(L_fold_128_B_register);
 8009   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8010   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8011   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8012   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8013   // save last that has no multiplicand
 8014   vextracti64x2(xmm7, xmm4, 3);
 8015 
 8016   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8017   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8018   // Needed later in reduction loop
 8019   movdqu(xmm10, Address(table, 1 * 16));
 8020   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8021   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8022 
 8023   // Swap 1,0,3,2 - 01 00 11 10
 8024   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8025   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8026   vextracti128(xmm5, xmm8, 1);
 8027   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8028 
 8029   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8030   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8031   addl(len, 128 - 16);
 8032   jcc(Assembler::less, L_final_reduction_for_128);
 8033 
 8034   bind(L_16B_reduction_loop);
 8035   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8036   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8037   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8038   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8039   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8040   addl(pos, 16);
 8041   subl(len, 16);
 8042   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8043 
 8044   bind(L_final_reduction_for_128);
 8045   addl(len, 16);
 8046   jcc(Assembler::equal, L_128_done);
 8047 
 8048   bind(L_get_last_two_xmms);
 8049   movdqu(xmm2, xmm7);
 8050   addl(pos, len);
 8051   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8052   subl(pos, len);
 8053 
 8054   // get rid of the extra data that was loaded before
 8055   // load the shift constant
 8056   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8057   movdqu(xmm0, Address(rax, len));
 8058   addl(rax, len);
 8059 
 8060   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8061   //Change mask to 512
 8062   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8063   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8064 
 8065   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8066   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8067   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8068   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8069   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8070 
 8071   bind(L_128_done);
 8072   // compute crc of a 128-bit value
 8073   movdqu(xmm10, Address(table, 3 * 16));
 8074   movdqu(xmm0, xmm7);
 8075 
 8076   // 64b fold
 8077   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8078   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8079   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8080 
 8081   // 32b fold
 8082   movdqu(xmm0, xmm7);
 8083   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8084   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8085   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8086   jmp(L_barrett);
 8087 
 8088   bind(L_less_than_256);
 8089   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8090 
 8091   //barrett reduction
 8092   bind(L_barrett);
 8093   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8094   movdqu(xmm1, xmm7);
 8095   movdqu(xmm2, xmm7);
 8096   movdqu(xmm10, Address(table, 4 * 16));
 8097 
 8098   pclmulqdq(xmm7, xmm10, 0x0);
 8099   pxor(xmm7, xmm2);
 8100   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8101   movdqu(xmm2, xmm7);
 8102   pclmulqdq(xmm7, xmm10, 0x10);
 8103   pxor(xmm7, xmm2);
 8104   pxor(xmm7, xmm1);
 8105   pextrd(crc, xmm7, 2);
 8106 
 8107   bind(L_cleanup);
 8108   addptr(rsp, 16 * 2 + 8);
 8109   pop(r12);
 8110 }
 8111 
 8112 // S. Gueron / Information Processing Letters 112 (2012) 184
 8113 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8114 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8115 // Output: the 64-bit carry-less product of B * CONST
 8116 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8117                                      Register tmp1, Register tmp2, Register tmp3) {
 8118   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8119   if (n > 0) {
 8120     addq(tmp3, n * 256 * 8);
 8121   }
 8122   //    Q1 = TABLEExt[n][B & 0xFF];
 8123   movl(tmp1, in);
 8124   andl(tmp1, 0x000000FF);
 8125   shll(tmp1, 3);
 8126   addq(tmp1, tmp3);
 8127   movq(tmp1, Address(tmp1, 0));
 8128 
 8129   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8130   movl(tmp2, in);
 8131   shrl(tmp2, 8);
 8132   andl(tmp2, 0x000000FF);
 8133   shll(tmp2, 3);
 8134   addq(tmp2, tmp3);
 8135   movq(tmp2, Address(tmp2, 0));
 8136 
 8137   shlq(tmp2, 8);
 8138   xorq(tmp1, tmp2);
 8139 
 8140   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8141   movl(tmp2, in);
 8142   shrl(tmp2, 16);
 8143   andl(tmp2, 0x000000FF);
 8144   shll(tmp2, 3);
 8145   addq(tmp2, tmp3);
 8146   movq(tmp2, Address(tmp2, 0));
 8147 
 8148   shlq(tmp2, 16);
 8149   xorq(tmp1, tmp2);
 8150 
 8151   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8152   shrl(in, 24);
 8153   andl(in, 0x000000FF);
 8154   shll(in, 3);
 8155   addq(in, tmp3);
 8156   movq(in, Address(in, 0));
 8157 
 8158   shlq(in, 24);
 8159   xorq(in, tmp1);
 8160   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8161 }
 8162 
 8163 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8164                                       Register in_out,
 8165                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8166                                       XMMRegister w_xtmp2,
 8167                                       Register tmp1,
 8168                                       Register n_tmp2, Register n_tmp3) {
 8169   if (is_pclmulqdq_supported) {
 8170     movdl(w_xtmp1, in_out); // modified blindly
 8171 
 8172     movl(tmp1, const_or_pre_comp_const_index);
 8173     movdl(w_xtmp2, tmp1);
 8174     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8175 
 8176     movdq(in_out, w_xtmp1);
 8177   } else {
 8178     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8179   }
 8180 }
 8181 
 8182 // Recombination Alternative 2: No bit-reflections
 8183 // T1 = (CRC_A * U1) << 1
 8184 // T2 = (CRC_B * U2) << 1
 8185 // C1 = T1 >> 32
 8186 // C2 = T2 >> 32
 8187 // T1 = T1 & 0xFFFFFFFF
 8188 // T2 = T2 & 0xFFFFFFFF
 8189 // T1 = CRC32(0, T1)
 8190 // T2 = CRC32(0, T2)
 8191 // C1 = C1 ^ T1
 8192 // C2 = C2 ^ T2
 8193 // CRC = C1 ^ C2 ^ CRC_C
 8194 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8195                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8196                                      Register tmp1, Register tmp2,
 8197                                      Register n_tmp3) {
 8198   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8199   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8200   shlq(in_out, 1);
 8201   movl(tmp1, in_out);
 8202   shrq(in_out, 32);
 8203   xorl(tmp2, tmp2);
 8204   crc32(tmp2, tmp1, 4);
 8205   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8206   shlq(in1, 1);
 8207   movl(tmp1, in1);
 8208   shrq(in1, 32);
 8209   xorl(tmp2, tmp2);
 8210   crc32(tmp2, tmp1, 4);
 8211   xorl(in1, tmp2);
 8212   xorl(in_out, in1);
 8213   xorl(in_out, in2);
 8214 }
 8215 
 8216 // Set N to predefined value
 8217 // Subtract from a length of a buffer
 8218 // execute in a loop:
 8219 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8220 // for i = 1 to N do
 8221 //  CRC_A = CRC32(CRC_A, A[i])
 8222 //  CRC_B = CRC32(CRC_B, B[i])
 8223 //  CRC_C = CRC32(CRC_C, C[i])
 8224 // end for
 8225 // Recombine
 8226 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8227                                        Register in_out1, Register in_out2, Register in_out3,
 8228                                        Register tmp1, Register tmp2, Register tmp3,
 8229                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8230                                        Register tmp4, Register tmp5,
 8231                                        Register n_tmp6) {
 8232   Label L_processPartitions;
 8233   Label L_processPartition;
 8234   Label L_exit;
 8235 
 8236   bind(L_processPartitions);
 8237   cmpl(in_out1, 3 * size);
 8238   jcc(Assembler::less, L_exit);
 8239     xorl(tmp1, tmp1);
 8240     xorl(tmp2, tmp2);
 8241     movq(tmp3, in_out2);
 8242     addq(tmp3, size);
 8243 
 8244     bind(L_processPartition);
 8245       crc32(in_out3, Address(in_out2, 0), 8);
 8246       crc32(tmp1, Address(in_out2, size), 8);
 8247       crc32(tmp2, Address(in_out2, size * 2), 8);
 8248       addq(in_out2, 8);
 8249       cmpq(in_out2, tmp3);
 8250       jcc(Assembler::less, L_processPartition);
 8251     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8252             w_xtmp1, w_xtmp2, w_xtmp3,
 8253             tmp4, tmp5,
 8254             n_tmp6);
 8255     addq(in_out2, 2 * size);
 8256     subl(in_out1, 3 * size);
 8257     jmp(L_processPartitions);
 8258 
 8259   bind(L_exit);
 8260 }
 8261 #else
 8262 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 8263                                      Register tmp1, Register tmp2, Register tmp3,
 8264                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 8265   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8266   if (n > 0) {
 8267     addl(tmp3, n * 256 * 8);
 8268   }
 8269   //    Q1 = TABLEExt[n][B & 0xFF];
 8270   movl(tmp1, in_out);
 8271   andl(tmp1, 0x000000FF);
 8272   shll(tmp1, 3);
 8273   addl(tmp1, tmp3);
 8274   movq(xtmp1, Address(tmp1, 0));
 8275 
 8276   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8277   movl(tmp2, in_out);
 8278   shrl(tmp2, 8);
 8279   andl(tmp2, 0x000000FF);
 8280   shll(tmp2, 3);
 8281   addl(tmp2, tmp3);
 8282   movq(xtmp2, Address(tmp2, 0));
 8283 
 8284   psllq(xtmp2, 8);
 8285   pxor(xtmp1, xtmp2);
 8286 
 8287   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8288   movl(tmp2, in_out);
 8289   shrl(tmp2, 16);
 8290   andl(tmp2, 0x000000FF);
 8291   shll(tmp2, 3);
 8292   addl(tmp2, tmp3);
 8293   movq(xtmp2, Address(tmp2, 0));
 8294 
 8295   psllq(xtmp2, 16);
 8296   pxor(xtmp1, xtmp2);
 8297 
 8298   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8299   shrl(in_out, 24);
 8300   andl(in_out, 0x000000FF);
 8301   shll(in_out, 3);
 8302   addl(in_out, tmp3);
 8303   movq(xtmp2, Address(in_out, 0));
 8304 
 8305   psllq(xtmp2, 24);
 8306   pxor(xtmp1, xtmp2); // Result in CXMM
 8307   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8308 }
 8309 
 8310 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8311                                       Register in_out,
 8312                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8313                                       XMMRegister w_xtmp2,
 8314                                       Register tmp1,
 8315                                       Register n_tmp2, Register n_tmp3) {
 8316   if (is_pclmulqdq_supported) {
 8317     movdl(w_xtmp1, in_out);
 8318 
 8319     movl(tmp1, const_or_pre_comp_const_index);
 8320     movdl(w_xtmp2, tmp1);
 8321     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8322     // Keep result in XMM since GPR is 32 bit in length
 8323   } else {
 8324     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 8325   }
 8326 }
 8327 
 8328 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8329                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8330                                      Register tmp1, Register tmp2,
 8331                                      Register n_tmp3) {
 8332   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8333   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8334 
 8335   psllq(w_xtmp1, 1);
 8336   movdl(tmp1, w_xtmp1);
 8337   psrlq(w_xtmp1, 32);
 8338   movdl(in_out, w_xtmp1);
 8339 
 8340   xorl(tmp2, tmp2);
 8341   crc32(tmp2, tmp1, 4);
 8342   xorl(in_out, tmp2);
 8343 
 8344   psllq(w_xtmp2, 1);
 8345   movdl(tmp1, w_xtmp2);
 8346   psrlq(w_xtmp2, 32);
 8347   movdl(in1, w_xtmp2);
 8348 
 8349   xorl(tmp2, tmp2);
 8350   crc32(tmp2, tmp1, 4);
 8351   xorl(in1, tmp2);
 8352   xorl(in_out, in1);
 8353   xorl(in_out, in2);
 8354 }
 8355 
 8356 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8357                                        Register in_out1, Register in_out2, Register in_out3,
 8358                                        Register tmp1, Register tmp2, Register tmp3,
 8359                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8360                                        Register tmp4, Register tmp5,
 8361                                        Register n_tmp6) {
 8362   Label L_processPartitions;
 8363   Label L_processPartition;
 8364   Label L_exit;
 8365 
 8366   bind(L_processPartitions);
 8367   cmpl(in_out1, 3 * size);
 8368   jcc(Assembler::less, L_exit);
 8369     xorl(tmp1, tmp1);
 8370     xorl(tmp2, tmp2);
 8371     movl(tmp3, in_out2);
 8372     addl(tmp3, size);
 8373 
 8374     bind(L_processPartition);
 8375       crc32(in_out3, Address(in_out2, 0), 4);
 8376       crc32(tmp1, Address(in_out2, size), 4);
 8377       crc32(tmp2, Address(in_out2, size*2), 4);
 8378       crc32(in_out3, Address(in_out2, 0+4), 4);
 8379       crc32(tmp1, Address(in_out2, size+4), 4);
 8380       crc32(tmp2, Address(in_out2, size*2+4), 4);
 8381       addl(in_out2, 8);
 8382       cmpl(in_out2, tmp3);
 8383       jcc(Assembler::less, L_processPartition);
 8384 
 8385         push(tmp3);
 8386         push(in_out1);
 8387         push(in_out2);
 8388         tmp4 = tmp3;
 8389         tmp5 = in_out1;
 8390         n_tmp6 = in_out2;
 8391 
 8392       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 8393             w_xtmp1, w_xtmp2, w_xtmp3,
 8394             tmp4, tmp5,
 8395             n_tmp6);
 8396 
 8397         pop(in_out2);
 8398         pop(in_out1);
 8399         pop(tmp3);
 8400 
 8401     addl(in_out2, 2 * size);
 8402     subl(in_out1, 3 * size);
 8403     jmp(L_processPartitions);
 8404 
 8405   bind(L_exit);
 8406 }
 8407 #endif //LP64
 8408 
 8409 #ifdef _LP64
 8410 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 8411 // Input: A buffer I of L bytes.
 8412 // Output: the CRC32C value of the buffer.
 8413 // Notations:
 8414 // Write L = 24N + r, with N = floor (L/24).
 8415 // r = L mod 24 (0 <= r < 24).
 8416 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 8417 // N quadwords, and R consists of r bytes.
 8418 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 8419 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 8420 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 8421 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 8422 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8423                                           Register tmp1, Register tmp2, Register tmp3,
 8424                                           Register tmp4, Register tmp5, Register tmp6,
 8425                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8426                                           bool is_pclmulqdq_supported) {
 8427   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8428   Label L_wordByWord;
 8429   Label L_byteByByteProlog;
 8430   Label L_byteByByte;
 8431   Label L_exit;
 8432 
 8433   if (is_pclmulqdq_supported ) {
 8434     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8435     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
 8436 
 8437     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8438     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8439 
 8440     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8441     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8442     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 8443   } else {
 8444     const_or_pre_comp_const_index[0] = 1;
 8445     const_or_pre_comp_const_index[1] = 0;
 8446 
 8447     const_or_pre_comp_const_index[2] = 3;
 8448     const_or_pre_comp_const_index[3] = 2;
 8449 
 8450     const_or_pre_comp_const_index[4] = 5;
 8451     const_or_pre_comp_const_index[5] = 4;
 8452    }
 8453   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8454                     in2, in1, in_out,
 8455                     tmp1, tmp2, tmp3,
 8456                     w_xtmp1, w_xtmp2, w_xtmp3,
 8457                     tmp4, tmp5,
 8458                     tmp6);
 8459   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8460                     in2, in1, in_out,
 8461                     tmp1, tmp2, tmp3,
 8462                     w_xtmp1, w_xtmp2, w_xtmp3,
 8463                     tmp4, tmp5,
 8464                     tmp6);
 8465   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8466                     in2, in1, in_out,
 8467                     tmp1, tmp2, tmp3,
 8468                     w_xtmp1, w_xtmp2, w_xtmp3,
 8469                     tmp4, tmp5,
 8470                     tmp6);
 8471   movl(tmp1, in2);
 8472   andl(tmp1, 0x00000007);
 8473   negl(tmp1);
 8474   addl(tmp1, in2);
 8475   addq(tmp1, in1);
 8476 
 8477   cmpq(in1, tmp1);
 8478   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 8479   align(16);
 8480   BIND(L_wordByWord);
 8481     crc32(in_out, Address(in1, 0), 8);
 8482     addq(in1, 8);
 8483     cmpq(in1, tmp1);
 8484     jcc(Assembler::less, L_wordByWord);
 8485 
 8486   BIND(L_byteByByteProlog);
 8487   andl(in2, 0x00000007);
 8488   movl(tmp2, 1);
 8489 
 8490   cmpl(tmp2, in2);
 8491   jccb(Assembler::greater, L_exit);
 8492   BIND(L_byteByByte);
 8493     crc32(in_out, Address(in1, 0), 1);
 8494     incq(in1);
 8495     incl(tmp2);
 8496     cmpl(tmp2, in2);
 8497     jcc(Assembler::lessEqual, L_byteByByte);
 8498 
 8499   BIND(L_exit);
 8500 }
 8501 #else
 8502 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 8503                                           Register tmp1, Register  tmp2, Register tmp3,
 8504                                           Register tmp4, Register  tmp5, Register tmp6,
 8505                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8506                                           bool is_pclmulqdq_supported) {
 8507   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 8508   Label L_wordByWord;
 8509   Label L_byteByByteProlog;
 8510   Label L_byteByByte;
 8511   Label L_exit;
 8512 
 8513   if (is_pclmulqdq_supported) {
 8514     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 8515     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
 8516 
 8517     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 8518     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 8519 
 8520     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 8521     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 8522   } else {
 8523     const_or_pre_comp_const_index[0] = 1;
 8524     const_or_pre_comp_const_index[1] = 0;
 8525 
 8526     const_or_pre_comp_const_index[2] = 3;
 8527     const_or_pre_comp_const_index[3] = 2;
 8528 
 8529     const_or_pre_comp_const_index[4] = 5;
 8530     const_or_pre_comp_const_index[5] = 4;
 8531   }
 8532   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 8533                     in2, in1, in_out,
 8534                     tmp1, tmp2, tmp3,
 8535                     w_xtmp1, w_xtmp2, w_xtmp3,
 8536                     tmp4, tmp5,
 8537                     tmp6);
 8538   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 8539                     in2, in1, in_out,
 8540                     tmp1, tmp2, tmp3,
 8541                     w_xtmp1, w_xtmp2, w_xtmp3,
 8542                     tmp4, tmp5,
 8543                     tmp6);
 8544   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 8545                     in2, in1, in_out,
 8546                     tmp1, tmp2, tmp3,
 8547                     w_xtmp1, w_xtmp2, w_xtmp3,
 8548                     tmp4, tmp5,
 8549                     tmp6);
 8550   movl(tmp1, in2);
 8551   andl(tmp1, 0x00000007);
 8552   negl(tmp1);
 8553   addl(tmp1, in2);
 8554   addl(tmp1, in1);
 8555 
 8556   BIND(L_wordByWord);
 8557   cmpl(in1, tmp1);
 8558   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 8559     crc32(in_out, Address(in1,0), 4);
 8560     addl(in1, 4);
 8561     jmp(L_wordByWord);
 8562 
 8563   BIND(L_byteByByteProlog);
 8564   andl(in2, 0x00000007);
 8565   movl(tmp2, 1);
 8566 
 8567   BIND(L_byteByByte);
 8568   cmpl(tmp2, in2);
 8569   jccb(Assembler::greater, L_exit);
 8570     movb(tmp1, Address(in1, 0));
 8571     crc32(in_out, tmp1, 1);
 8572     incl(in1);
 8573     incl(tmp2);
 8574     jmp(L_byteByByte);
 8575 
 8576   BIND(L_exit);
 8577 }
 8578 #endif // LP64
 8579 #undef BIND
 8580 #undef BLOCK_COMMENT
 8581 
 8582 // Compress char[] array to byte[].
 8583 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
 8584 // Return the array length if every element in array can be encoded,
 8585 // otherwise, the index of first non-latin1 (> 0xff) character.
 8586 //   @IntrinsicCandidate
 8587 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 8588 //     for (int i = 0; i < len; i++) {
 8589 //       char c = src[srcOff];
 8590 //       if (c > 0xff) {
 8591 //           return i;  // return index of non-latin1 char
 8592 //       }
 8593 //       dst[dstOff] = (byte)c;
 8594 //       srcOff++;
 8595 //       dstOff++;
 8596 //     }
 8597 //     return len;
 8598 //   }
 8599 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 8600   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 8601   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 8602   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 8603   Label copy_chars_loop, done, reset_sp, copy_tail;
 8604 
 8605   // rsi: src
 8606   // rdi: dst
 8607   // rdx: len
 8608   // rcx: tmp5
 8609   // rax: result
 8610 
 8611   // rsi holds start addr of source char[] to be compressed
 8612   // rdi holds start addr of destination byte[]
 8613   // rdx holds length
 8614 
 8615   assert(len != result, "");
 8616 
 8617   // save length for return
 8618   movl(result, len);
 8619 
 8620   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 8621     VM_Version::supports_avx512vlbw() &&
 8622     VM_Version::supports_bmi2()) {
 8623 
 8624     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
 8625 
 8626     // alignment
 8627     Label post_alignment;
 8628 
 8629     // if length of the string is less than 32, handle it the old fashioned way
 8630     testl(len, -32);
 8631     jcc(Assembler::zero, below_threshold);
 8632 
 8633     // First check whether a character is compressible ( <= 0xFF).
 8634     // Create mask to test for Unicode chars inside zmm vector
 8635     movl(tmp5, 0x00FF);
 8636     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
 8637 
 8638     testl(len, -64);
 8639     jccb(Assembler::zero, post_alignment);
 8640 
 8641     movl(tmp5, dst);
 8642     andl(tmp5, (32 - 1));
 8643     negl(tmp5);
 8644     andl(tmp5, (32 - 1));
 8645 
 8646     // bail out when there is nothing to be done
 8647     testl(tmp5, 0xFFFFFFFF);
 8648     jccb(Assembler::zero, post_alignment);
 8649 
 8650     // ~(~0 << len), where len is the # of remaining elements to process
 8651     movl(len, 0xFFFFFFFF);
 8652     shlxl(len, len, tmp5);
 8653     notl(len);
 8654     kmovdl(mask2, len);
 8655     movl(len, result);
 8656 
 8657     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 8658     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 8659     ktestd(mask1, mask2);
 8660     jcc(Assembler::carryClear, copy_tail);
 8661 
 8662     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 8663 
 8664     addptr(src, tmp5);
 8665     addptr(src, tmp5);
 8666     addptr(dst, tmp5);
 8667     subl(len, tmp5);
 8668 
 8669     bind(post_alignment);
 8670     // end of alignment
 8671 
 8672     movl(tmp5, len);
 8673     andl(tmp5, (32 - 1));    // tail count (in chars)
 8674     andl(len, ~(32 - 1));    // vector count (in chars)
 8675     jccb(Assembler::zero, copy_loop_tail);
 8676 
 8677     lea(src, Address(src, len, Address::times_2));
 8678     lea(dst, Address(dst, len, Address::times_1));
 8679     negptr(len);
 8680 
 8681     bind(copy_32_loop);
 8682     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 8683     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 8684     kortestdl(mask1, mask1);
 8685     jccb(Assembler::carryClear, reset_for_copy_tail);
 8686 
 8687     // All elements in current processed chunk are valid candidates for
 8688     // compression. Write a truncated byte elements to the memory.
 8689     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 8690     addptr(len, 32);
 8691     jccb(Assembler::notZero, copy_32_loop);
 8692 
 8693     bind(copy_loop_tail);
 8694     // bail out when there is nothing to be done
 8695     testl(tmp5, 0xFFFFFFFF);
 8696     jcc(Assembler::zero, done);
 8697 
 8698     movl(len, tmp5);
 8699 
 8700     // ~(~0 << len), where len is the # of remaining elements to process
 8701     movl(tmp5, 0xFFFFFFFF);
 8702     shlxl(tmp5, tmp5, len);
 8703     notl(tmp5);
 8704 
 8705     kmovdl(mask2, tmp5);
 8706 
 8707     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 8708     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 8709     ktestd(mask1, mask2);
 8710     jcc(Assembler::carryClear, copy_tail);
 8711 
 8712     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 8713     jmp(done);
 8714 
 8715     bind(reset_for_copy_tail);
 8716     lea(src, Address(src, tmp5, Address::times_2));
 8717     lea(dst, Address(dst, tmp5, Address::times_1));
 8718     subptr(len, tmp5);
 8719     jmp(copy_chars_loop);
 8720 
 8721     bind(below_threshold);
 8722   }
 8723 
 8724   if (UseSSE42Intrinsics) {
 8725     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
 8726 
 8727     // vectored compression
 8728     testl(len, 0xfffffff8);
 8729     jcc(Assembler::zero, copy_tail);
 8730 
 8731     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 8732     movdl(tmp1Reg, tmp5);
 8733     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 8734 
 8735     andl(len, 0xfffffff0);
 8736     jccb(Assembler::zero, copy_16);
 8737 
 8738     // compress 16 chars per iter
 8739     pxor(tmp4Reg, tmp4Reg);
 8740 
 8741     lea(src, Address(src, len, Address::times_2));
 8742     lea(dst, Address(dst, len, Address::times_1));
 8743     negptr(len);
 8744 
 8745     bind(copy_32_loop);
 8746     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 8747     por(tmp4Reg, tmp2Reg);
 8748     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 8749     por(tmp4Reg, tmp3Reg);
 8750     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 8751     jccb(Assembler::notZero, reset_for_copy_tail);
 8752     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 8753     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 8754     addptr(len, 16);
 8755     jccb(Assembler::notZero, copy_32_loop);
 8756 
 8757     // compress next vector of 8 chars (if any)
 8758     bind(copy_16);
 8759     // len = 0
 8760     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
 8761     jccb(Assembler::zero, copy_tail_sse);
 8762 
 8763     pxor(tmp3Reg, tmp3Reg);
 8764 
 8765     movdqu(tmp2Reg, Address(src, 0));
 8766     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 8767     jccb(Assembler::notZero, reset_for_copy_tail);
 8768     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 8769     movq(Address(dst, 0), tmp2Reg);
 8770     addptr(src, 16);
 8771     addptr(dst, 8);
 8772     jmpb(copy_tail_sse);
 8773 
 8774     bind(reset_for_copy_tail);
 8775     movl(tmp5, result);
 8776     andl(tmp5, 0x0000000f);
 8777     lea(src, Address(src, tmp5, Address::times_2));
 8778     lea(dst, Address(dst, tmp5, Address::times_1));
 8779     subptr(len, tmp5);
 8780     jmpb(copy_chars_loop);
 8781 
 8782     bind(copy_tail_sse);
 8783     movl(len, result);
 8784     andl(len, 0x00000007);    // tail count (in chars)
 8785   }
 8786   // compress 1 char per iter
 8787   bind(copy_tail);
 8788   testl(len, len);
 8789   jccb(Assembler::zero, done);
 8790   lea(src, Address(src, len, Address::times_2));
 8791   lea(dst, Address(dst, len, Address::times_1));
 8792   negptr(len);
 8793 
 8794   bind(copy_chars_loop);
 8795   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
 8796   testl(tmp5, 0xff00);      // check if Unicode char
 8797   jccb(Assembler::notZero, reset_sp);
 8798   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
 8799   increment(len);
 8800   jccb(Assembler::notZero, copy_chars_loop);
 8801 
 8802   // add len then return (len will be zero if compress succeeded, otherwise negative)
 8803   bind(reset_sp);
 8804   addl(result, len);
 8805 
 8806   bind(done);
 8807 }
 8808 
 8809 // Inflate byte[] array to char[].
 8810 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 8811 //   @IntrinsicCandidate
 8812 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 8813 //     for (int i = 0; i < len; i++) {
 8814 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 8815 //     }
 8816 //   }
 8817 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 8818   XMMRegister tmp1, Register tmp2, KRegister mask) {
 8819   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 8820   // rsi: src
 8821   // rdi: dst
 8822   // rdx: len
 8823   // rcx: tmp2
 8824 
 8825   // rsi holds start addr of source byte[] to be inflated
 8826   // rdi holds start addr of destination char[]
 8827   // rdx holds length
 8828   assert_different_registers(src, dst, len, tmp2);
 8829   movl(tmp2, len);
 8830   if ((UseAVX > 2) && // AVX512
 8831     VM_Version::supports_avx512vlbw() &&
 8832     VM_Version::supports_bmi2()) {
 8833 
 8834     Label copy_32_loop, copy_tail;
 8835     Register tmp3_aliased = len;
 8836 
 8837     // if length of the string is less than 16, handle it in an old fashioned way
 8838     testl(len, -16);
 8839     jcc(Assembler::zero, below_threshold);
 8840 
 8841     testl(len, -1 * AVX3Threshold);
 8842     jcc(Assembler::zero, avx3_threshold);
 8843 
 8844     // In order to use only one arithmetic operation for the main loop we use
 8845     // this pre-calculation
 8846     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 8847     andl(len, -32);     // vector count
 8848     jccb(Assembler::zero, copy_tail);
 8849 
 8850     lea(src, Address(src, len, Address::times_1));
 8851     lea(dst, Address(dst, len, Address::times_2));
 8852     negptr(len);
 8853 
 8854 
 8855     // inflate 32 chars per iter
 8856     bind(copy_32_loop);
 8857     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 8858     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 8859     addptr(len, 32);
 8860     jcc(Assembler::notZero, copy_32_loop);
 8861 
 8862     bind(copy_tail);
 8863     // bail out when there is nothing to be done
 8864     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 8865     jcc(Assembler::zero, done);
 8866 
 8867     // ~(~0 << length), where length is the # of remaining elements to process
 8868     movl(tmp3_aliased, -1);
 8869     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 8870     notl(tmp3_aliased);
 8871     kmovdl(mask, tmp3_aliased);
 8872     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 8873     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 8874 
 8875     jmp(done);
 8876     bind(avx3_threshold);
 8877   }
 8878   if (UseSSE42Intrinsics) {
 8879     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 8880 
 8881     if (UseAVX > 1) {
 8882       andl(tmp2, (16 - 1));
 8883       andl(len, -16);
 8884       jccb(Assembler::zero, copy_new_tail);
 8885     } else {
 8886       andl(tmp2, 0x00000007);   // tail count (in chars)
 8887       andl(len, 0xfffffff8);    // vector count (in chars)
 8888       jccb(Assembler::zero, copy_tail);
 8889     }
 8890 
 8891     // vectored inflation
 8892     lea(src, Address(src, len, Address::times_1));
 8893     lea(dst, Address(dst, len, Address::times_2));
 8894     negptr(len);
 8895 
 8896     if (UseAVX > 1) {
 8897       bind(copy_16_loop);
 8898       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 8899       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 8900       addptr(len, 16);
 8901       jcc(Assembler::notZero, copy_16_loop);
 8902 
 8903       bind(below_threshold);
 8904       bind(copy_new_tail);
 8905       movl(len, tmp2);
 8906       andl(tmp2, 0x00000007);
 8907       andl(len, 0xFFFFFFF8);
 8908       jccb(Assembler::zero, copy_tail);
 8909 
 8910       pmovzxbw(tmp1, Address(src, 0));
 8911       movdqu(Address(dst, 0), tmp1);
 8912       addptr(src, 8);
 8913       addptr(dst, 2 * 8);
 8914 
 8915       jmp(copy_tail, true);
 8916     }
 8917 
 8918     // inflate 8 chars per iter
 8919     bind(copy_8_loop);
 8920     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 8921     movdqu(Address(dst, len, Address::times_2), tmp1);
 8922     addptr(len, 8);
 8923     jcc(Assembler::notZero, copy_8_loop);
 8924 
 8925     bind(copy_tail);
 8926     movl(len, tmp2);
 8927 
 8928     cmpl(len, 4);
 8929     jccb(Assembler::less, copy_bytes);
 8930 
 8931     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 8932     pmovzxbw(tmp1, tmp1);
 8933     movq(Address(dst, 0), tmp1);
 8934     subptr(len, 4);
 8935     addptr(src, 4);
 8936     addptr(dst, 8);
 8937 
 8938     bind(copy_bytes);
 8939   } else {
 8940     bind(below_threshold);
 8941   }
 8942 
 8943   testl(len, len);
 8944   jccb(Assembler::zero, done);
 8945   lea(src, Address(src, len, Address::times_1));
 8946   lea(dst, Address(dst, len, Address::times_2));
 8947   negptr(len);
 8948 
 8949   // inflate 1 char per iter
 8950   bind(copy_chars_loop);
 8951   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 8952   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 8953   increment(len);
 8954   jcc(Assembler::notZero, copy_chars_loop);
 8955 
 8956   bind(done);
 8957 }
 8958 
 8959 
 8960 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 8961   switch(type) {
 8962     case T_BYTE:
 8963     case T_BOOLEAN:
 8964       evmovdqub(dst, kmask, src, merge, vector_len);
 8965       break;
 8966     case T_CHAR:
 8967     case T_SHORT:
 8968       evmovdquw(dst, kmask, src, merge, vector_len);
 8969       break;
 8970     case T_INT:
 8971     case T_FLOAT:
 8972       evmovdqul(dst, kmask, src, merge, vector_len);
 8973       break;
 8974     case T_LONG:
 8975     case T_DOUBLE:
 8976       evmovdquq(dst, kmask, src, merge, vector_len);
 8977       break;
 8978     default:
 8979       fatal("Unexpected type argument %s", type2name(type));
 8980       break;
 8981   }
 8982 }
 8983 
 8984 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 8985   switch(type) {
 8986     case T_BYTE:
 8987     case T_BOOLEAN:
 8988       evmovdqub(dst, kmask, src, merge, vector_len);
 8989       break;
 8990     case T_CHAR:
 8991     case T_SHORT:
 8992       evmovdquw(dst, kmask, src, merge, vector_len);
 8993       break;
 8994     case T_INT:
 8995     case T_FLOAT:
 8996       evmovdqul(dst, kmask, src, merge, vector_len);
 8997       break;
 8998     case T_LONG:
 8999     case T_DOUBLE:
 9000       evmovdquq(dst, kmask, src, merge, vector_len);
 9001       break;
 9002     default:
 9003       fatal("Unexpected type argument %s", type2name(type));
 9004       break;
 9005   }
 9006 }
 9007 
 9008 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9009   switch(masklen) {
 9010     case 2:
 9011        knotbl(dst, src);
 9012        movl(rtmp, 3);
 9013        kmovbl(ktmp, rtmp);
 9014        kandbl(dst, ktmp, dst);
 9015        break;
 9016     case 4:
 9017        knotbl(dst, src);
 9018        movl(rtmp, 15);
 9019        kmovbl(ktmp, rtmp);
 9020        kandbl(dst, ktmp, dst);
 9021        break;
 9022     case 8:
 9023        knotbl(dst, src);
 9024        break;
 9025     case 16:
 9026        knotwl(dst, src);
 9027        break;
 9028     case 32:
 9029        knotdl(dst, src);
 9030        break;
 9031     case 64:
 9032        knotql(dst, src);
 9033        break;
 9034     default:
 9035       fatal("Unexpected vector length %d", masklen);
 9036       break;
 9037   }
 9038 }
 9039 
 9040 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9041   switch(type) {
 9042     case T_BOOLEAN:
 9043     case T_BYTE:
 9044        kandbl(dst, src1, src2);
 9045        break;
 9046     case T_CHAR:
 9047     case T_SHORT:
 9048        kandwl(dst, src1, src2);
 9049        break;
 9050     case T_INT:
 9051     case T_FLOAT:
 9052        kanddl(dst, src1, src2);
 9053        break;
 9054     case T_LONG:
 9055     case T_DOUBLE:
 9056        kandql(dst, src1, src2);
 9057        break;
 9058     default:
 9059       fatal("Unexpected type argument %s", type2name(type));
 9060       break;
 9061   }
 9062 }
 9063 
 9064 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9065   switch(type) {
 9066     case T_BOOLEAN:
 9067     case T_BYTE:
 9068        korbl(dst, src1, src2);
 9069        break;
 9070     case T_CHAR:
 9071     case T_SHORT:
 9072        korwl(dst, src1, src2);
 9073        break;
 9074     case T_INT:
 9075     case T_FLOAT:
 9076        kordl(dst, src1, src2);
 9077        break;
 9078     case T_LONG:
 9079     case T_DOUBLE:
 9080        korql(dst, src1, src2);
 9081        break;
 9082     default:
 9083       fatal("Unexpected type argument %s", type2name(type));
 9084       break;
 9085   }
 9086 }
 9087 
 9088 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9089   switch(type) {
 9090     case T_BOOLEAN:
 9091     case T_BYTE:
 9092        kxorbl(dst, src1, src2);
 9093        break;
 9094     case T_CHAR:
 9095     case T_SHORT:
 9096        kxorwl(dst, src1, src2);
 9097        break;
 9098     case T_INT:
 9099     case T_FLOAT:
 9100        kxordl(dst, src1, src2);
 9101        break;
 9102     case T_LONG:
 9103     case T_DOUBLE:
 9104        kxorql(dst, src1, src2);
 9105        break;
 9106     default:
 9107       fatal("Unexpected type argument %s", type2name(type));
 9108       break;
 9109   }
 9110 }
 9111 
 9112 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9113   switch(type) {
 9114     case T_BOOLEAN:
 9115     case T_BYTE:
 9116       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9117     case T_CHAR:
 9118     case T_SHORT:
 9119       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9120     case T_INT:
 9121     case T_FLOAT:
 9122       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9123     case T_LONG:
 9124     case T_DOUBLE:
 9125       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9126     default:
 9127       fatal("Unexpected type argument %s", type2name(type)); break;
 9128   }
 9129 }
 9130 
 9131 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9132   switch(type) {
 9133     case T_BOOLEAN:
 9134     case T_BYTE:
 9135       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9136     case T_CHAR:
 9137     case T_SHORT:
 9138       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9139     case T_INT:
 9140     case T_FLOAT:
 9141       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9142     case T_LONG:
 9143     case T_DOUBLE:
 9144       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9145     default:
 9146       fatal("Unexpected type argument %s", type2name(type)); break;
 9147   }
 9148 }
 9149 
 9150 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9151   switch(type) {
 9152     case T_BYTE:
 9153       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9154     case T_SHORT:
 9155       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9156     case T_INT:
 9157       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9158     case T_LONG:
 9159       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9160     default:
 9161       fatal("Unexpected type argument %s", type2name(type)); break;
 9162   }
 9163 }
 9164 
 9165 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9166   switch(type) {
 9167     case T_BYTE:
 9168       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9169     case T_SHORT:
 9170       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9171     case T_INT:
 9172       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9173     case T_LONG:
 9174       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9175     default:
 9176       fatal("Unexpected type argument %s", type2name(type)); break;
 9177   }
 9178 }
 9179 
 9180 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9181   switch(type) {
 9182     case T_BYTE:
 9183       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9184     case T_SHORT:
 9185       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9186     case T_INT:
 9187       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9188     case T_LONG:
 9189       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9190     default:
 9191       fatal("Unexpected type argument %s", type2name(type)); break;
 9192   }
 9193 }
 9194 
 9195 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9196   switch(type) {
 9197     case T_BYTE:
 9198       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9199     case T_SHORT:
 9200       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9201     case T_INT:
 9202       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9203     case T_LONG:
 9204       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9205     default:
 9206       fatal("Unexpected type argument %s", type2name(type)); break;
 9207   }
 9208 }
 9209 
 9210 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9211   switch(type) {
 9212     case T_INT:
 9213       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9214     case T_LONG:
 9215       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9216     default:
 9217       fatal("Unexpected type argument %s", type2name(type)); break;
 9218   }
 9219 }
 9220 
 9221 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9222   switch(type) {
 9223     case T_INT:
 9224       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9225     case T_LONG:
 9226       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9227     default:
 9228       fatal("Unexpected type argument %s", type2name(type)); break;
 9229   }
 9230 }
 9231 
 9232 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9233   switch(type) {
 9234     case T_INT:
 9235       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9236     case T_LONG:
 9237       evporq(dst, mask, nds, src, merge, vector_len); break;
 9238     default:
 9239       fatal("Unexpected type argument %s", type2name(type)); break;
 9240   }
 9241 }
 9242 
 9243 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9244   switch(type) {
 9245     case T_INT:
 9246       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9247     case T_LONG:
 9248       evporq(dst, mask, nds, src, merge, vector_len); break;
 9249     default:
 9250       fatal("Unexpected type argument %s", type2name(type)); break;
 9251   }
 9252 }
 9253 
 9254 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9255   switch(type) {
 9256     case T_INT:
 9257       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9258     case T_LONG:
 9259       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9260     default:
 9261       fatal("Unexpected type argument %s", type2name(type)); break;
 9262   }
 9263 }
 9264 
 9265 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9266   switch(type) {
 9267     case T_INT:
 9268       evpandd(dst, mask, nds, src, merge, vector_len); break;
 9269     case T_LONG:
 9270       evpandq(dst, mask, nds, src, merge, vector_len); break;
 9271     default:
 9272       fatal("Unexpected type argument %s", type2name(type)); break;
 9273   }
 9274 }
 9275 
 9276 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
 9277   switch(masklen) {
 9278     case 8:
 9279        kortestbl(src1, src2);
 9280        break;
 9281     case 16:
 9282        kortestwl(src1, src2);
 9283        break;
 9284     case 32:
 9285        kortestdl(src1, src2);
 9286        break;
 9287     case 64:
 9288        kortestql(src1, src2);
 9289        break;
 9290     default:
 9291       fatal("Unexpected mask length %d", masklen);
 9292       break;
 9293   }
 9294 }
 9295 
 9296 
 9297 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
 9298   switch(masklen)  {
 9299     case 8:
 9300        ktestbl(src1, src2);
 9301        break;
 9302     case 16:
 9303        ktestwl(src1, src2);
 9304        break;
 9305     case 32:
 9306        ktestdl(src1, src2);
 9307        break;
 9308     case 64:
 9309        ktestql(src1, src2);
 9310        break;
 9311     default:
 9312       fatal("Unexpected mask length %d", masklen);
 9313       break;
 9314   }
 9315 }
 9316 
 9317 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9318   switch(type) {
 9319     case T_INT:
 9320       evprold(dst, mask, src, shift, merge, vlen_enc); break;
 9321     case T_LONG:
 9322       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
 9323     default:
 9324       fatal("Unexpected type argument %s", type2name(type)); break;
 9325       break;
 9326   }
 9327 }
 9328 
 9329 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
 9330   switch(type) {
 9331     case T_INT:
 9332       evprord(dst, mask, src, shift, merge, vlen_enc); break;
 9333     case T_LONG:
 9334       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
 9335     default:
 9336       fatal("Unexpected type argument %s", type2name(type)); break;
 9337   }
 9338 }
 9339 
 9340 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9341   switch(type) {
 9342     case T_INT:
 9343       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9344     case T_LONG:
 9345       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9346     default:
 9347       fatal("Unexpected type argument %s", type2name(type)); break;
 9348   }
 9349 }
 9350 
 9351 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
 9352   switch(type) {
 9353     case T_INT:
 9354       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
 9355     case T_LONG:
 9356       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
 9357     default:
 9358       fatal("Unexpected type argument %s", type2name(type)); break;
 9359   }
 9360 }
 9361 
 9362 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9363   assert(rscratch != noreg || always_reachable(src), "missing");
 9364 
 9365   if (reachable(src)) {
 9366     evpandq(dst, nds, as_Address(src), vector_len);
 9367   } else {
 9368     lea(rscratch, src);
 9369     evpandq(dst, nds, Address(rscratch, 0), vector_len);
 9370   }
 9371 }
 9372 
 9373 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 9374   assert(rscratch != noreg || always_reachable(src), "missing");
 9375 
 9376   if (reachable(src)) {
 9377     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
 9378   } else {
 9379     lea(rscratch, src);
 9380     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 9381   }
 9382 }
 9383 
 9384 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9385   assert(rscratch != noreg || always_reachable(src), "missing");
 9386 
 9387   if (reachable(src)) {
 9388     evporq(dst, nds, as_Address(src), vector_len);
 9389   } else {
 9390     lea(rscratch, src);
 9391     evporq(dst, nds, Address(rscratch, 0), vector_len);
 9392   }
 9393 }
 9394 
 9395 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 9396   assert(rscratch != noreg || always_reachable(src), "missing");
 9397 
 9398   if (reachable(src)) {
 9399     vpshufb(dst, nds, as_Address(src), vector_len);
 9400   } else {
 9401     lea(rscratch, src);
 9402     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
 9403   }
 9404 }
 9405 
 9406 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
 9407   assert(rscratch != noreg || always_reachable(src3), "missing");
 9408 
 9409   if (reachable(src3)) {
 9410     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
 9411   } else {
 9412     lea(rscratch, src3);
 9413     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
 9414   }
 9415 }
 9416 
 9417 #if COMPILER2_OR_JVMCI
 9418 
 9419 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
 9420                                  Register length, Register temp, int vec_enc) {
 9421   // Computing mask for predicated vector store.
 9422   movptr(temp, -1);
 9423   bzhiq(temp, temp, length);
 9424   kmov(mask, temp);
 9425   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
 9426 }
 9427 
 9428 // Set memory operation for length "less than" 64 bytes.
 9429 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
 9430                                        XMMRegister xmm, KRegister mask, Register length,
 9431                                        Register temp, bool use64byteVector) {
 9432   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9433   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9434   if (!use64byteVector) {
 9435     fill32(dst, disp, xmm);
 9436     subptr(length, 32 >> shift);
 9437     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
 9438   } else {
 9439     assert(MaxVectorSize == 64, "vector length != 64");
 9440     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
 9441   }
 9442 }
 9443 
 9444 
 9445 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
 9446                                        XMMRegister xmm, KRegister mask, Register length,
 9447                                        Register temp) {
 9448   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9449   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
 9450   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
 9451 }
 9452 
 9453 
 9454 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
 9455   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9456   vmovdqu(dst, xmm);
 9457 }
 9458 
 9459 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
 9460   fill32(Address(dst, disp), xmm);
 9461 }
 9462 
 9463 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
 9464   assert(MaxVectorSize >= 32, "vector length should be >= 32");
 9465   if (!use64byteVector) {
 9466     fill32(dst, xmm);
 9467     fill32(dst.plus_disp(32), xmm);
 9468   } else {
 9469     evmovdquq(dst, xmm, Assembler::AVX_512bit);
 9470   }
 9471 }
 9472 
 9473 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
 9474   fill64(Address(dst, disp), xmm, use64byteVector);
 9475 }
 9476 
 9477 #ifdef _LP64
 9478 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
 9479                                         Register count, Register rtmp, XMMRegister xtmp) {
 9480   Label L_exit;
 9481   Label L_fill_start;
 9482   Label L_fill_64_bytes;
 9483   Label L_fill_96_bytes;
 9484   Label L_fill_128_bytes;
 9485   Label L_fill_128_bytes_loop;
 9486   Label L_fill_128_loop_header;
 9487   Label L_fill_128_bytes_loop_header;
 9488   Label L_fill_128_bytes_loop_pre_header;
 9489   Label L_fill_zmm_sequence;
 9490 
 9491   int shift = -1;
 9492   int avx3threshold = VM_Version::avx3_threshold();
 9493   switch(type) {
 9494     case T_BYTE:  shift = 0;
 9495       break;
 9496     case T_SHORT: shift = 1;
 9497       break;
 9498     case T_INT:   shift = 2;
 9499       break;
 9500     /* Uncomment when LONG fill stubs are supported.
 9501     case T_LONG:  shift = 3;
 9502       break;
 9503     */
 9504     default:
 9505       fatal("Unhandled type: %s\n", type2name(type));
 9506   }
 9507 
 9508   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
 9509 
 9510     if (MaxVectorSize == 64) {
 9511       cmpq(count, avx3threshold >> shift);
 9512       jcc(Assembler::greater, L_fill_zmm_sequence);
 9513     }
 9514 
 9515     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
 9516 
 9517     bind(L_fill_start);
 9518 
 9519     cmpq(count, 32 >> shift);
 9520     jccb(Assembler::greater, L_fill_64_bytes);
 9521     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9522     jmp(L_exit);
 9523 
 9524     bind(L_fill_64_bytes);
 9525     cmpq(count, 64 >> shift);
 9526     jccb(Assembler::greater, L_fill_96_bytes);
 9527     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
 9528     jmp(L_exit);
 9529 
 9530     bind(L_fill_96_bytes);
 9531     cmpq(count, 96 >> shift);
 9532     jccb(Assembler::greater, L_fill_128_bytes);
 9533     fill64(to, 0, xtmp);
 9534     subq(count, 64 >> shift);
 9535     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
 9536     jmp(L_exit);
 9537 
 9538     bind(L_fill_128_bytes);
 9539     cmpq(count, 128 >> shift);
 9540     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
 9541     fill64(to, 0, xtmp);
 9542     fill32(to, 64, xtmp);
 9543     subq(count, 96 >> shift);
 9544     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
 9545     jmp(L_exit);
 9546 
 9547     bind(L_fill_128_bytes_loop_pre_header);
 9548     {
 9549       mov(rtmp, to);
 9550       andq(rtmp, 31);
 9551       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
 9552       negq(rtmp);
 9553       addq(rtmp, 32);
 9554       mov64(r8, -1L);
 9555       bzhiq(r8, r8, rtmp);
 9556       kmovql(k2, r8);
 9557       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
 9558       addq(to, rtmp);
 9559       shrq(rtmp, shift);
 9560       subq(count, rtmp);
 9561     }
 9562 
 9563     cmpq(count, 128 >> shift);
 9564     jcc(Assembler::less, L_fill_start);
 9565 
 9566     bind(L_fill_128_bytes_loop_header);
 9567     subq(count, 128 >> shift);
 9568 
 9569     align32();
 9570     bind(L_fill_128_bytes_loop);
 9571       fill64(to, 0, xtmp);
 9572       fill64(to, 64, xtmp);
 9573       addq(to, 128);
 9574       subq(count, 128 >> shift);
 9575       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
 9576 
 9577     addq(count, 128 >> shift);
 9578     jcc(Assembler::zero, L_exit);
 9579     jmp(L_fill_start);
 9580   }
 9581 
 9582   if (MaxVectorSize == 64) {
 9583     // Sequence using 64 byte ZMM register.
 9584     Label L_fill_128_bytes_zmm;
 9585     Label L_fill_192_bytes_zmm;
 9586     Label L_fill_192_bytes_loop_zmm;
 9587     Label L_fill_192_bytes_loop_header_zmm;
 9588     Label L_fill_192_bytes_loop_pre_header_zmm;
 9589     Label L_fill_start_zmm_sequence;
 9590 
 9591     bind(L_fill_zmm_sequence);
 9592     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
 9593 
 9594     bind(L_fill_start_zmm_sequence);
 9595     cmpq(count, 64 >> shift);
 9596     jccb(Assembler::greater, L_fill_128_bytes_zmm);
 9597     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
 9598     jmp(L_exit);
 9599 
 9600     bind(L_fill_128_bytes_zmm);
 9601     cmpq(count, 128 >> shift);
 9602     jccb(Assembler::greater, L_fill_192_bytes_zmm);
 9603     fill64(to, 0, xtmp, true);
 9604     subq(count, 64 >> shift);
 9605     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
 9606     jmp(L_exit);
 9607 
 9608     bind(L_fill_192_bytes_zmm);
 9609     cmpq(count, 192 >> shift);
 9610     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
 9611     fill64(to, 0, xtmp, true);
 9612     fill64(to, 64, xtmp, true);
 9613     subq(count, 128 >> shift);
 9614     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
 9615     jmp(L_exit);
 9616 
 9617     bind(L_fill_192_bytes_loop_pre_header_zmm);
 9618     {
 9619       movq(rtmp, to);
 9620       andq(rtmp, 63);
 9621       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
 9622       negq(rtmp);
 9623       addq(rtmp, 64);
 9624       mov64(r8, -1L);
 9625       bzhiq(r8, r8, rtmp);
 9626       kmovql(k2, r8);
 9627       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
 9628       addq(to, rtmp);
 9629       shrq(rtmp, shift);
 9630       subq(count, rtmp);
 9631     }
 9632 
 9633     cmpq(count, 192 >> shift);
 9634     jcc(Assembler::less, L_fill_start_zmm_sequence);
 9635 
 9636     bind(L_fill_192_bytes_loop_header_zmm);
 9637     subq(count, 192 >> shift);
 9638 
 9639     align32();
 9640     bind(L_fill_192_bytes_loop_zmm);
 9641       fill64(to, 0, xtmp, true);
 9642       fill64(to, 64, xtmp, true);
 9643       fill64(to, 128, xtmp, true);
 9644       addq(to, 192);
 9645       subq(count, 192 >> shift);
 9646       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
 9647 
 9648     addq(count, 192 >> shift);
 9649     jcc(Assembler::zero, L_exit);
 9650     jmp(L_fill_start_zmm_sequence);
 9651   }
 9652   bind(L_exit);
 9653 }
 9654 #endif
 9655 #endif //COMPILER2_OR_JVMCI
 9656 
 9657 
 9658 #ifdef _LP64
 9659 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
 9660   Label done;
 9661   cvttss2sil(dst, src);
 9662   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
 9663   cmpl(dst, 0x80000000); // float_sign_flip
 9664   jccb(Assembler::notEqual, done);
 9665   subptr(rsp, 8);
 9666   movflt(Address(rsp, 0), src);
 9667   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
 9668   pop(dst);
 9669   bind(done);
 9670 }
 9671 
 9672 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
 9673   Label done;
 9674   cvttsd2sil(dst, src);
 9675   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
 9676   cmpl(dst, 0x80000000); // float_sign_flip
 9677   jccb(Assembler::notEqual, done);
 9678   subptr(rsp, 8);
 9679   movdbl(Address(rsp, 0), src);
 9680   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
 9681   pop(dst);
 9682   bind(done);
 9683 }
 9684 
 9685 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
 9686   Label done;
 9687   cvttss2siq(dst, src);
 9688   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
 9689   jccb(Assembler::notEqual, done);
 9690   subptr(rsp, 8);
 9691   movflt(Address(rsp, 0), src);
 9692   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
 9693   pop(dst);
 9694   bind(done);
 9695 }
 9696 
 9697 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
 9698   // Following code is line by line assembly translation rounding algorithm.
 9699   // Please refer to java.lang.Math.round(float) algorithm for details.
 9700   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
 9701   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
 9702   const int32_t FloatConsts_EXP_BIAS = 127;
 9703   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
 9704   const int32_t MINUS_32 = 0xFFFFFFE0;
 9705   Label L_special_case, L_block1, L_exit;
 9706   movl(rtmp, FloatConsts_EXP_BIT_MASK);
 9707   movdl(dst, src);
 9708   andl(dst, rtmp);
 9709   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
 9710   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
 9711   subl(rtmp, dst);
 9712   movl(rcx, rtmp);
 9713   movl(dst, MINUS_32);
 9714   testl(rtmp, dst);
 9715   jccb(Assembler::notEqual, L_special_case);
 9716   movdl(dst, src);
 9717   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
 9718   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
 9719   movdl(rtmp, src);
 9720   testl(rtmp, rtmp);
 9721   jccb(Assembler::greaterEqual, L_block1);
 9722   negl(dst);
 9723   bind(L_block1);
 9724   sarl(dst);
 9725   addl(dst, 0x1);
 9726   sarl(dst, 0x1);
 9727   jmp(L_exit);
 9728   bind(L_special_case);
 9729   convert_f2i(dst, src);
 9730   bind(L_exit);
 9731 }
 9732 
 9733 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
 9734   // Following code is line by line assembly translation rounding algorithm.
 9735   // Please refer to java.lang.Math.round(double) algorithm for details.
 9736   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
 9737   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
 9738   const int64_t DoubleConsts_EXP_BIAS = 1023;
 9739   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
 9740   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
 9741   Label L_special_case, L_block1, L_exit;
 9742   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
 9743   movq(dst, src);
 9744   andq(dst, rtmp);
 9745   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
 9746   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
 9747   subq(rtmp, dst);
 9748   movq(rcx, rtmp);
 9749   mov64(dst, MINUS_64);
 9750   testq(rtmp, dst);
 9751   jccb(Assembler::notEqual, L_special_case);
 9752   movq(dst, src);
 9753   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
 9754   andq(dst, rtmp);
 9755   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
 9756   orq(dst, rtmp);
 9757   movq(rtmp, src);
 9758   testq(rtmp, rtmp);
 9759   jccb(Assembler::greaterEqual, L_block1);
 9760   negq(dst);
 9761   bind(L_block1);
 9762   sarq(dst);
 9763   addq(dst, 0x1);
 9764   sarq(dst, 0x1);
 9765   jmp(L_exit);
 9766   bind(L_special_case);
 9767   convert_d2l(dst, src);
 9768   bind(L_exit);
 9769 }
 9770 
 9771 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
 9772   Label done;
 9773   cvttsd2siq(dst, src);
 9774   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
 9775   jccb(Assembler::notEqual, done);
 9776   subptr(rsp, 8);
 9777   movdbl(Address(rsp, 0), src);
 9778   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
 9779   pop(dst);
 9780   bind(done);
 9781 }
 9782 
 9783 void MacroAssembler::cache_wb(Address line)
 9784 {
 9785   // 64 bit cpus always support clflush
 9786   assert(VM_Version::supports_clflush(), "clflush should be available");
 9787   bool optimized = VM_Version::supports_clflushopt();
 9788   bool no_evict = VM_Version::supports_clwb();
 9789 
 9790   // prefer clwb (writeback without evict) otherwise
 9791   // prefer clflushopt (potentially parallel writeback with evict)
 9792   // otherwise fallback on clflush (serial writeback with evict)
 9793 
 9794   if (optimized) {
 9795     if (no_evict) {
 9796       clwb(line);
 9797     } else {
 9798       clflushopt(line);
 9799     }
 9800   } else {
 9801     // no need for fence when using CLFLUSH
 9802     clflush(line);
 9803   }
 9804 }
 9805 
 9806 void MacroAssembler::cache_wbsync(bool is_pre)
 9807 {
 9808   assert(VM_Version::supports_clflush(), "clflush should be available");
 9809   bool optimized = VM_Version::supports_clflushopt();
 9810   bool no_evict = VM_Version::supports_clwb();
 9811 
 9812   // pick the correct implementation
 9813 
 9814   if (!is_pre && (optimized || no_evict)) {
 9815     // need an sfence for post flush when using clflushopt or clwb
 9816     // otherwise no no need for any synchroniaztion
 9817 
 9818     sfence();
 9819   }
 9820 }
 9821 
 9822 #endif // _LP64
 9823 
 9824 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
 9825   switch (cond) {
 9826     // Note some conditions are synonyms for others
 9827     case Assembler::zero:         return Assembler::notZero;
 9828     case Assembler::notZero:      return Assembler::zero;
 9829     case Assembler::less:         return Assembler::greaterEqual;
 9830     case Assembler::lessEqual:    return Assembler::greater;
 9831     case Assembler::greater:      return Assembler::lessEqual;
 9832     case Assembler::greaterEqual: return Assembler::less;
 9833     case Assembler::below:        return Assembler::aboveEqual;
 9834     case Assembler::belowEqual:   return Assembler::above;
 9835     case Assembler::above:        return Assembler::belowEqual;
 9836     case Assembler::aboveEqual:   return Assembler::below;
 9837     case Assembler::overflow:     return Assembler::noOverflow;
 9838     case Assembler::noOverflow:   return Assembler::overflow;
 9839     case Assembler::negative:     return Assembler::positive;
 9840     case Assembler::positive:     return Assembler::negative;
 9841     case Assembler::parity:       return Assembler::noParity;
 9842     case Assembler::noParity:     return Assembler::parity;
 9843   }
 9844   ShouldNotReachHere(); return Assembler::overflow;
 9845 }
 9846 
 9847 SkipIfEqual::SkipIfEqual(
 9848     MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
 9849   _masm = masm;
 9850   _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
 9851   _masm->jcc(Assembler::equal, _label);
 9852 }
 9853 
 9854 SkipIfEqual::~SkipIfEqual() {
 9855   _masm->bind(_label);
 9856 }
 9857 
 9858 // 32-bit Windows has its own fast-path implementation
 9859 // of get_thread
 9860 #if !defined(WIN32) || defined(_LP64)
 9861 
 9862 // This is simply a call to Thread::current()
 9863 void MacroAssembler::get_thread(Register thread) {
 9864   if (thread != rax) {
 9865     push(rax);
 9866   }
 9867   LP64_ONLY(push(rdi);)
 9868   LP64_ONLY(push(rsi);)
 9869   push(rdx);
 9870   push(rcx);
 9871 #ifdef _LP64
 9872   push(r8);
 9873   push(r9);
 9874   push(r10);
 9875   push(r11);
 9876 #endif
 9877 
 9878   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
 9879 
 9880 #ifdef _LP64
 9881   pop(r11);
 9882   pop(r10);
 9883   pop(r9);
 9884   pop(r8);
 9885 #endif
 9886   pop(rcx);
 9887   pop(rdx);
 9888   LP64_ONLY(pop(rsi);)
 9889   LP64_ONLY(pop(rdi);)
 9890   if (thread != rax) {
 9891     mov(thread, rax);
 9892     pop(rax);
 9893   }
 9894 }
 9895 
 9896 
 9897 #endif // !WIN32 || _LP64
 9898 
 9899 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
 9900   Label L_stack_ok;
 9901   if (bias == 0) {
 9902     testptr(sp, 2 * wordSize - 1);
 9903   } else {
 9904     // lea(tmp, Address(rsp, bias);
 9905     mov(tmp, sp);
 9906     addptr(tmp, bias);
 9907     testptr(tmp, 2 * wordSize - 1);
 9908   }
 9909   jcc(Assembler::equal, L_stack_ok);
 9910   block_comment(msg);
 9911   stop(msg);
 9912   bind(L_stack_ok);
 9913 }
 9914 
 9915 // Implements lightweight-locking.
 9916 //
 9917 // obj: the object to be locked
 9918 // reg_rax: rax
 9919 // thread: the thread which attempts to lock obj
 9920 // tmp: a temporary register
 9921 void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
 9922   assert(reg_rax == rax, "");
 9923   assert_different_registers(obj, reg_rax, thread, tmp);
 9924 
 9925   Label push;
 9926   const Register top = tmp;
 9927 
 9928   // Preload the markWord. It is important that this is the first
 9929   // instruction emitted as it is part of C1's null check semantics.
 9930   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
 9931 
 9932   // Load top.
 9933   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 9934 
 9935   // Check if the lock-stack is full.
 9936   cmpl(top, LockStack::end_offset());
 9937   jcc(Assembler::greaterEqual, slow);
 9938 
 9939   // Check for recursion.
 9940   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
 9941   jcc(Assembler::equal, push);
 9942 
 9943   // Check header for monitor (0b10).
 9944   testptr(reg_rax, markWord::monitor_value);
 9945   jcc(Assembler::notZero, slow);
 9946 
 9947   // Try to lock. Transition lock bits 0b01 => 0b00
 9948   movptr(tmp, reg_rax);
 9949   andptr(tmp, ~(int32_t)markWord::unlocked_value);
 9950   orptr(reg_rax, markWord::unlocked_value);
 9951   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
 9952   jcc(Assembler::notEqual, slow);
 9953 
 9954   // Restore top, CAS clobbers register.
 9955   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 9956 
 9957   bind(push);
 9958   // After successful lock, push object on lock-stack.
 9959   movptr(Address(thread, top), obj);
 9960   incrementl(top, oopSize);
 9961   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
 9962 }
 9963 
 9964 // Implements lightweight-unlocking.
 9965 //
 9966 // obj: the object to be unlocked
 9967 // reg_rax: rax
 9968 // thread: the thread
 9969 // tmp: a temporary register
 9970 //
 9971 // x86_32 Note: reg_rax and thread may alias each other due to limited register
 9972 //              availiability.
 9973 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
 9974   assert(reg_rax == rax, "");
 9975   assert_different_registers(obj, reg_rax, tmp);
 9976   LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
 9977 
 9978   Label unlocked, push_and_slow;
 9979   const Register top = tmp;
 9980 
 9981   // Check if obj is top of lock-stack.
 9982   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 9983   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
 9984   jcc(Assembler::notEqual, slow);
 9985 
 9986   // Pop lock-stack.
 9987   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
 9988   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
 9989 
 9990   // Check if recursive.
 9991   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
 9992   jcc(Assembler::equal, unlocked);
 9993 
 9994   // Not recursive. Check header for monitor (0b10).
 9995   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
 9996   testptr(reg_rax, markWord::monitor_value);
 9997   jcc(Assembler::notZero, push_and_slow);
 9998 
 9999 #ifdef ASSERT
10000   // Check header not unlocked (0b01).
10001   Label not_unlocked;
10002   testptr(reg_rax, markWord::unlocked_value);
10003   jcc(Assembler::zero, not_unlocked);
10004   stop("lightweight_unlock already unlocked");
10005   bind(not_unlocked);
10006 #endif
10007 
10008   // Try to unlock. Transition lock bits 0b00 => 0b01
10009   movptr(tmp, reg_rax);
10010   orptr(tmp, markWord::unlocked_value);
10011   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10012   jcc(Assembler::equal, unlocked);
10013 
10014   bind(push_and_slow);
10015   // Restore lock-stack and handle the unlock in runtime.
10016   if (thread == reg_rax) {
10017     // On x86_32 we may lose the thread.
10018     get_thread(thread);
10019   }
10020 #ifdef ASSERT
10021   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10022   movptr(Address(thread, top), obj);
10023 #endif
10024   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10025   jmp(slow);
10026 
10027   bind(unlocked);
10028 }