1 /*
   2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/compiler_globals.hpp"
  30 #include "compiler/disassembler.hpp"
  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/collectedHeap.inline.hpp"
  34 #include "gc/shared/tlab_globals.hpp"
  35 #include "interpreter/bytecodeHistogram.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "memory/resourceArea.hpp"
  38 #include "memory/universe.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/compressedOops.inline.hpp"
  41 #include "oops/klass.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/flags/flagSetting.hpp"
  44 #include "runtime/interfaceSupport.inline.hpp"
  45 #include "runtime/jniHandles.hpp"
  46 #include "runtime/objectMonitor.hpp"
  47 #include "runtime/os.hpp"
  48 #include "runtime/safepoint.hpp"
  49 #include "runtime/safepointMechanism.hpp"
  50 #include "runtime/sharedRuntime.hpp"
  51 #include "runtime/stubRoutines.hpp"
  52 #include "runtime/thread.hpp"
  53 #include "utilities/macros.hpp"
  54 #include "crc32c.h"
  55 
  56 #ifdef PRODUCT
  57 #define BLOCK_COMMENT(str) /* nothing */
  58 #define STOP(error) stop(error)
  59 #else
  60 #define BLOCK_COMMENT(str) block_comment(str)
  61 #define STOP(error) block_comment(error); stop(error)
  62 #endif
  63 
  64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  65 
  66 #ifdef ASSERT
  67 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  68 #endif
  69 
  70 static Assembler::Condition reverse[] = {
  71     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  72     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  73     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  74     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  75     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  76     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  77     Assembler::above          /* belowEqual    = 0x6 */ ,
  78     Assembler::belowEqual     /* above         = 0x7 */ ,
  79     Assembler::positive       /* negative      = 0x8 */ ,
  80     Assembler::negative       /* positive      = 0x9 */ ,
  81     Assembler::noParity       /* parity        = 0xa */ ,
  82     Assembler::parity         /* noParity      = 0xb */ ,
  83     Assembler::greaterEqual   /* less          = 0xc */ ,
  84     Assembler::less           /* greaterEqual  = 0xd */ ,
  85     Assembler::greater        /* lessEqual     = 0xe */ ,
  86     Assembler::lessEqual      /* greater       = 0xf, */
  87 
  88 };
  89 
  90 
  91 // Implementation of MacroAssembler
  92 
  93 // First all the versions that have distinct versions depending on 32/64 bit
  94 // Unless the difference is trivial (1 line or so).
  95 
  96 #ifndef _LP64
  97 
  98 // 32bit versions
  99 
 100 Address MacroAssembler::as_Address(AddressLiteral adr) {
 101   return Address(adr.target(), adr.rspec());
 102 }
 103 
 104 Address MacroAssembler::as_Address(ArrayAddress adr) {
 105   return Address::make_array(adr);
 106 }
 107 
 108 void MacroAssembler::call_VM_leaf_base(address entry_point,
 109                                        int number_of_arguments) {
 110   call(RuntimeAddress(entry_point));
 111   increment(rsp, number_of_arguments * wordSize);
 112 }
 113 
 114 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 115   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 116 }
 117 
 118 
 119 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 120   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 121 }
 122 
 123 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 124   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 125 }
 126 
 127 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 128   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 129 }
 130 
 131 void MacroAssembler::extend_sign(Register hi, Register lo) {
 132   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 133   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 134     cdql();
 135   } else {
 136     movl(hi, lo);
 137     sarl(hi, 31);
 138   }
 139 }
 140 
 141 void MacroAssembler::jC2(Register tmp, Label& L) {
 142   // set parity bit if FPU flag C2 is set (via rax)
 143   save_rax(tmp);
 144   fwait(); fnstsw_ax();
 145   sahf();
 146   restore_rax(tmp);
 147   // branch
 148   jcc(Assembler::parity, L);
 149 }
 150 
 151 void MacroAssembler::jnC2(Register tmp, Label& L) {
 152   // set parity bit if FPU flag C2 is set (via rax)
 153   save_rax(tmp);
 154   fwait(); fnstsw_ax();
 155   sahf();
 156   restore_rax(tmp);
 157   // branch
 158   jcc(Assembler::noParity, L);
 159 }
 160 
 161 // 32bit can do a case table jump in one instruction but we no longer allow the base
 162 // to be installed in the Address class
 163 void MacroAssembler::jump(ArrayAddress entry) {
 164   jmp(as_Address(entry));
 165 }
 166 
 167 // Note: y_lo will be destroyed
 168 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 169   // Long compare for Java (semantics as described in JVM spec.)
 170   Label high, low, done;
 171 
 172   cmpl(x_hi, y_hi);
 173   jcc(Assembler::less, low);
 174   jcc(Assembler::greater, high);
 175   // x_hi is the return register
 176   xorl(x_hi, x_hi);
 177   cmpl(x_lo, y_lo);
 178   jcc(Assembler::below, low);
 179   jcc(Assembler::equal, done);
 180 
 181   bind(high);
 182   xorl(x_hi, x_hi);
 183   increment(x_hi);
 184   jmp(done);
 185 
 186   bind(low);
 187   xorl(x_hi, x_hi);
 188   decrementl(x_hi);
 189 
 190   bind(done);
 191 }
 192 
 193 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 194     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 195 }
 196 
 197 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 198   // leal(dst, as_Address(adr));
 199   // see note in movl as to why we must use a move
 200   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 201 }
 202 
 203 void MacroAssembler::leave() {
 204   mov(rsp, rbp);
 205   pop(rbp);
 206 }
 207 
 208 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 209   // Multiplication of two Java long values stored on the stack
 210   // as illustrated below. Result is in rdx:rax.
 211   //
 212   // rsp ---> [  ??  ] \               \
 213   //            ....    | y_rsp_offset  |
 214   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 215   //          [ y_hi ]                  | (in bytes)
 216   //            ....                    |
 217   //          [ x_lo ]                 /
 218   //          [ x_hi ]
 219   //            ....
 220   //
 221   // Basic idea: lo(result) = lo(x_lo * y_lo)
 222   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 223   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 224   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 225   Label quick;
 226   // load x_hi, y_hi and check if quick
 227   // multiplication is possible
 228   movl(rbx, x_hi);
 229   movl(rcx, y_hi);
 230   movl(rax, rbx);
 231   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 232   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 233   // do full multiplication
 234   // 1st step
 235   mull(y_lo);                                    // x_hi * y_lo
 236   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 237   // 2nd step
 238   movl(rax, x_lo);
 239   mull(rcx);                                     // x_lo * y_hi
 240   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 241   // 3rd step
 242   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 243   movl(rax, x_lo);
 244   mull(y_lo);                                    // x_lo * y_lo
 245   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 246 }
 247 
 248 void MacroAssembler::lneg(Register hi, Register lo) {
 249   negl(lo);
 250   adcl(hi, 0);
 251   negl(hi);
 252 }
 253 
 254 void MacroAssembler::lshl(Register hi, Register lo) {
 255   // Java shift left long support (semantics as described in JVM spec., p.305)
 256   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 257   // shift value is in rcx !
 258   assert(hi != rcx, "must not use rcx");
 259   assert(lo != rcx, "must not use rcx");
 260   const Register s = rcx;                        // shift count
 261   const int      n = BitsPerWord;
 262   Label L;
 263   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 264   cmpl(s, n);                                    // if (s < n)
 265   jcc(Assembler::less, L);                       // else (s >= n)
 266   movl(hi, lo);                                  // x := x << n
 267   xorl(lo, lo);
 268   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 269   bind(L);                                       // s (mod n) < n
 270   shldl(hi, lo);                                 // x := x << s
 271   shll(lo);
 272 }
 273 
 274 
 275 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 276   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 277   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 278   assert(hi != rcx, "must not use rcx");
 279   assert(lo != rcx, "must not use rcx");
 280   const Register s = rcx;                        // shift count
 281   const int      n = BitsPerWord;
 282   Label L;
 283   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 284   cmpl(s, n);                                    // if (s < n)
 285   jcc(Assembler::less, L);                       // else (s >= n)
 286   movl(lo, hi);                                  // x := x >> n
 287   if (sign_extension) sarl(hi, 31);
 288   else                xorl(hi, hi);
 289   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 290   bind(L);                                       // s (mod n) < n
 291   shrdl(lo, hi);                                 // x := x >> s
 292   if (sign_extension) sarl(hi);
 293   else                shrl(hi);
 294 }
 295 
 296 void MacroAssembler::movoop(Register dst, jobject obj) {
 297   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 298 }
 299 
 300 void MacroAssembler::movoop(Address dst, jobject obj) {
 301   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 302 }
 303 
 304 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 305   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 306 }
 307 
 308 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 309   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 310 }
 311 
 312 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 313   // scratch register is not used,
 314   // it is defined to match parameters of 64-bit version of this method.
 315   if (src.is_lval()) {
 316     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 317   } else {
 318     movl(dst, as_Address(src));
 319   }
 320 }
 321 
 322 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 323   movl(as_Address(dst), src);
 324 }
 325 
 326 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 327   movl(dst, as_Address(src));
 328 }
 329 
 330 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 331 void MacroAssembler::movptr(Address dst, intptr_t src) {
 332   movl(dst, src);
 333 }
 334 
 335 
 336 void MacroAssembler::pop_callee_saved_registers() {
 337   pop(rcx);
 338   pop(rdx);
 339   pop(rdi);
 340   pop(rsi);
 341 }
 342 
 343 void MacroAssembler::push_callee_saved_registers() {
 344   push(rsi);
 345   push(rdi);
 346   push(rdx);
 347   push(rcx);
 348 }
 349 
 350 void MacroAssembler::pushoop(jobject obj) {
 351   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 352 }
 353 
 354 void MacroAssembler::pushklass(Metadata* obj) {
 355   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 356 }
 357 
 358 void MacroAssembler::pushptr(AddressLiteral src) {
 359   if (src.is_lval()) {
 360     push_literal32((int32_t)src.target(), src.rspec());
 361   } else {
 362     pushl(as_Address(src));
 363   }
 364 }
 365 
 366 static void pass_arg0(MacroAssembler* masm, Register arg) {
 367   masm->push(arg);
 368 }
 369 
 370 static void pass_arg1(MacroAssembler* masm, Register arg) {
 371   masm->push(arg);
 372 }
 373 
 374 static void pass_arg2(MacroAssembler* masm, Register arg) {
 375   masm->push(arg);
 376 }
 377 
 378 static void pass_arg3(MacroAssembler* masm, Register arg) {
 379   masm->push(arg);
 380 }
 381 
 382 #ifndef PRODUCT
 383 extern "C" void findpc(intptr_t x);
 384 #endif
 385 
 386 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 387   // In order to get locks to work, we need to fake a in_VM state
 388   JavaThread* thread = JavaThread::current();
 389   JavaThreadState saved_state = thread->thread_state();
 390   thread->set_thread_state(_thread_in_vm);
 391   if (ShowMessageBoxOnError) {
 392     JavaThread* thread = JavaThread::current();
 393     JavaThreadState saved_state = thread->thread_state();
 394     thread->set_thread_state(_thread_in_vm);
 395     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 396       ttyLocker ttyl;
 397       BytecodeCounter::print();
 398     }
 399     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 400     // This is the value of eip which points to where verify_oop will return.
 401     if (os::message_box(msg, "Execution stopped, print registers?")) {
 402       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 403       BREAKPOINT;
 404     }
 405   }
 406   fatal("DEBUG MESSAGE: %s", msg);
 407 }
 408 
 409 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 410   ttyLocker ttyl;
 411   FlagSetting fs(Debugging, true);
 412   tty->print_cr("eip = 0x%08x", eip);
 413 #ifndef PRODUCT
 414   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 415     tty->cr();
 416     findpc(eip);
 417     tty->cr();
 418   }
 419 #endif
 420 #define PRINT_REG(rax) \
 421   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 422   PRINT_REG(rax);
 423   PRINT_REG(rbx);
 424   PRINT_REG(rcx);
 425   PRINT_REG(rdx);
 426   PRINT_REG(rdi);
 427   PRINT_REG(rsi);
 428   PRINT_REG(rbp);
 429   PRINT_REG(rsp);
 430 #undef PRINT_REG
 431   // Print some words near top of staack.
 432   int* dump_sp = (int*) rsp;
 433   for (int col1 = 0; col1 < 8; col1++) {
 434     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 435     os::print_location(tty, *dump_sp++);
 436   }
 437   for (int row = 0; row < 16; row++) {
 438     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 439     for (int col = 0; col < 8; col++) {
 440       tty->print(" 0x%08x", *dump_sp++);
 441     }
 442     tty->cr();
 443   }
 444   // Print some instructions around pc:
 445   Disassembler::decode((address)eip-64, (address)eip);
 446   tty->print_cr("--------");
 447   Disassembler::decode((address)eip, (address)eip+32);
 448 }
 449 
 450 void MacroAssembler::stop(const char* msg) {
 451   ExternalAddress message((address)msg);
 452   // push address of message
 453   pushptr(message.addr());
 454   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 455   pusha();                                            // push registers
 456   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 457   hlt();
 458 }
 459 
 460 void MacroAssembler::warn(const char* msg) {
 461   push_CPU_state();
 462 
 463   ExternalAddress message((address) msg);
 464   // push address of message
 465   pushptr(message.addr());
 466 
 467   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 468   addl(rsp, wordSize);       // discard argument
 469   pop_CPU_state();
 470 }
 471 
 472 void MacroAssembler::print_state() {
 473   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 474   pusha();                                            // push registers
 475 
 476   push_CPU_state();
 477   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 478   pop_CPU_state();
 479 
 480   popa();
 481   addl(rsp, wordSize);
 482 }
 483 
 484 #else // _LP64
 485 
 486 // 64 bit versions
 487 
 488 Address MacroAssembler::as_Address(AddressLiteral adr) {
 489   // amd64 always does this as a pc-rel
 490   // we can be absolute or disp based on the instruction type
 491   // jmp/call are displacements others are absolute
 492   assert(!adr.is_lval(), "must be rval");
 493   assert(reachable(adr), "must be");
 494   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 495 
 496 }
 497 
 498 Address MacroAssembler::as_Address(ArrayAddress adr) {
 499   AddressLiteral base = adr.base();
 500   lea(rscratch1, base);
 501   Address index = adr.index();
 502   assert(index._disp == 0, "must not have disp"); // maybe it can?
 503   Address array(rscratch1, index._index, index._scale, index._disp);
 504   return array;
 505 }
 506 
 507 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 508   Label L, E;
 509 
 510 #ifdef _WIN64
 511   // Windows always allocates space for it's register args
 512   assert(num_args <= 4, "only register arguments supported");
 513   subq(rsp,  frame::arg_reg_save_area_bytes);
 514 #endif
 515 
 516   // Align stack if necessary
 517   testl(rsp, 15);
 518   jcc(Assembler::zero, L);
 519 
 520   subq(rsp, 8);
 521   {
 522     call(RuntimeAddress(entry_point));
 523   }
 524   addq(rsp, 8);
 525   jmp(E);
 526 
 527   bind(L);
 528   {
 529     call(RuntimeAddress(entry_point));
 530   }
 531 
 532   bind(E);
 533 
 534 #ifdef _WIN64
 535   // restore stack pointer
 536   addq(rsp, frame::arg_reg_save_area_bytes);
 537 #endif
 538 
 539 }
 540 
 541 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 542   assert(!src2.is_lval(), "should use cmpptr");
 543 
 544   if (reachable(src2)) {
 545     cmpq(src1, as_Address(src2));
 546   } else {
 547     lea(rscratch1, src2);
 548     Assembler::cmpq(src1, Address(rscratch1, 0));
 549   }
 550 }
 551 
 552 int MacroAssembler::corrected_idivq(Register reg) {
 553   // Full implementation of Java ldiv and lrem; checks for special
 554   // case as described in JVM spec., p.243 & p.271.  The function
 555   // returns the (pc) offset of the idivl instruction - may be needed
 556   // for implicit exceptions.
 557   //
 558   //         normal case                           special case
 559   //
 560   // input : rax: dividend                         min_long
 561   //         reg: divisor   (may not be eax/edx)   -1
 562   //
 563   // output: rax: quotient  (= rax idiv reg)       min_long
 564   //         rdx: remainder (= rax irem reg)       0
 565   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 566   static const int64_t min_long = 0x8000000000000000;
 567   Label normal_case, special_case;
 568 
 569   // check for special case
 570   cmp64(rax, ExternalAddress((address) &min_long));
 571   jcc(Assembler::notEqual, normal_case);
 572   xorl(rdx, rdx); // prepare rdx for possible special case (where
 573                   // remainder = 0)
 574   cmpq(reg, -1);
 575   jcc(Assembler::equal, special_case);
 576 
 577   // handle normal case
 578   bind(normal_case);
 579   cdqq();
 580   int idivq_offset = offset();
 581   idivq(reg);
 582 
 583   // normal and special case exit
 584   bind(special_case);
 585 
 586   return idivq_offset;
 587 }
 588 
 589 void MacroAssembler::decrementq(Register reg, int value) {
 590   if (value == min_jint) { subq(reg, value); return; }
 591   if (value <  0) { incrementq(reg, -value); return; }
 592   if (value == 0) {                        ; return; }
 593   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 594   /* else */      { subq(reg, value)       ; return; }
 595 }
 596 
 597 void MacroAssembler::decrementq(Address dst, int value) {
 598   if (value == min_jint) { subq(dst, value); return; }
 599   if (value <  0) { incrementq(dst, -value); return; }
 600   if (value == 0) {                        ; return; }
 601   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 602   /* else */      { subq(dst, value)       ; return; }
 603 }
 604 
 605 void MacroAssembler::incrementq(AddressLiteral dst) {
 606   if (reachable(dst)) {
 607     incrementq(as_Address(dst));
 608   } else {
 609     lea(rscratch1, dst);
 610     incrementq(Address(rscratch1, 0));
 611   }
 612 }
 613 
 614 void MacroAssembler::incrementq(Register reg, int value) {
 615   if (value == min_jint) { addq(reg, value); return; }
 616   if (value <  0) { decrementq(reg, -value); return; }
 617   if (value == 0) {                        ; return; }
 618   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 619   /* else */      { addq(reg, value)       ; return; }
 620 }
 621 
 622 void MacroAssembler::incrementq(Address dst, int value) {
 623   if (value == min_jint) { addq(dst, value); return; }
 624   if (value <  0) { decrementq(dst, -value); return; }
 625   if (value == 0) {                        ; return; }
 626   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 627   /* else */      { addq(dst, value)       ; return; }
 628 }
 629 
 630 // 32bit can do a case table jump in one instruction but we no longer allow the base
 631 // to be installed in the Address class
 632 void MacroAssembler::jump(ArrayAddress entry) {
 633   lea(rscratch1, entry.base());
 634   Address dispatch = entry.index();
 635   assert(dispatch._base == noreg, "must be");
 636   dispatch._base = rscratch1;
 637   jmp(dispatch);
 638 }
 639 
 640 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 641   ShouldNotReachHere(); // 64bit doesn't use two regs
 642   cmpq(x_lo, y_lo);
 643 }
 644 
 645 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 646     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 647 }
 648 
 649 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 650   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 651   movptr(dst, rscratch1);
 652 }
 653 
 654 void MacroAssembler::leave() {
 655   // %%% is this really better? Why not on 32bit too?
 656   emit_int8((unsigned char)0xC9); // LEAVE
 657 }
 658 
 659 void MacroAssembler::lneg(Register hi, Register lo) {
 660   ShouldNotReachHere(); // 64bit doesn't use two regs
 661   negq(lo);
 662 }
 663 
 664 void MacroAssembler::movoop(Register dst, jobject obj) {
 665   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 666 }
 667 
 668 void MacroAssembler::movoop(Address dst, jobject obj) {
 669   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 670   movq(dst, rscratch1);
 671 }
 672 
 673 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 674   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 675 }
 676 
 677 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 678   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 679   movq(dst, rscratch1);
 680 }
 681 
 682 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 683   if (src.is_lval()) {
 684     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 685   } else {
 686     if (reachable(src)) {
 687       movq(dst, as_Address(src));
 688     } else {
 689       lea(scratch, src);
 690       movq(dst, Address(scratch, 0));
 691     }
 692   }
 693 }
 694 
 695 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 696   movq(as_Address(dst), src);
 697 }
 698 
 699 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 700   movq(dst, as_Address(src));
 701 }
 702 
 703 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 704 void MacroAssembler::movptr(Address dst, intptr_t src) {
 705   if (is_simm32(src)) {
 706     movptr(dst, checked_cast<int32_t>(src));
 707   } else {
 708     mov64(rscratch1, src);
 709     movq(dst, rscratch1);
 710   }
 711 }
 712 
 713 // These are mostly for initializing NULL
 714 void MacroAssembler::movptr(Address dst, int32_t src) {
 715   movslq(dst, src);
 716 }
 717 
 718 void MacroAssembler::movptr(Register dst, int32_t src) {
 719   mov64(dst, (intptr_t)src);
 720 }
 721 
 722 void MacroAssembler::pushoop(jobject obj) {
 723   movoop(rscratch1, obj);
 724   push(rscratch1);
 725 }
 726 
 727 void MacroAssembler::pushklass(Metadata* obj) {
 728   mov_metadata(rscratch1, obj);
 729   push(rscratch1);
 730 }
 731 
 732 void MacroAssembler::pushptr(AddressLiteral src) {
 733   lea(rscratch1, src);
 734   if (src.is_lval()) {
 735     push(rscratch1);
 736   } else {
 737     pushq(Address(rscratch1, 0));
 738   }
 739 }
 740 
 741 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 742   reset_last_Java_frame(r15_thread, clear_fp);
 743 }
 744 
 745 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 746                                          Register last_java_fp,
 747                                          address  last_java_pc) {
 748   vzeroupper();
 749   // determine last_java_sp register
 750   if (!last_java_sp->is_valid()) {
 751     last_java_sp = rsp;
 752   }
 753 
 754   // last_java_fp is optional
 755   if (last_java_fp->is_valid()) {
 756     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 757            last_java_fp);
 758   }
 759 
 760   // last_java_pc is optional
 761   if (last_java_pc != NULL) {
 762     Address java_pc(r15_thread,
 763                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 764     lea(rscratch1, InternalAddress(last_java_pc));
 765     movptr(java_pc, rscratch1);
 766   }
 767 
 768   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 769 }
 770 
 771 static void pass_arg0(MacroAssembler* masm, Register arg) {
 772   if (c_rarg0 != arg ) {
 773     masm->mov(c_rarg0, arg);
 774   }
 775 }
 776 
 777 static void pass_arg1(MacroAssembler* masm, Register arg) {
 778   if (c_rarg1 != arg ) {
 779     masm->mov(c_rarg1, arg);
 780   }
 781 }
 782 
 783 static void pass_arg2(MacroAssembler* masm, Register arg) {
 784   if (c_rarg2 != arg ) {
 785     masm->mov(c_rarg2, arg);
 786   }
 787 }
 788 
 789 static void pass_arg3(MacroAssembler* masm, Register arg) {
 790   if (c_rarg3 != arg ) {
 791     masm->mov(c_rarg3, arg);
 792   }
 793 }
 794 
 795 void MacroAssembler::stop(const char* msg) {
 796   if (ShowMessageBoxOnError) {
 797     address rip = pc();
 798     pusha(); // get regs on stack
 799     lea(c_rarg1, InternalAddress(rip));
 800     movq(c_rarg2, rsp); // pass pointer to regs array
 801   }
 802   lea(c_rarg0, ExternalAddress((address) msg));
 803   andq(rsp, -16); // align stack as required by ABI
 804   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 805   hlt();
 806 }
 807 
 808 void MacroAssembler::warn(const char* msg) {
 809   push(rbp);
 810   movq(rbp, rsp);
 811   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 812   push_CPU_state();   // keeps alignment at 16 bytes
 813   lea(c_rarg0, ExternalAddress((address) msg));
 814   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 815   call(rax);
 816   pop_CPU_state();
 817   mov(rsp, rbp);
 818   pop(rbp);
 819 }
 820 
 821 void MacroAssembler::print_state() {
 822   address rip = pc();
 823   pusha();            // get regs on stack
 824   push(rbp);
 825   movq(rbp, rsp);
 826   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 827   push_CPU_state();   // keeps alignment at 16 bytes
 828 
 829   lea(c_rarg0, InternalAddress(rip));
 830   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 831   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 832 
 833   pop_CPU_state();
 834   mov(rsp, rbp);
 835   pop(rbp);
 836   popa();
 837 }
 838 
 839 #ifndef PRODUCT
 840 extern "C" void findpc(intptr_t x);
 841 #endif
 842 
 843 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 844   // In order to get locks to work, we need to fake a in_VM state
 845   if (ShowMessageBoxOnError) {
 846     JavaThread* thread = JavaThread::current();
 847     JavaThreadState saved_state = thread->thread_state();
 848     thread->set_thread_state(_thread_in_vm);
 849 #ifndef PRODUCT
 850     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 851       ttyLocker ttyl;
 852       BytecodeCounter::print();
 853     }
 854 #endif
 855     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 856     // XXX correct this offset for amd64
 857     // This is the value of eip which points to where verify_oop will return.
 858     if (os::message_box(msg, "Execution stopped, print registers?")) {
 859       print_state64(pc, regs);
 860       BREAKPOINT;
 861     }
 862   }
 863   fatal("DEBUG MESSAGE: %s", msg);
 864 }
 865 
 866 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 867   ttyLocker ttyl;
 868   FlagSetting fs(Debugging, true);
 869   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 870 #ifndef PRODUCT
 871   tty->cr();
 872   findpc(pc);
 873   tty->cr();
 874 #endif
 875 #define PRINT_REG(rax, value) \
 876   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 877   PRINT_REG(rax, regs[15]);
 878   PRINT_REG(rbx, regs[12]);
 879   PRINT_REG(rcx, regs[14]);
 880   PRINT_REG(rdx, regs[13]);
 881   PRINT_REG(rdi, regs[8]);
 882   PRINT_REG(rsi, regs[9]);
 883   PRINT_REG(rbp, regs[10]);
 884   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
 885   PRINT_REG(rsp, (intptr_t)(&regs[16]));
 886   PRINT_REG(r8 , regs[7]);
 887   PRINT_REG(r9 , regs[6]);
 888   PRINT_REG(r10, regs[5]);
 889   PRINT_REG(r11, regs[4]);
 890   PRINT_REG(r12, regs[3]);
 891   PRINT_REG(r13, regs[2]);
 892   PRINT_REG(r14, regs[1]);
 893   PRINT_REG(r15, regs[0]);
 894 #undef PRINT_REG
 895   // Print some words near the top of the stack.
 896   int64_t* rsp = &regs[16];
 897   int64_t* dump_sp = rsp;
 898   for (int col1 = 0; col1 < 8; col1++) {
 899     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 900     os::print_location(tty, *dump_sp++);
 901   }
 902   for (int row = 0; row < 25; row++) {
 903     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 904     for (int col = 0; col < 4; col++) {
 905       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 906     }
 907     tty->cr();
 908   }
 909   // Print some instructions around pc:
 910   Disassembler::decode((address)pc-64, (address)pc);
 911   tty->print_cr("--------");
 912   Disassembler::decode((address)pc, (address)pc+32);
 913 }
 914 
 915 // The java_calling_convention describes stack locations as ideal slots on
 916 // a frame with no abi restrictions. Since we must observe abi restrictions
 917 // (like the placement of the register window) the slots must be biased by
 918 // the following value.
 919 static int reg2offset_in(VMReg r) {
 920   // Account for saved rbp and return address
 921   // This should really be in_preserve_stack_slots
 922   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
 923 }
 924 
 925 static int reg2offset_out(VMReg r) {
 926   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 927 }
 928 
 929 // A long move
 930 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst) {
 931 
 932   // The calling conventions assures us that each VMregpair is either
 933   // all really one physical register or adjacent stack slots.
 934 
 935   if (src.is_single_phys_reg() ) {
 936     if (dst.is_single_phys_reg()) {
 937       if (dst.first() != src.first()) {
 938         mov(dst.first()->as_Register(), src.first()->as_Register());
 939       }
 940     } else {
 941       assert(dst.is_single_reg(), "not a stack pair");
 942       movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 943     }
 944   } else if (dst.is_single_phys_reg()) {
 945     assert(src.is_single_reg(),  "not a stack pair");
 946     movq(dst.first()->as_Register(), Address(rbp, reg2offset_out(src.first())));
 947   } else {
 948     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 949     movq(rax, Address(rbp, reg2offset_in(src.first())));
 950     movq(Address(rsp, reg2offset_out(dst.first())), rax);
 951   }
 952 }
 953 
 954 // A double move
 955 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst) {
 956 
 957   // The calling conventions assures us that each VMregpair is either
 958   // all really one physical register or adjacent stack slots.
 959 
 960   if (src.is_single_phys_reg() ) {
 961     if (dst.is_single_phys_reg()) {
 962       // In theory these overlap but the ordering is such that this is likely a nop
 963       if ( src.first() != dst.first()) {
 964         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
 965       }
 966     } else {
 967       assert(dst.is_single_reg(), "not a stack pair");
 968       movdbl(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
 969     }
 970   } else if (dst.is_single_phys_reg()) {
 971     assert(src.is_single_reg(),  "not a stack pair");
 972     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_out(src.first())));
 973   } else {
 974     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 975     movq(rax, Address(rbp, reg2offset_in(src.first())));
 976     movq(Address(rsp, reg2offset_out(dst.first())), rax);
 977   }
 978 }
 979 
 980 
 981 // A float arg may have to do float reg int reg conversion
 982 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst) {
 983   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
 984 
 985   // The calling conventions assures us that each VMregpair is either
 986   // all really one physical register or adjacent stack slots.
 987 
 988   if (src.first()->is_stack()) {
 989     if (dst.first()->is_stack()) {
 990       movl(rax, Address(rbp, reg2offset_in(src.first())));
 991       movptr(Address(rsp, reg2offset_out(dst.first())), rax);
 992     } else {
 993       // stack to reg
 994       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
 995       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first())));
 996     }
 997   } else if (dst.first()->is_stack()) {
 998     // reg to stack
 999     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
1000     movflt(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1001   } else {
1002     // reg to reg
1003     // In theory these overlap but the ordering is such that this is likely a nop
1004     if ( src.first() != dst.first()) {
1005       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
1006     }
1007   }
1008 }
1009 
1010 // On 64 bit we will store integer like items to the stack as
1011 // 64 bits items (x86_32/64 abi) even though java would only store
1012 // 32bits for a parameter. On 32bit it will simply be 32 bits
1013 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
1014 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst) {
1015   if (src.first()->is_stack()) {
1016     if (dst.first()->is_stack()) {
1017       // stack to stack
1018       movslq(rax, Address(rbp, reg2offset_in(src.first())));
1019       movq(Address(rsp, reg2offset_out(dst.first())), rax);
1020     } else {
1021       // stack to reg
1022       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1023     }
1024   } else if (dst.first()->is_stack()) {
1025     // reg to stack
1026     // Do we really have to sign extend???
1027     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1028     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1029   } else {
1030     // Do we really have to sign extend???
1031     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
1032     if (dst.first() != src.first()) {
1033       movq(dst.first()->as_Register(), src.first()->as_Register());
1034     }
1035   }
1036 }
1037 
1038 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
1039   if (src.first()->is_stack()) {
1040     if (dst.first()->is_stack()) {
1041       // stack to stack
1042       movq(rax, Address(rbp, reg2offset_in(src.first())));
1043       movq(Address(rsp, reg2offset_out(dst.first())), rax);
1044     } else {
1045       // stack to reg
1046       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1047     }
1048   } else if (dst.first()->is_stack()) {
1049     // reg to stack
1050     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1051   } else {
1052     if (dst.first() != src.first()) {
1053       movq(dst.first()->as_Register(), src.first()->as_Register());
1054     }
1055   }
1056 }
1057 
1058 // An oop arg. Must pass a handle not the oop itself
1059 void MacroAssembler::object_move(OopMap* map,
1060                         int oop_handle_offset,
1061                         int framesize_in_slots,
1062                         VMRegPair src,
1063                         VMRegPair dst,
1064                         bool is_receiver,
1065                         int* receiver_offset) {
1066 
1067   // must pass a handle. First figure out the location we use as a handle
1068 
1069   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
1070 
1071   // See if oop is NULL if it is we need no handle
1072 
1073   if (src.first()->is_stack()) {
1074 
1075     // Oop is already on the stack as an argument
1076     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1077     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
1078     if (is_receiver) {
1079       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
1080     }
1081 
1082     cmpptr(Address(rbp, reg2offset_in(src.first())), (int32_t)NULL_WORD);
1083     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
1084     // conditionally move a NULL
1085     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
1086   } else {
1087 
1088     // Oop is in an a register we must store it to the space we reserve
1089     // on the stack for oop_handles and pass a handle if oop is non-NULL
1090 
1091     const Register rOop = src.first()->as_Register();
1092     int oop_slot;
1093     if (rOop == j_rarg0)
1094       oop_slot = 0;
1095     else if (rOop == j_rarg1)
1096       oop_slot = 1;
1097     else if (rOop == j_rarg2)
1098       oop_slot = 2;
1099     else if (rOop == j_rarg3)
1100       oop_slot = 3;
1101     else if (rOop == j_rarg4)
1102       oop_slot = 4;
1103     else {
1104       assert(rOop == j_rarg5, "wrong register");
1105       oop_slot = 5;
1106     }
1107 
1108     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
1109     int offset = oop_slot*VMRegImpl::stack_slot_size;
1110 
1111     map->set_oop(VMRegImpl::stack2reg(oop_slot));
1112     // Store oop in handle area, may be NULL
1113     movptr(Address(rsp, offset), rOop);
1114     if (is_receiver) {
1115       *receiver_offset = offset;
1116     }
1117 
1118     cmpptr(rOop, (int32_t)NULL_WORD);
1119     lea(rHandle, Address(rsp, offset));
1120     // conditionally move a NULL from the handle area where it was just stored
1121     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
1122   }
1123 
1124   // If arg is on the stack then place it otherwise it is already in correct reg.
1125   if (dst.first()->is_stack()) {
1126     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
1127   }
1128 }
1129 
1130 #endif // _LP64
1131 
1132 // Now versions that are common to 32/64 bit
1133 
1134 void MacroAssembler::addptr(Register dst, int32_t imm32) {
1135   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
1136 }
1137 
1138 void MacroAssembler::addptr(Register dst, Register src) {
1139   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
1140 }
1141 
1142 void MacroAssembler::addptr(Address dst, Register src) {
1143   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
1144 }
1145 
1146 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
1147   if (reachable(src)) {
1148     Assembler::addsd(dst, as_Address(src));
1149   } else {
1150     lea(rscratch1, src);
1151     Assembler::addsd(dst, Address(rscratch1, 0));
1152   }
1153 }
1154 
1155 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
1156   if (reachable(src)) {
1157     addss(dst, as_Address(src));
1158   } else {
1159     lea(rscratch1, src);
1160     addss(dst, Address(rscratch1, 0));
1161   }
1162 }
1163 
1164 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
1165   if (reachable(src)) {
1166     Assembler::addpd(dst, as_Address(src));
1167   } else {
1168     lea(rscratch1, src);
1169     Assembler::addpd(dst, Address(rscratch1, 0));
1170   }
1171 }
1172 
1173 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
1174 // Stub code is generated once and never copied.
1175 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
1176 void MacroAssembler::align64() {
1177   align(64, (unsigned long long) pc());
1178 }
1179 
1180 void MacroAssembler::align(int modulus) {
1181   // 8273459: Ensure alignment is possible with current segment alignment
1182   assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
1183   align(modulus, offset());
1184 }
1185 
1186 void MacroAssembler::align(int modulus, int target) {
1187   if (target % modulus != 0) {
1188     nop(modulus - (target % modulus));
1189   }
1190 }
1191 
1192 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
1193   // Used in sign-masking with aligned address.
1194   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1195   if (reachable(src)) {
1196     Assembler::andpd(dst, as_Address(src));
1197   } else {
1198     lea(scratch_reg, src);
1199     Assembler::andpd(dst, Address(scratch_reg, 0));
1200   }
1201 }
1202 
1203 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
1204   // Used in sign-masking with aligned address.
1205   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1206   if (reachable(src)) {
1207     Assembler::andps(dst, as_Address(src));
1208   } else {
1209     lea(scratch_reg, src);
1210     Assembler::andps(dst, Address(scratch_reg, 0));
1211   }
1212 }
1213 
1214 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1215   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1216 }
1217 
1218 void MacroAssembler::atomic_incl(Address counter_addr) {
1219   lock();
1220   incrementl(counter_addr);
1221 }
1222 
1223 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1224   if (reachable(counter_addr)) {
1225     atomic_incl(as_Address(counter_addr));
1226   } else {
1227     lea(scr, counter_addr);
1228     atomic_incl(Address(scr, 0));
1229   }
1230 }
1231 
1232 #ifdef _LP64
1233 void MacroAssembler::atomic_incq(Address counter_addr) {
1234   lock();
1235   incrementq(counter_addr);
1236 }
1237 
1238 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1239   if (reachable(counter_addr)) {
1240     atomic_incq(as_Address(counter_addr));
1241   } else {
1242     lea(scr, counter_addr);
1243     atomic_incq(Address(scr, 0));
1244   }
1245 }
1246 #endif
1247 
1248 // Writes to stack successive pages until offset reached to check for
1249 // stack overflow + shadow pages.  This clobbers tmp.
1250 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1251   movptr(tmp, rsp);
1252   // Bang stack for total size given plus shadow page size.
1253   // Bang one page at a time because large size can bang beyond yellow and
1254   // red zones.
1255   Label loop;
1256   bind(loop);
1257   movl(Address(tmp, (-os::vm_page_size())), size );
1258   subptr(tmp, os::vm_page_size());
1259   subl(size, os::vm_page_size());
1260   jcc(Assembler::greater, loop);
1261 
1262   // Bang down shadow pages too.
1263   // At this point, (tmp-0) is the last address touched, so don't
1264   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1265   // was post-decremented.)  Skip this address by starting at i=1, and
1266   // touch a few more pages below.  N.B.  It is important to touch all
1267   // the way down including all pages in the shadow zone.
1268   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1269     // this could be any sized move but this is can be a debugging crumb
1270     // so the bigger the better.
1271     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1272   }
1273 }
1274 
1275 void MacroAssembler::reserved_stack_check() {
1276     // testing if reserved zone needs to be enabled
1277     Label no_reserved_zone_enabling;
1278     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1279     NOT_LP64(get_thread(rsi);)
1280 
1281     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1282     jcc(Assembler::below, no_reserved_zone_enabling);
1283 
1284     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1285     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1286     should_not_reach_here();
1287 
1288     bind(no_reserved_zone_enabling);
1289 }
1290 
1291 void MacroAssembler::c2bool(Register x) {
1292   // implements x == 0 ? 0 : 1
1293   // note: must only look at least-significant byte of x
1294   //       since C-style booleans are stored in one byte
1295   //       only! (was bug)
1296   andl(x, 0xFF);
1297   setb(Assembler::notZero, x);
1298 }
1299 
1300 // Wouldn't need if AddressLiteral version had new name
1301 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
1302   Assembler::call(L, rtype);
1303 }
1304 
1305 void MacroAssembler::call(Register entry) {
1306   Assembler::call(entry);
1307 }
1308 
1309 void MacroAssembler::call(AddressLiteral entry) {
1310   if (reachable(entry)) {
1311     Assembler::call_literal(entry.target(), entry.rspec());
1312   } else {
1313     lea(rscratch1, entry);
1314     Assembler::call(rscratch1);
1315   }
1316 }
1317 
1318 void MacroAssembler::ic_call(address entry, jint method_index) {
1319   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
1320   movptr(rax, (intptr_t)Universe::non_oop_word());
1321   call(AddressLiteral(entry, rh));
1322 }
1323 
1324 // Implementation of call_VM versions
1325 
1326 void MacroAssembler::call_VM(Register oop_result,
1327                              address entry_point,
1328                              bool check_exceptions) {
1329   Label C, E;
1330   call(C, relocInfo::none);
1331   jmp(E);
1332 
1333   bind(C);
1334   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
1335   ret(0);
1336 
1337   bind(E);
1338 }
1339 
1340 void MacroAssembler::call_VM(Register oop_result,
1341                              address entry_point,
1342                              Register arg_1,
1343                              bool check_exceptions) {
1344   Label C, E;
1345   call(C, relocInfo::none);
1346   jmp(E);
1347 
1348   bind(C);
1349   pass_arg1(this, arg_1);
1350   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
1351   ret(0);
1352 
1353   bind(E);
1354 }
1355 
1356 void MacroAssembler::call_VM(Register oop_result,
1357                              address entry_point,
1358                              Register arg_1,
1359                              Register arg_2,
1360                              bool check_exceptions) {
1361   Label C, E;
1362   call(C, relocInfo::none);
1363   jmp(E);
1364 
1365   bind(C);
1366 
1367   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1368 
1369   pass_arg2(this, arg_2);
1370   pass_arg1(this, arg_1);
1371   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
1372   ret(0);
1373 
1374   bind(E);
1375 }
1376 
1377 void MacroAssembler::call_VM(Register oop_result,
1378                              address entry_point,
1379                              Register arg_1,
1380                              Register arg_2,
1381                              Register arg_3,
1382                              bool check_exceptions) {
1383   Label C, E;
1384   call(C, relocInfo::none);
1385   jmp(E);
1386 
1387   bind(C);
1388 
1389   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
1390   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
1391   pass_arg3(this, arg_3);
1392 
1393   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1394   pass_arg2(this, arg_2);
1395 
1396   pass_arg1(this, arg_1);
1397   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
1398   ret(0);
1399 
1400   bind(E);
1401 }
1402 
1403 void MacroAssembler::call_VM(Register oop_result,
1404                              Register last_java_sp,
1405                              address entry_point,
1406                              int number_of_arguments,
1407                              bool check_exceptions) {
1408   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
1409   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1410 }
1411 
1412 void MacroAssembler::call_VM(Register oop_result,
1413                              Register last_java_sp,
1414                              address entry_point,
1415                              Register arg_1,
1416                              bool check_exceptions) {
1417   pass_arg1(this, arg_1);
1418   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1419 }
1420 
1421 void MacroAssembler::call_VM(Register oop_result,
1422                              Register last_java_sp,
1423                              address entry_point,
1424                              Register arg_1,
1425                              Register arg_2,
1426                              bool check_exceptions) {
1427 
1428   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1429   pass_arg2(this, arg_2);
1430   pass_arg1(this, arg_1);
1431   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1432 }
1433 
1434 void MacroAssembler::call_VM(Register oop_result,
1435                              Register last_java_sp,
1436                              address entry_point,
1437                              Register arg_1,
1438                              Register arg_2,
1439                              Register arg_3,
1440                              bool check_exceptions) {
1441   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
1442   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
1443   pass_arg3(this, arg_3);
1444   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1445   pass_arg2(this, arg_2);
1446   pass_arg1(this, arg_1);
1447   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1448 }
1449 
1450 void MacroAssembler::super_call_VM(Register oop_result,
1451                                    Register last_java_sp,
1452                                    address entry_point,
1453                                    int number_of_arguments,
1454                                    bool check_exceptions) {
1455   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
1456   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1457 }
1458 
1459 void MacroAssembler::super_call_VM(Register oop_result,
1460                                    Register last_java_sp,
1461                                    address entry_point,
1462                                    Register arg_1,
1463                                    bool check_exceptions) {
1464   pass_arg1(this, arg_1);
1465   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1466 }
1467 
1468 void MacroAssembler::super_call_VM(Register oop_result,
1469                                    Register last_java_sp,
1470                                    address entry_point,
1471                                    Register arg_1,
1472                                    Register arg_2,
1473                                    bool check_exceptions) {
1474 
1475   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1476   pass_arg2(this, arg_2);
1477   pass_arg1(this, arg_1);
1478   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1479 }
1480 
1481 void MacroAssembler::super_call_VM(Register oop_result,
1482                                    Register last_java_sp,
1483                                    address entry_point,
1484                                    Register arg_1,
1485                                    Register arg_2,
1486                                    Register arg_3,
1487                                    bool check_exceptions) {
1488   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
1489   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
1490   pass_arg3(this, arg_3);
1491   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1492   pass_arg2(this, arg_2);
1493   pass_arg1(this, arg_1);
1494   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1495 }
1496 
1497 void MacroAssembler::call_VM_base(Register oop_result,
1498                                   Register java_thread,
1499                                   Register last_java_sp,
1500                                   address  entry_point,
1501                                   int      number_of_arguments,
1502                                   bool     check_exceptions) {
1503   // determine java_thread register
1504   if (!java_thread->is_valid()) {
1505 #ifdef _LP64
1506     java_thread = r15_thread;
1507 #else
1508     java_thread = rdi;
1509     get_thread(java_thread);
1510 #endif // LP64
1511   }
1512   // determine last_java_sp register
1513   if (!last_java_sp->is_valid()) {
1514     last_java_sp = rsp;
1515   }
1516   // debugging support
1517   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
1518   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
1519 #ifdef ASSERT
1520   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
1521   // r12 is the heapbase.
1522   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
1523 #endif // ASSERT
1524 
1525   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
1526   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
1527 
1528   // push java thread (becomes first argument of C function)
1529 
1530   NOT_LP64(push(java_thread); number_of_arguments++);
1531   LP64_ONLY(mov(c_rarg0, r15_thread));
1532 
1533   // set last Java frame before call
1534   assert(last_java_sp != rbp, "can't use ebp/rbp");
1535 
1536   // Only interpreter should have to set fp
1537   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
1538 
1539   // do the call, remove parameters
1540   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1541 
1542   // restore the thread (cannot use the pushed argument since arguments
1543   // may be overwritten by C code generated by an optimizing compiler);
1544   // however can use the register value directly if it is callee saved.
1545   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
1546     // rdi & rsi (also r15) are callee saved -> nothing to do
1547 #ifdef ASSERT
1548     guarantee(java_thread != rax, "change this code");
1549     push(rax);
1550     { Label L;
1551       get_thread(rax);
1552       cmpptr(java_thread, rax);
1553       jcc(Assembler::equal, L);
1554       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
1555       bind(L);
1556     }
1557     pop(rax);
1558 #endif
1559   } else {
1560     get_thread(java_thread);
1561   }
1562   // reset last Java frame
1563   // Only interpreter should have to clear fp
1564   reset_last_Java_frame(java_thread, true);
1565 
1566    // C++ interp handles this in the interpreter
1567   check_and_handle_popframe(java_thread);
1568   check_and_handle_earlyret(java_thread);
1569 
1570   if (check_exceptions) {
1571     // check for pending exceptions (java_thread is set upon return)
1572     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
1573 #ifndef _LP64
1574     jump_cc(Assembler::notEqual,
1575             RuntimeAddress(StubRoutines::forward_exception_entry()));
1576 #else
1577     // This used to conditionally jump to forward_exception however it is
1578     // possible if we relocate that the branch will not reach. So we must jump
1579     // around so we can always reach
1580 
1581     Label ok;
1582     jcc(Assembler::equal, ok);
1583     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1584     bind(ok);
1585 #endif // LP64
1586   }
1587 
1588   // get oop result if there is one and reset the value in the thread
1589   if (oop_result->is_valid()) {
1590     get_vm_result(oop_result, java_thread);
1591   }
1592 }
1593 
1594 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
1595 
1596   // Calculate the value for last_Java_sp
1597   // somewhat subtle. call_VM does an intermediate call
1598   // which places a return address on the stack just under the
1599   // stack pointer as the user finsihed with it. This allows
1600   // use to retrieve last_Java_pc from last_Java_sp[-1].
1601   // On 32bit we then have to push additional args on the stack to accomplish
1602   // the actual requested call. On 64bit call_VM only can use register args
1603   // so the only extra space is the return address that call_VM created.
1604   // This hopefully explains the calculations here.
1605 
1606 #ifdef _LP64
1607   // We've pushed one address, correct last_Java_sp
1608   lea(rax, Address(rsp, wordSize));
1609 #else
1610   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
1611 #endif // LP64
1612 
1613   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
1614 
1615 }
1616 
1617 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
1618 void MacroAssembler::call_VM_leaf0(address entry_point) {
1619   MacroAssembler::call_VM_leaf_base(entry_point, 0);
1620 }
1621 
1622 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1623   call_VM_leaf_base(entry_point, number_of_arguments);
1624 }
1625 
1626 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1627   pass_arg0(this, arg_0);
1628   call_VM_leaf(entry_point, 1);
1629 }
1630 
1631 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1632 
1633   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1634   pass_arg1(this, arg_1);
1635   pass_arg0(this, arg_0);
1636   call_VM_leaf(entry_point, 2);
1637 }
1638 
1639 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1640   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1641   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1642   pass_arg2(this, arg_2);
1643   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1644   pass_arg1(this, arg_1);
1645   pass_arg0(this, arg_0);
1646   call_VM_leaf(entry_point, 3);
1647 }
1648 
1649 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1650   pass_arg0(this, arg_0);
1651   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1652 }
1653 
1654 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1655 
1656   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1657   pass_arg1(this, arg_1);
1658   pass_arg0(this, arg_0);
1659   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1660 }
1661 
1662 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1663   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1664   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1665   pass_arg2(this, arg_2);
1666   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1667   pass_arg1(this, arg_1);
1668   pass_arg0(this, arg_0);
1669   MacroAssembler::call_VM_leaf_base(entry_point, 3);
1670 }
1671 
1672 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1673   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
1674   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
1675   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
1676   pass_arg3(this, arg_3);
1677   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1678   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1679   pass_arg2(this, arg_2);
1680   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1681   pass_arg1(this, arg_1);
1682   pass_arg0(this, arg_0);
1683   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1684 }
1685 
1686 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
1687   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
1688   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
1689   verify_oop_msg(oop_result, "broken oop in call_VM_base");
1690 }
1691 
1692 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
1693   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
1694   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
1695 }
1696 
1697 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
1698 }
1699 
1700 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
1701 }
1702 
1703 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
1704   if (reachable(src1)) {
1705     cmpl(as_Address(src1), imm);
1706   } else {
1707     lea(rscratch1, src1);
1708     cmpl(Address(rscratch1, 0), imm);
1709   }
1710 }
1711 
1712 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
1713   assert(!src2.is_lval(), "use cmpptr");
1714   if (reachable(src2)) {
1715     cmpl(src1, as_Address(src2));
1716   } else {
1717     lea(rscratch1, src2);
1718     cmpl(src1, Address(rscratch1, 0));
1719   }
1720 }
1721 
1722 void MacroAssembler::cmp32(Register src1, int32_t imm) {
1723   Assembler::cmpl(src1, imm);
1724 }
1725 
1726 void MacroAssembler::cmp32(Register src1, Address src2) {
1727   Assembler::cmpl(src1, src2);
1728 }
1729 
1730 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1731   ucomisd(opr1, opr2);
1732 
1733   Label L;
1734   if (unordered_is_less) {
1735     movl(dst, -1);
1736     jcc(Assembler::parity, L);
1737     jcc(Assembler::below , L);
1738     movl(dst, 0);
1739     jcc(Assembler::equal , L);
1740     increment(dst);
1741   } else { // unordered is greater
1742     movl(dst, 1);
1743     jcc(Assembler::parity, L);
1744     jcc(Assembler::above , L);
1745     movl(dst, 0);
1746     jcc(Assembler::equal , L);
1747     decrementl(dst);
1748   }
1749   bind(L);
1750 }
1751 
1752 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1753   ucomiss(opr1, opr2);
1754 
1755   Label L;
1756   if (unordered_is_less) {
1757     movl(dst, -1);
1758     jcc(Assembler::parity, L);
1759     jcc(Assembler::below , L);
1760     movl(dst, 0);
1761     jcc(Assembler::equal , L);
1762     increment(dst);
1763   } else { // unordered is greater
1764     movl(dst, 1);
1765     jcc(Assembler::parity, L);
1766     jcc(Assembler::above , L);
1767     movl(dst, 0);
1768     jcc(Assembler::equal , L);
1769     decrementl(dst);
1770   }
1771   bind(L);
1772 }
1773 
1774 
1775 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
1776   if (reachable(src1)) {
1777     cmpb(as_Address(src1), imm);
1778   } else {
1779     lea(rscratch1, src1);
1780     cmpb(Address(rscratch1, 0), imm);
1781   }
1782 }
1783 
1784 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
1785 #ifdef _LP64
1786   if (src2.is_lval()) {
1787     movptr(rscratch1, src2);
1788     Assembler::cmpq(src1, rscratch1);
1789   } else if (reachable(src2)) {
1790     cmpq(src1, as_Address(src2));
1791   } else {
1792     lea(rscratch1, src2);
1793     Assembler::cmpq(src1, Address(rscratch1, 0));
1794   }
1795 #else
1796   if (src2.is_lval()) {
1797     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
1798   } else {
1799     cmpl(src1, as_Address(src2));
1800   }
1801 #endif // _LP64
1802 }
1803 
1804 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
1805   assert(src2.is_lval(), "not a mem-mem compare");
1806 #ifdef _LP64
1807   // moves src2's literal address
1808   movptr(rscratch1, src2);
1809   Assembler::cmpq(src1, rscratch1);
1810 #else
1811   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
1812 #endif // _LP64
1813 }
1814 
1815 void MacroAssembler::cmpoop(Register src1, Register src2) {
1816   cmpptr(src1, src2);
1817 }
1818 
1819 void MacroAssembler::cmpoop(Register src1, Address src2) {
1820   cmpptr(src1, src2);
1821 }
1822 
1823 #ifdef _LP64
1824 void MacroAssembler::cmpoop(Register src1, jobject src2) {
1825   movoop(rscratch1, src2);
1826   cmpptr(src1, rscratch1);
1827 }
1828 #endif
1829 
1830 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
1831   if (reachable(adr)) {
1832     lock();
1833     cmpxchgptr(reg, as_Address(adr));
1834   } else {
1835     lea(rscratch1, adr);
1836     lock();
1837     cmpxchgptr(reg, Address(rscratch1, 0));
1838   }
1839 }
1840 
1841 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
1842   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
1843 }
1844 
1845 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
1846   if (reachable(src)) {
1847     Assembler::comisd(dst, as_Address(src));
1848   } else {
1849     lea(rscratch1, src);
1850     Assembler::comisd(dst, Address(rscratch1, 0));
1851   }
1852 }
1853 
1854 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
1855   if (reachable(src)) {
1856     Assembler::comiss(dst, as_Address(src));
1857   } else {
1858     lea(rscratch1, src);
1859     Assembler::comiss(dst, Address(rscratch1, 0));
1860   }
1861 }
1862 
1863 
1864 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
1865   Condition negated_cond = negate_condition(cond);
1866   Label L;
1867   jcc(negated_cond, L);
1868   pushf(); // Preserve flags
1869   atomic_incl(counter_addr);
1870   popf();
1871   bind(L);
1872 }
1873 
1874 int MacroAssembler::corrected_idivl(Register reg) {
1875   // Full implementation of Java idiv and irem; checks for
1876   // special case as described in JVM spec., p.243 & p.271.
1877   // The function returns the (pc) offset of the idivl
1878   // instruction - may be needed for implicit exceptions.
1879   //
1880   //         normal case                           special case
1881   //
1882   // input : rax,: dividend                         min_int
1883   //         reg: divisor   (may not be rax,/rdx)   -1
1884   //
1885   // output: rax,: quotient  (= rax, idiv reg)       min_int
1886   //         rdx: remainder (= rax, irem reg)       0
1887   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
1888   const int min_int = 0x80000000;
1889   Label normal_case, special_case;
1890 
1891   // check for special case
1892   cmpl(rax, min_int);
1893   jcc(Assembler::notEqual, normal_case);
1894   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
1895   cmpl(reg, -1);
1896   jcc(Assembler::equal, special_case);
1897 
1898   // handle normal case
1899   bind(normal_case);
1900   cdql();
1901   int idivl_offset = offset();
1902   idivl(reg);
1903 
1904   // normal and special case exit
1905   bind(special_case);
1906 
1907   return idivl_offset;
1908 }
1909 
1910 
1911 
1912 void MacroAssembler::decrementl(Register reg, int value) {
1913   if (value == min_jint) {subl(reg, value) ; return; }
1914   if (value <  0) { incrementl(reg, -value); return; }
1915   if (value == 0) {                        ; return; }
1916   if (value == 1 && UseIncDec) { decl(reg) ; return; }
1917   /* else */      { subl(reg, value)       ; return; }
1918 }
1919 
1920 void MacroAssembler::decrementl(Address dst, int value) {
1921   if (value == min_jint) {subl(dst, value) ; return; }
1922   if (value <  0) { incrementl(dst, -value); return; }
1923   if (value == 0) {                        ; return; }
1924   if (value == 1 && UseIncDec) { decl(dst) ; return; }
1925   /* else */      { subl(dst, value)       ; return; }
1926 }
1927 
1928 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
1929   assert (shift_value > 0, "illegal shift value");
1930   Label _is_positive;
1931   testl (reg, reg);
1932   jcc (Assembler::positive, _is_positive);
1933   int offset = (1 << shift_value) - 1 ;
1934 
1935   if (offset == 1) {
1936     incrementl(reg);
1937   } else {
1938     addl(reg, offset);
1939   }
1940 
1941   bind (_is_positive);
1942   sarl(reg, shift_value);
1943 }
1944 
1945 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
1946   if (reachable(src)) {
1947     Assembler::divsd(dst, as_Address(src));
1948   } else {
1949     lea(rscratch1, src);
1950     Assembler::divsd(dst, Address(rscratch1, 0));
1951   }
1952 }
1953 
1954 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
1955   if (reachable(src)) {
1956     Assembler::divss(dst, as_Address(src));
1957   } else {
1958     lea(rscratch1, src);
1959     Assembler::divss(dst, Address(rscratch1, 0));
1960   }
1961 }
1962 
1963 void MacroAssembler::enter() {
1964   push(rbp);
1965   mov(rbp, rsp);
1966 }
1967 
1968 // A 5 byte nop that is safe for patching (see patch_verified_entry)
1969 void MacroAssembler::fat_nop() {
1970   if (UseAddressNop) {
1971     addr_nop_5();
1972   } else {
1973     emit_int8(0x26); // es:
1974     emit_int8(0x2e); // cs:
1975     emit_int8(0x64); // fs:
1976     emit_int8(0x65); // gs:
1977     emit_int8((unsigned char)0x90);
1978   }
1979 }
1980 
1981 #ifndef _LP64
1982 void MacroAssembler::fcmp(Register tmp) {
1983   fcmp(tmp, 1, true, true);
1984 }
1985 
1986 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
1987   assert(!pop_right || pop_left, "usage error");
1988   if (VM_Version::supports_cmov()) {
1989     assert(tmp == noreg, "unneeded temp");
1990     if (pop_left) {
1991       fucomip(index);
1992     } else {
1993       fucomi(index);
1994     }
1995     if (pop_right) {
1996       fpop();
1997     }
1998   } else {
1999     assert(tmp != noreg, "need temp");
2000     if (pop_left) {
2001       if (pop_right) {
2002         fcompp();
2003       } else {
2004         fcomp(index);
2005       }
2006     } else {
2007       fcom(index);
2008     }
2009     // convert FPU condition into eflags condition via rax,
2010     save_rax(tmp);
2011     fwait(); fnstsw_ax();
2012     sahf();
2013     restore_rax(tmp);
2014   }
2015   // condition codes set as follows:
2016   //
2017   // CF (corresponds to C0) if x < y
2018   // PF (corresponds to C2) if unordered
2019   // ZF (corresponds to C3) if x = y
2020 }
2021 
2022 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
2023   fcmp2int(dst, unordered_is_less, 1, true, true);
2024 }
2025 
2026 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
2027   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
2028   Label L;
2029   if (unordered_is_less) {
2030     movl(dst, -1);
2031     jcc(Assembler::parity, L);
2032     jcc(Assembler::below , L);
2033     movl(dst, 0);
2034     jcc(Assembler::equal , L);
2035     increment(dst);
2036   } else { // unordered is greater
2037     movl(dst, 1);
2038     jcc(Assembler::parity, L);
2039     jcc(Assembler::above , L);
2040     movl(dst, 0);
2041     jcc(Assembler::equal , L);
2042     decrementl(dst);
2043   }
2044   bind(L);
2045 }
2046 
2047 void MacroAssembler::fld_d(AddressLiteral src) {
2048   fld_d(as_Address(src));
2049 }
2050 
2051 void MacroAssembler::fld_s(AddressLiteral src) {
2052   fld_s(as_Address(src));
2053 }
2054 
2055 void MacroAssembler::fldcw(AddressLiteral src) {
2056   Assembler::fldcw(as_Address(src));
2057 }
2058 
2059 void MacroAssembler::fpop() {
2060   ffree();
2061   fincstp();
2062 }
2063 
2064 void MacroAssembler::fremr(Register tmp) {
2065   save_rax(tmp);
2066   { Label L;
2067     bind(L);
2068     fprem();
2069     fwait(); fnstsw_ax();
2070     sahf();
2071     jcc(Assembler::parity, L);
2072   }
2073   restore_rax(tmp);
2074   // Result is in ST0.
2075   // Note: fxch & fpop to get rid of ST1
2076   // (otherwise FPU stack could overflow eventually)
2077   fxch(1);
2078   fpop();
2079 }
2080 
2081 void MacroAssembler::empty_FPU_stack() {
2082   if (VM_Version::supports_mmx()) {
2083     emms();
2084   } else {
2085     for (int i = 8; i-- > 0; ) ffree(i);
2086   }
2087 }
2088 #endif // !LP64
2089 
2090 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
2091   if (reachable(src)) {
2092     Assembler::mulpd(dst, as_Address(src));
2093   } else {
2094     lea(rscratch1, src);
2095     Assembler::mulpd(dst, Address(rscratch1, 0));
2096   }
2097 }
2098 
2099 void MacroAssembler::load_float(Address src) {
2100 #ifdef _LP64
2101   movflt(xmm0, src);
2102 #else
2103   if (UseSSE >= 1) {
2104     movflt(xmm0, src);
2105   } else {
2106     fld_s(src);
2107   }
2108 #endif // LP64
2109 }
2110 
2111 void MacroAssembler::store_float(Address dst) {
2112 #ifdef _LP64
2113   movflt(dst, xmm0);
2114 #else
2115   if (UseSSE >= 1) {
2116     movflt(dst, xmm0);
2117   } else {
2118     fstp_s(dst);
2119   }
2120 #endif // LP64
2121 }
2122 
2123 void MacroAssembler::load_double(Address src) {
2124 #ifdef _LP64
2125   movdbl(xmm0, src);
2126 #else
2127   if (UseSSE >= 2) {
2128     movdbl(xmm0, src);
2129   } else {
2130     fld_d(src);
2131   }
2132 #endif // LP64
2133 }
2134 
2135 void MacroAssembler::store_double(Address dst) {
2136 #ifdef _LP64
2137   movdbl(dst, xmm0);
2138 #else
2139   if (UseSSE >= 2) {
2140     movdbl(dst, xmm0);
2141   } else {
2142     fstp_d(dst);
2143   }
2144 #endif // LP64
2145 }
2146 
2147 // dst = c = a * b + c
2148 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2149   Assembler::vfmadd231sd(c, a, b);
2150   if (dst != c) {
2151     movdbl(dst, c);
2152   }
2153 }
2154 
2155 // dst = c = a * b + c
2156 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2157   Assembler::vfmadd231ss(c, a, b);
2158   if (dst != c) {
2159     movflt(dst, c);
2160   }
2161 }
2162 
2163 // dst = c = a * b + c
2164 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2165   Assembler::vfmadd231pd(c, a, b, vector_len);
2166   if (dst != c) {
2167     vmovdqu(dst, c);
2168   }
2169 }
2170 
2171 // dst = c = a * b + c
2172 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2173   Assembler::vfmadd231ps(c, a, b, vector_len);
2174   if (dst != c) {
2175     vmovdqu(dst, c);
2176   }
2177 }
2178 
2179 // dst = c = a * b + c
2180 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
2181   Assembler::vfmadd231pd(c, a, b, vector_len);
2182   if (dst != c) {
2183     vmovdqu(dst, c);
2184   }
2185 }
2186 
2187 // dst = c = a * b + c
2188 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
2189   Assembler::vfmadd231ps(c, a, b, vector_len);
2190   if (dst != c) {
2191     vmovdqu(dst, c);
2192   }
2193 }
2194 
2195 void MacroAssembler::incrementl(AddressLiteral dst) {
2196   if (reachable(dst)) {
2197     incrementl(as_Address(dst));
2198   } else {
2199     lea(rscratch1, dst);
2200     incrementl(Address(rscratch1, 0));
2201   }
2202 }
2203 
2204 void MacroAssembler::incrementl(ArrayAddress dst) {
2205   incrementl(as_Address(dst));
2206 }
2207 
2208 void MacroAssembler::incrementl(Register reg, int value) {
2209   if (value == min_jint) {addl(reg, value) ; return; }
2210   if (value <  0) { decrementl(reg, -value); return; }
2211   if (value == 0) {                        ; return; }
2212   if (value == 1 && UseIncDec) { incl(reg) ; return; }
2213   /* else */      { addl(reg, value)       ; return; }
2214 }
2215 
2216 void MacroAssembler::incrementl(Address dst, int value) {
2217   if (value == min_jint) {addl(dst, value) ; return; }
2218   if (value <  0) { decrementl(dst, -value); return; }
2219   if (value == 0) {                        ; return; }
2220   if (value == 1 && UseIncDec) { incl(dst) ; return; }
2221   /* else */      { addl(dst, value)       ; return; }
2222 }
2223 
2224 void MacroAssembler::jump(AddressLiteral dst) {
2225   if (reachable(dst)) {
2226     jmp_literal(dst.target(), dst.rspec());
2227   } else {
2228     lea(rscratch1, dst);
2229     jmp(rscratch1);
2230   }
2231 }
2232 
2233 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
2234   if (reachable(dst)) {
2235     InstructionMark im(this);
2236     relocate(dst.reloc());
2237     const int short_size = 2;
2238     const int long_size = 6;
2239     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
2240     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
2241       // 0111 tttn #8-bit disp
2242       emit_int8(0x70 | cc);
2243       emit_int8((offs - short_size) & 0xFF);
2244     } else {
2245       // 0000 1111 1000 tttn #32-bit disp
2246       emit_int8(0x0F);
2247       emit_int8((unsigned char)(0x80 | cc));
2248       emit_int32(offs - long_size);
2249     }
2250   } else {
2251 #ifdef ASSERT
2252     warning("reversing conditional branch");
2253 #endif /* ASSERT */
2254     Label skip;
2255     jccb(reverse[cc], skip);
2256     lea(rscratch1, dst);
2257     Assembler::jmp(rscratch1);
2258     bind(skip);
2259   }
2260 }
2261 
2262 void MacroAssembler::fld_x(AddressLiteral src) {
2263   Assembler::fld_x(as_Address(src));
2264 }
2265 
2266 void MacroAssembler::ldmxcsr(AddressLiteral src) {
2267   if (reachable(src)) {
2268     Assembler::ldmxcsr(as_Address(src));
2269   } else {
2270     lea(rscratch1, src);
2271     Assembler::ldmxcsr(Address(rscratch1, 0));
2272   }
2273 }
2274 
2275 int MacroAssembler::load_signed_byte(Register dst, Address src) {
2276   int off;
2277   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
2278     off = offset();
2279     movsbl(dst, src); // movsxb
2280   } else {
2281     off = load_unsigned_byte(dst, src);
2282     shll(dst, 24);
2283     sarl(dst, 24);
2284   }
2285   return off;
2286 }
2287 
2288 // Note: load_signed_short used to be called load_signed_word.
2289 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
2290 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
2291 // The term "word" in HotSpot means a 32- or 64-bit machine word.
2292 int MacroAssembler::load_signed_short(Register dst, Address src) {
2293   int off;
2294   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
2295     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
2296     // version but this is what 64bit has always done. This seems to imply
2297     // that users are only using 32bits worth.
2298     off = offset();
2299     movswl(dst, src); // movsxw
2300   } else {
2301     off = load_unsigned_short(dst, src);
2302     shll(dst, 16);
2303     sarl(dst, 16);
2304   }
2305   return off;
2306 }
2307 
2308 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
2309   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
2310   // and "3.9 Partial Register Penalties", p. 22).
2311   int off;
2312   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
2313     off = offset();
2314     movzbl(dst, src); // movzxb
2315   } else {
2316     xorl(dst, dst);
2317     off = offset();
2318     movb(dst, src);
2319   }
2320   return off;
2321 }
2322 
2323 // Note: load_unsigned_short used to be called load_unsigned_word.
2324 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
2325   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
2326   // and "3.9 Partial Register Penalties", p. 22).
2327   int off;
2328   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
2329     off = offset();
2330     movzwl(dst, src); // movzxw
2331   } else {
2332     xorl(dst, dst);
2333     off = offset();
2334     movw(dst, src);
2335   }
2336   return off;
2337 }
2338 
2339 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
2340   switch (size_in_bytes) {
2341 #ifndef _LP64
2342   case  8:
2343     assert(dst2 != noreg, "second dest register required");
2344     movl(dst,  src);
2345     movl(dst2, src.plus_disp(BytesPerInt));
2346     break;
2347 #else
2348   case  8:  movq(dst, src); break;
2349 #endif
2350   case  4:  movl(dst, src); break;
2351   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
2352   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
2353   default:  ShouldNotReachHere();
2354   }
2355 }
2356 
2357 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
2358   switch (size_in_bytes) {
2359 #ifndef _LP64
2360   case  8:
2361     assert(src2 != noreg, "second source register required");
2362     movl(dst,                        src);
2363     movl(dst.plus_disp(BytesPerInt), src2);
2364     break;
2365 #else
2366   case  8:  movq(dst, src); break;
2367 #endif
2368   case  4:  movl(dst, src); break;
2369   case  2:  movw(dst, src); break;
2370   case  1:  movb(dst, src); break;
2371   default:  ShouldNotReachHere();
2372   }
2373 }
2374 
2375 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
2376   if (reachable(dst)) {
2377     movl(as_Address(dst), src);
2378   } else {
2379     lea(rscratch1, dst);
2380     movl(Address(rscratch1, 0), src);
2381   }
2382 }
2383 
2384 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
2385   if (reachable(src)) {
2386     movl(dst, as_Address(src));
2387   } else {
2388     lea(rscratch1, src);
2389     movl(dst, Address(rscratch1, 0));
2390   }
2391 }
2392 
2393 // C++ bool manipulation
2394 
2395 void MacroAssembler::movbool(Register dst, Address src) {
2396   if(sizeof(bool) == 1)
2397     movb(dst, src);
2398   else if(sizeof(bool) == 2)
2399     movw(dst, src);
2400   else if(sizeof(bool) == 4)
2401     movl(dst, src);
2402   else
2403     // unsupported
2404     ShouldNotReachHere();
2405 }
2406 
2407 void MacroAssembler::movbool(Address dst, bool boolconst) {
2408   if(sizeof(bool) == 1)
2409     movb(dst, (int) boolconst);
2410   else if(sizeof(bool) == 2)
2411     movw(dst, (int) boolconst);
2412   else if(sizeof(bool) == 4)
2413     movl(dst, (int) boolconst);
2414   else
2415     // unsupported
2416     ShouldNotReachHere();
2417 }
2418 
2419 void MacroAssembler::movbool(Address dst, Register src) {
2420   if(sizeof(bool) == 1)
2421     movb(dst, src);
2422   else if(sizeof(bool) == 2)
2423     movw(dst, src);
2424   else if(sizeof(bool) == 4)
2425     movl(dst, src);
2426   else
2427     // unsupported
2428     ShouldNotReachHere();
2429 }
2430 
2431 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
2432   movb(as_Address(dst), src);
2433 }
2434 
2435 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
2436   if (reachable(src)) {
2437     movdl(dst, as_Address(src));
2438   } else {
2439     lea(rscratch1, src);
2440     movdl(dst, Address(rscratch1, 0));
2441   }
2442 }
2443 
2444 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
2445   if (reachable(src)) {
2446     movq(dst, as_Address(src));
2447   } else {
2448     lea(rscratch1, src);
2449     movq(dst, Address(rscratch1, 0));
2450   }
2451 }
2452 
2453 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
2454   if (reachable(src)) {
2455     if (UseXmmLoadAndClearUpper) {
2456       movsd (dst, as_Address(src));
2457     } else {
2458       movlpd(dst, as_Address(src));
2459     }
2460   } else {
2461     lea(rscratch1, src);
2462     if (UseXmmLoadAndClearUpper) {
2463       movsd (dst, Address(rscratch1, 0));
2464     } else {
2465       movlpd(dst, Address(rscratch1, 0));
2466     }
2467   }
2468 }
2469 
2470 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
2471   if (reachable(src)) {
2472     movss(dst, as_Address(src));
2473   } else {
2474     lea(rscratch1, src);
2475     movss(dst, Address(rscratch1, 0));
2476   }
2477 }
2478 
2479 void MacroAssembler::movptr(Register dst, Register src) {
2480   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2481 }
2482 
2483 void MacroAssembler::movptr(Register dst, Address src) {
2484   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2485 }
2486 
2487 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
2488 void MacroAssembler::movptr(Register dst, intptr_t src) {
2489   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
2490 }
2491 
2492 void MacroAssembler::movptr(Address dst, Register src) {
2493   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2494 }
2495 
2496 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
2497     assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2498     Assembler::movdqu(dst, src);
2499 }
2500 
2501 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
2502     assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2503     Assembler::movdqu(dst, src);
2504 }
2505 
2506 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
2507     assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2508     Assembler::movdqu(dst, src);
2509 }
2510 
2511 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
2512   if (reachable(src)) {
2513     movdqu(dst, as_Address(src));
2514   } else {
2515     lea(scratchReg, src);
2516     movdqu(dst, Address(scratchReg, 0));
2517   }
2518 }
2519 
2520 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
2521     assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2522     Assembler::vmovdqu(dst, src);
2523 }
2524 
2525 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
2526     assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2527     Assembler::vmovdqu(dst, src);
2528 }
2529 
2530 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2531     assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2532     Assembler::vmovdqu(dst, src);
2533 }
2534 
2535 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
2536   if (reachable(src)) {
2537     vmovdqu(dst, as_Address(src));
2538   }
2539   else {
2540     lea(scratch_reg, src);
2541     vmovdqu(dst, Address(scratch_reg, 0));
2542   }
2543 }
2544 
2545 void MacroAssembler::kmov(KRegister dst, Address src) {
2546   if (VM_Version::supports_avx512bw()) {
2547     kmovql(dst, src);
2548   } else {
2549     assert(VM_Version::supports_evex(), "");
2550     kmovwl(dst, src);
2551   }
2552 }
2553 
2554 void MacroAssembler::kmov(Address dst, KRegister src) {
2555   if (VM_Version::supports_avx512bw()) {
2556     kmovql(dst, src);
2557   } else {
2558     assert(VM_Version::supports_evex(), "");
2559     kmovwl(dst, src);
2560   }
2561 }
2562 
2563 void MacroAssembler::kmov(KRegister dst, KRegister src) {
2564   if (VM_Version::supports_avx512bw()) {
2565     kmovql(dst, src);
2566   } else {
2567     assert(VM_Version::supports_evex(), "");
2568     kmovwl(dst, src);
2569   }
2570 }
2571 
2572 void MacroAssembler::kmov(Register dst, KRegister src) {
2573   if (VM_Version::supports_avx512bw()) {
2574     kmovql(dst, src);
2575   } else {
2576     assert(VM_Version::supports_evex(), "");
2577     kmovwl(dst, src);
2578   }
2579 }
2580 
2581 void MacroAssembler::kmov(KRegister dst, Register src) {
2582   if (VM_Version::supports_avx512bw()) {
2583     kmovql(dst, src);
2584   } else {
2585     assert(VM_Version::supports_evex(), "");
2586     kmovwl(dst, src);
2587   }
2588 }
2589 
2590 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register scratch_reg) {
2591   if (reachable(src)) {
2592     kmovql(dst, as_Address(src));
2593   } else {
2594     lea(scratch_reg, src);
2595     kmovql(dst, Address(scratch_reg, 0));
2596   }
2597 }
2598 
2599 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register scratch_reg) {
2600   if (reachable(src)) {
2601     kmovwl(dst, as_Address(src));
2602   } else {
2603     lea(scratch_reg, src);
2604     kmovwl(dst, Address(scratch_reg, 0));
2605   }
2606 }
2607 
2608 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2609                                int vector_len, Register scratch_reg) {
2610   if (reachable(src)) {
2611     if (mask == k0) {
2612       Assembler::evmovdqub(dst, as_Address(src), merge, vector_len);
2613     } else {
2614       Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
2615     }
2616   } else {
2617     lea(scratch_reg, src);
2618     if (mask == k0) {
2619       Assembler::evmovdqub(dst, Address(scratch_reg, 0), merge, vector_len);
2620     } else {
2621       Assembler::evmovdqub(dst, mask, Address(scratch_reg, 0), merge, vector_len);
2622     }
2623   }
2624 }
2625 
2626 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2627                                int vector_len, Register scratch_reg) {
2628   if (reachable(src)) {
2629     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
2630   } else {
2631     lea(scratch_reg, src);
2632     Assembler::evmovdquw(dst, mask, Address(scratch_reg, 0), merge, vector_len);
2633   }
2634 }
2635 
2636 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2637                                int vector_len, Register scratch_reg) {
2638   if (reachable(src)) {
2639     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
2640   } else {
2641     lea(scratch_reg, src);
2642     Assembler::evmovdqul(dst, mask, Address(scratch_reg, 0), merge, vector_len);
2643   }
2644 }
2645 
2646 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2647                                int vector_len, Register scratch_reg) {
2648   if (reachable(src)) {
2649     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
2650   } else {
2651     lea(scratch_reg, src);
2652     Assembler::evmovdquq(dst, mask, Address(scratch_reg, 0), merge, vector_len);
2653   }
2654 }
2655 
2656 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2657   if (reachable(src)) {
2658     Assembler::evmovdquq(dst, as_Address(src), vector_len);
2659   } else {
2660     lea(rscratch, src);
2661     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
2662   }
2663 }
2664 
2665 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
2666   if (reachable(src)) {
2667     Assembler::movdqa(dst, as_Address(src));
2668   } else {
2669     lea(rscratch1, src);
2670     Assembler::movdqa(dst, Address(rscratch1, 0));
2671   }
2672 }
2673 
2674 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
2675   if (reachable(src)) {
2676     Assembler::movsd(dst, as_Address(src));
2677   } else {
2678     lea(rscratch1, src);
2679     Assembler::movsd(dst, Address(rscratch1, 0));
2680   }
2681 }
2682 
2683 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
2684   if (reachable(src)) {
2685     Assembler::movss(dst, as_Address(src));
2686   } else {
2687     lea(rscratch1, src);
2688     Assembler::movss(dst, Address(rscratch1, 0));
2689   }
2690 }
2691 
2692 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
2693   if (reachable(src)) {
2694     Assembler::mulsd(dst, as_Address(src));
2695   } else {
2696     lea(rscratch1, src);
2697     Assembler::mulsd(dst, Address(rscratch1, 0));
2698   }
2699 }
2700 
2701 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
2702   if (reachable(src)) {
2703     Assembler::mulss(dst, as_Address(src));
2704   } else {
2705     lea(rscratch1, src);
2706     Assembler::mulss(dst, Address(rscratch1, 0));
2707   }
2708 }
2709 
2710 void MacroAssembler::null_check(Register reg, int offset) {
2711   if (needs_explicit_null_check(offset)) {
2712     // provoke OS NULL exception if reg = NULL by
2713     // accessing M[reg] w/o changing any (non-CC) registers
2714     // NOTE: cmpl is plenty here to provoke a segv
2715     cmpptr(rax, Address(reg, 0));
2716     // Note: should probably use testl(rax, Address(reg, 0));
2717     //       may be shorter code (however, this version of
2718     //       testl needs to be implemented first)
2719   } else {
2720     // nothing to do, (later) access of M[reg + offset]
2721     // will provoke OS NULL exception if reg = NULL
2722   }
2723 }
2724 
2725 void MacroAssembler::os_breakpoint() {
2726   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
2727   // (e.g., MSVC can't call ps() otherwise)
2728   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
2729 }
2730 
2731 void MacroAssembler::unimplemented(const char* what) {
2732   const char* buf = NULL;
2733   {
2734     ResourceMark rm;
2735     stringStream ss;
2736     ss.print("unimplemented: %s", what);
2737     buf = code_string(ss.as_string());
2738   }
2739   stop(buf);
2740 }
2741 
2742 #ifdef _LP64
2743 #define XSTATE_BV 0x200
2744 #endif
2745 
2746 void MacroAssembler::pop_CPU_state() {
2747   pop_FPU_state();
2748   pop_IU_state();
2749 }
2750 
2751 void MacroAssembler::pop_FPU_state() {
2752 #ifndef _LP64
2753   frstor(Address(rsp, 0));
2754 #else
2755   fxrstor(Address(rsp, 0));
2756 #endif
2757   addptr(rsp, FPUStateSizeInWords * wordSize);
2758 }
2759 
2760 void MacroAssembler::pop_IU_state() {
2761   popa();
2762   LP64_ONLY(addq(rsp, 8));
2763   popf();
2764 }
2765 
2766 // Save Integer and Float state
2767 // Warning: Stack must be 16 byte aligned (64bit)
2768 void MacroAssembler::push_CPU_state() {
2769   push_IU_state();
2770   push_FPU_state();
2771 }
2772 
2773 void MacroAssembler::push_FPU_state() {
2774   subptr(rsp, FPUStateSizeInWords * wordSize);
2775 #ifndef _LP64
2776   fnsave(Address(rsp, 0));
2777   fwait();
2778 #else
2779   fxsave(Address(rsp, 0));
2780 #endif // LP64
2781 }
2782 
2783 void MacroAssembler::push_IU_state() {
2784   // Push flags first because pusha kills them
2785   pushf();
2786   // Make sure rsp stays 16-byte aligned
2787   LP64_ONLY(subq(rsp, 8));
2788   pusha();
2789 }
2790 
2791 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
2792   if (!java_thread->is_valid()) {
2793     java_thread = rdi;
2794     get_thread(java_thread);
2795   }
2796   // we must set sp to zero to clear frame
2797   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
2798   // must clear fp, so that compiled frames are not confused; it is
2799   // possible that we need it only for debugging
2800   if (clear_fp) {
2801     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
2802   }
2803   // Always clear the pc because it could have been set by make_walkable()
2804   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
2805   vzeroupper();
2806 }
2807 
2808 void MacroAssembler::restore_rax(Register tmp) {
2809   if (tmp == noreg) pop(rax);
2810   else if (tmp != rax) mov(rax, tmp);
2811 }
2812 
2813 void MacroAssembler::round_to(Register reg, int modulus) {
2814   addptr(reg, modulus - 1);
2815   andptr(reg, -modulus);
2816 }
2817 
2818 void MacroAssembler::save_rax(Register tmp) {
2819   if (tmp == noreg) push(rax);
2820   else if (tmp != rax) mov(tmp, rax);
2821 }
2822 
2823 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
2824   if (at_return) {
2825     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
2826     // we may safely use rsp instead to perform the stack watermark check.
2827     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
2828     jcc(Assembler::above, slow_path);
2829     return;
2830   }
2831   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2832   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
2833 }
2834 
2835 // Calls to C land
2836 //
2837 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
2838 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
2839 // has to be reset to 0. This is required to allow proper stack traversal.
2840 void MacroAssembler::set_last_Java_frame(Register java_thread,
2841                                          Register last_java_sp,
2842                                          Register last_java_fp,
2843                                          address  last_java_pc) {
2844   vzeroupper();
2845   // determine java_thread register
2846   if (!java_thread->is_valid()) {
2847     java_thread = rdi;
2848     get_thread(java_thread);
2849   }
2850   // determine last_java_sp register
2851   if (!last_java_sp->is_valid()) {
2852     last_java_sp = rsp;
2853   }
2854 
2855   // last_java_fp is optional
2856 
2857   if (last_java_fp->is_valid()) {
2858     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
2859   }
2860 
2861   // last_java_pc is optional
2862 
2863   if (last_java_pc != NULL) {
2864     lea(Address(java_thread,
2865                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
2866         InternalAddress(last_java_pc));
2867 
2868   }
2869   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
2870 }
2871 
2872 void MacroAssembler::shlptr(Register dst, int imm8) {
2873   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
2874 }
2875 
2876 void MacroAssembler::shrptr(Register dst, int imm8) {
2877   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
2878 }
2879 
2880 void MacroAssembler::sign_extend_byte(Register reg) {
2881   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
2882     movsbl(reg, reg); // movsxb
2883   } else {
2884     shll(reg, 24);
2885     sarl(reg, 24);
2886   }
2887 }
2888 
2889 void MacroAssembler::sign_extend_short(Register reg) {
2890   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
2891     movswl(reg, reg); // movsxw
2892   } else {
2893     shll(reg, 16);
2894     sarl(reg, 16);
2895   }
2896 }
2897 
2898 void MacroAssembler::testl(Register dst, AddressLiteral src) {
2899   assert(reachable(src), "Address should be reachable");
2900   testl(dst, as_Address(src));
2901 }
2902 
2903 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
2904   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2905   Assembler::pcmpeqb(dst, src);
2906 }
2907 
2908 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
2909   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2910   Assembler::pcmpeqw(dst, src);
2911 }
2912 
2913 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2914   assert((dst->encoding() < 16),"XMM register should be 0-15");
2915   Assembler::pcmpestri(dst, src, imm8);
2916 }
2917 
2918 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2919   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2920   Assembler::pcmpestri(dst, src, imm8);
2921 }
2922 
2923 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2924   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2925   Assembler::pmovzxbw(dst, src);
2926 }
2927 
2928 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
2929   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2930   Assembler::pmovzxbw(dst, src);
2931 }
2932 
2933 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
2934   assert((src->encoding() < 16),"XMM register should be 0-15");
2935   Assembler::pmovmskb(dst, src);
2936 }
2937 
2938 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
2939   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2940   Assembler::ptest(dst, src);
2941 }
2942 
2943 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
2944   if (reachable(src)) {
2945     Assembler::sqrtsd(dst, as_Address(src));
2946   } else {
2947     lea(rscratch1, src);
2948     Assembler::sqrtsd(dst, Address(rscratch1, 0));
2949   }
2950 }
2951 
2952 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
2953   if (reachable(src)) {
2954     Assembler::sqrtss(dst, as_Address(src));
2955   } else {
2956     lea(rscratch1, src);
2957     Assembler::sqrtss(dst, Address(rscratch1, 0));
2958   }
2959 }
2960 
2961 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
2962   if (reachable(src)) {
2963     Assembler::subsd(dst, as_Address(src));
2964   } else {
2965     lea(rscratch1, src);
2966     Assembler::subsd(dst, Address(rscratch1, 0));
2967   }
2968 }
2969 
2970 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register scratch_reg) {
2971   if (reachable(src)) {
2972     Assembler::roundsd(dst, as_Address(src), rmode);
2973   } else {
2974     lea(scratch_reg, src);
2975     Assembler::roundsd(dst, Address(scratch_reg, 0), rmode);
2976   }
2977 }
2978 
2979 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
2980   if (reachable(src)) {
2981     Assembler::subss(dst, as_Address(src));
2982   } else {
2983     lea(rscratch1, src);
2984     Assembler::subss(dst, Address(rscratch1, 0));
2985   }
2986 }
2987 
2988 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
2989   if (reachable(src)) {
2990     Assembler::ucomisd(dst, as_Address(src));
2991   } else {
2992     lea(rscratch1, src);
2993     Assembler::ucomisd(dst, Address(rscratch1, 0));
2994   }
2995 }
2996 
2997 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
2998   if (reachable(src)) {
2999     Assembler::ucomiss(dst, as_Address(src));
3000   } else {
3001     lea(rscratch1, src);
3002     Assembler::ucomiss(dst, Address(rscratch1, 0));
3003   }
3004 }
3005 
3006 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
3007   // Used in sign-bit flipping with aligned address.
3008   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3009   if (reachable(src)) {
3010     Assembler::xorpd(dst, as_Address(src));
3011   } else {
3012     lea(scratch_reg, src);
3013     Assembler::xorpd(dst, Address(scratch_reg, 0));
3014   }
3015 }
3016 
3017 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
3018   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3019     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3020   }
3021   else {
3022     Assembler::xorpd(dst, src);
3023   }
3024 }
3025 
3026 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
3027   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3028     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3029   } else {
3030     Assembler::xorps(dst, src);
3031   }
3032 }
3033 
3034 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
3035   // Used in sign-bit flipping with aligned address.
3036   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3037   if (reachable(src)) {
3038     Assembler::xorps(dst, as_Address(src));
3039   } else {
3040     lea(scratch_reg, src);
3041     Assembler::xorps(dst, Address(scratch_reg, 0));
3042   }
3043 }
3044 
3045 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3046   // Used in sign-bit flipping with aligned address.
3047   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3048   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3049   if (reachable(src)) {
3050     Assembler::pshufb(dst, as_Address(src));
3051   } else {
3052     lea(rscratch1, src);
3053     Assembler::pshufb(dst, Address(rscratch1, 0));
3054   }
3055 }
3056 
3057 // AVX 3-operands instructions
3058 
3059 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3060   if (reachable(src)) {
3061     vaddsd(dst, nds, as_Address(src));
3062   } else {
3063     lea(rscratch1, src);
3064     vaddsd(dst, nds, Address(rscratch1, 0));
3065   }
3066 }
3067 
3068 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3069   if (reachable(src)) {
3070     vaddss(dst, nds, as_Address(src));
3071   } else {
3072     lea(rscratch1, src);
3073     vaddss(dst, nds, Address(rscratch1, 0));
3074   }
3075 }
3076 
3077 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3078   assert(UseAVX > 0, "requires some form of AVX");
3079   if (reachable(src)) {
3080     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
3081   } else {
3082     lea(rscratch, src);
3083     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
3084   }
3085 }
3086 
3087 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3088   assert(UseAVX > 0, "requires some form of AVX");
3089   if (reachable(src)) {
3090     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
3091   } else {
3092     lea(rscratch, src);
3093     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
3094   }
3095 }
3096 
3097 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
3098   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3099   vandps(dst, nds, negate_field, vector_len);
3100 }
3101 
3102 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
3103   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3104   vandpd(dst, nds, negate_field, vector_len);
3105 }
3106 
3107 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3108   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3109   Assembler::vpaddb(dst, nds, src, vector_len);
3110 }
3111 
3112 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3113   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3114   Assembler::vpaddb(dst, nds, src, vector_len);
3115 }
3116 
3117 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3118   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3119   Assembler::vpaddw(dst, nds, src, vector_len);
3120 }
3121 
3122 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3123   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3124   Assembler::vpaddw(dst, nds, src, vector_len);
3125 }
3126 
3127 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3128   if (reachable(src)) {
3129     Assembler::vpand(dst, nds, as_Address(src), vector_len);
3130   } else {
3131     lea(scratch_reg, src);
3132     Assembler::vpand(dst, nds, Address(scratch_reg, 0), vector_len);
3133   }
3134 }
3135 
3136 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
3137   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3138   Assembler::vpbroadcastw(dst, src, vector_len);
3139 }
3140 
3141 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3142   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3143   Assembler::vpcmpeqb(dst, nds, src, vector_len);
3144 }
3145 
3146 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3147   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3148   Assembler::vpcmpeqw(dst, nds, src, vector_len);
3149 }
3150 
3151 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds,
3152                                AddressLiteral src, int vector_len, Register scratch_reg) {
3153   if (reachable(src)) {
3154     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
3155   } else {
3156     lea(scratch_reg, src);
3157     Assembler::evpcmpeqd(kdst, mask, nds, Address(scratch_reg, 0), vector_len);
3158   }
3159 }
3160 
3161 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3162                              int comparison, bool is_signed, int vector_len, Register scratch_reg) {
3163   if (reachable(src)) {
3164     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3165   } else {
3166     lea(scratch_reg, src);
3167     Assembler::evpcmpd(kdst, mask, nds, Address(scratch_reg, 0), comparison, is_signed, vector_len);
3168   }
3169 }
3170 
3171 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3172                              int comparison, bool is_signed, int vector_len, Register scratch_reg) {
3173   if (reachable(src)) {
3174     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3175   } else {
3176     lea(scratch_reg, src);
3177     Assembler::evpcmpq(kdst, mask, nds, Address(scratch_reg, 0), comparison, is_signed, vector_len);
3178   }
3179 }
3180 
3181 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3182                              int comparison, bool is_signed, int vector_len, Register scratch_reg) {
3183   if (reachable(src)) {
3184     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3185   } else {
3186     lea(scratch_reg, src);
3187     Assembler::evpcmpb(kdst, mask, nds, Address(scratch_reg, 0), comparison, is_signed, vector_len);
3188   }
3189 }
3190 
3191 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3192                              int comparison, bool is_signed, int vector_len, Register scratch_reg) {
3193   if (reachable(src)) {
3194     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3195   } else {
3196     lea(scratch_reg, src);
3197     Assembler::evpcmpw(kdst, mask, nds, Address(scratch_reg, 0), comparison, is_signed, vector_len);
3198   }
3199 }
3200 
3201 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
3202   if (width == Assembler::Q) {
3203     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
3204   } else {
3205     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
3206   }
3207 }
3208 
3209 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, ComparisonPredicate cond, Width width, int vector_len, Register scratch_reg) {
3210   int eq_cond_enc = 0x29;
3211   int gt_cond_enc = 0x37;
3212   if (width != Assembler::Q) {
3213     eq_cond_enc = 0x74 + width;
3214     gt_cond_enc = 0x64 + width;
3215   }
3216   switch (cond) {
3217   case eq:
3218     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3219     break;
3220   case neq:
3221     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3222     vpxor(dst, dst, ExternalAddress(StubRoutines::x86::vector_all_bits_set()), vector_len, scratch_reg);
3223     break;
3224   case le:
3225     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3226     vpxor(dst, dst, ExternalAddress(StubRoutines::x86::vector_all_bits_set()), vector_len, scratch_reg);
3227     break;
3228   case nlt:
3229     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3230     vpxor(dst, dst, ExternalAddress(StubRoutines::x86::vector_all_bits_set()), vector_len, scratch_reg);
3231     break;
3232   case lt:
3233     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3234     break;
3235   case nle:
3236     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3237     break;
3238   default:
3239     assert(false, "Should not reach here");
3240   }
3241 }
3242 
3243 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3244   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3245   Assembler::vpmovzxbw(dst, src, vector_len);
3246 }
3247 
3248 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
3249   assert((src->encoding() < 16),"XMM register should be 0-15");
3250   Assembler::vpmovmskb(dst, src, vector_len);
3251 }
3252 
3253 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3254   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3255   Assembler::vpmullw(dst, nds, src, vector_len);
3256 }
3257 
3258 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3259   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3260   Assembler::vpmullw(dst, nds, src, vector_len);
3261 }
3262 
3263 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3264   assert((UseAVX > 0), "AVX support is needed");
3265   if (reachable(src)) {
3266     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
3267   } else {
3268     lea(scratch_reg, src);
3269     Assembler::vpmulld(dst, nds, Address(scratch_reg, 0), vector_len);
3270   }
3271 }
3272 
3273 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3274   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3275   Assembler::vpsubb(dst, nds, src, vector_len);
3276 }
3277 
3278 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3279   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3280   Assembler::vpsubb(dst, nds, src, vector_len);
3281 }
3282 
3283 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3284   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3285   Assembler::vpsubw(dst, nds, src, vector_len);
3286 }
3287 
3288 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3289   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3290   Assembler::vpsubw(dst, nds, src, vector_len);
3291 }
3292 
3293 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3294   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3295   Assembler::vpsraw(dst, nds, shift, vector_len);
3296 }
3297 
3298 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3299   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3300   Assembler::vpsraw(dst, nds, shift, vector_len);
3301 }
3302 
3303 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3304   assert(UseAVX > 2,"");
3305   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3306      vector_len = 2;
3307   }
3308   Assembler::evpsraq(dst, nds, shift, vector_len);
3309 }
3310 
3311 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3312   assert(UseAVX > 2,"");
3313   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3314      vector_len = 2;
3315   }
3316   Assembler::evpsraq(dst, nds, shift, vector_len);
3317 }
3318 
3319 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3320   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3321   Assembler::vpsrlw(dst, nds, shift, vector_len);
3322 }
3323 
3324 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3325   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3326   Assembler::vpsrlw(dst, nds, shift, vector_len);
3327 }
3328 
3329 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3330   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3331   Assembler::vpsllw(dst, nds, shift, vector_len);
3332 }
3333 
3334 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3335   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3336   Assembler::vpsllw(dst, nds, shift, vector_len);
3337 }
3338 
3339 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
3340   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
3341   Assembler::vptest(dst, src);
3342 }
3343 
3344 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3345   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3346   Assembler::punpcklbw(dst, src);
3347 }
3348 
3349 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
3350   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3351   Assembler::pshufd(dst, src, mode);
3352 }
3353 
3354 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3355   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3356   Assembler::pshuflw(dst, src, mode);
3357 }
3358 
3359 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3360   if (reachable(src)) {
3361     vandpd(dst, nds, as_Address(src), vector_len);
3362   } else {
3363     lea(scratch_reg, src);
3364     vandpd(dst, nds, Address(scratch_reg, 0), vector_len);
3365   }
3366 }
3367 
3368 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3369   if (reachable(src)) {
3370     vandps(dst, nds, as_Address(src), vector_len);
3371   } else {
3372     lea(scratch_reg, src);
3373     vandps(dst, nds, Address(scratch_reg, 0), vector_len);
3374   }
3375 }
3376 
3377 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
3378                             bool merge, int vector_len, Register scratch_reg) {
3379   if (reachable(src)) {
3380     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
3381   } else {
3382     lea(scratch_reg, src);
3383     Assembler::evpord(dst, mask, nds, Address(scratch_reg, 0), merge, vector_len);
3384   }
3385 }
3386 
3387 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3388   if (reachable(src)) {
3389     vdivsd(dst, nds, as_Address(src));
3390   } else {
3391     lea(rscratch1, src);
3392     vdivsd(dst, nds, Address(rscratch1, 0));
3393   }
3394 }
3395 
3396 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3397   if (reachable(src)) {
3398     vdivss(dst, nds, as_Address(src));
3399   } else {
3400     lea(rscratch1, src);
3401     vdivss(dst, nds, Address(rscratch1, 0));
3402   }
3403 }
3404 
3405 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3406   if (reachable(src)) {
3407     vmulsd(dst, nds, as_Address(src));
3408   } else {
3409     lea(rscratch1, src);
3410     vmulsd(dst, nds, Address(rscratch1, 0));
3411   }
3412 }
3413 
3414 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3415   if (reachable(src)) {
3416     vmulss(dst, nds, as_Address(src));
3417   } else {
3418     lea(rscratch1, src);
3419     vmulss(dst, nds, Address(rscratch1, 0));
3420   }
3421 }
3422 
3423 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3424   if (reachable(src)) {
3425     vsubsd(dst, nds, as_Address(src));
3426   } else {
3427     lea(rscratch1, src);
3428     vsubsd(dst, nds, Address(rscratch1, 0));
3429   }
3430 }
3431 
3432 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3433   if (reachable(src)) {
3434     vsubss(dst, nds, as_Address(src));
3435   } else {
3436     lea(rscratch1, src);
3437     vsubss(dst, nds, Address(rscratch1, 0));
3438   }
3439 }
3440 
3441 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3442   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3443   vxorps(dst, nds, src, Assembler::AVX_128bit);
3444 }
3445 
3446 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3447   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3448   vxorpd(dst, nds, src, Assembler::AVX_128bit);
3449 }
3450 
3451 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3452   if (reachable(src)) {
3453     vxorpd(dst, nds, as_Address(src), vector_len);
3454   } else {
3455     lea(scratch_reg, src);
3456     vxorpd(dst, nds, Address(scratch_reg, 0), vector_len);
3457   }
3458 }
3459 
3460 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3461   if (reachable(src)) {
3462     vxorps(dst, nds, as_Address(src), vector_len);
3463   } else {
3464     lea(scratch_reg, src);
3465     vxorps(dst, nds, Address(scratch_reg, 0), vector_len);
3466   }
3467 }
3468 
3469 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3470   if (UseAVX > 1 || (vector_len < 1)) {
3471     if (reachable(src)) {
3472       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
3473     } else {
3474       lea(scratch_reg, src);
3475       Assembler::vpxor(dst, nds, Address(scratch_reg, 0), vector_len);
3476     }
3477   }
3478   else {
3479     MacroAssembler::vxorpd(dst, nds, src, vector_len, scratch_reg);
3480   }
3481 }
3482 
3483 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3484   if (reachable(src)) {
3485     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
3486   } else {
3487     lea(scratch_reg, src);
3488     Assembler::vpermd(dst, nds, Address(scratch_reg, 0), vector_len);
3489   }
3490 }
3491 
3492 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
3493   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
3494   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
3495   // The inverted mask is sign-extended
3496   andptr(possibly_jweak, inverted_jweak_mask);
3497 }
3498 
3499 void MacroAssembler::resolve_jobject(Register value,
3500                                      Register thread,
3501                                      Register tmp) {
3502   assert_different_registers(value, thread, tmp);
3503   Label done, not_weak;
3504   testptr(value, value);
3505   jcc(Assembler::zero, done);                // Use NULL as-is.
3506   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
3507   jcc(Assembler::zero, not_weak);
3508   // Resolve jweak.
3509   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3510                  value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
3511   verify_oop(value);
3512   jmp(done);
3513   bind(not_weak);
3514   // Resolve (untagged) jobject.
3515   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
3516   verify_oop(value);
3517   bind(done);
3518 }
3519 
3520 void MacroAssembler::subptr(Register dst, int32_t imm32) {
3521   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
3522 }
3523 
3524 // Force generation of a 4 byte immediate value even if it fits into 8bit
3525 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
3526   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
3527 }
3528 
3529 void MacroAssembler::subptr(Register dst, Register src) {
3530   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
3531 }
3532 
3533 // C++ bool manipulation
3534 void MacroAssembler::testbool(Register dst) {
3535   if(sizeof(bool) == 1)
3536     testb(dst, 0xff);
3537   else if(sizeof(bool) == 2) {
3538     // testw implementation needed for two byte bools
3539     ShouldNotReachHere();
3540   } else if(sizeof(bool) == 4)
3541     testl(dst, dst);
3542   else
3543     // unsupported
3544     ShouldNotReachHere();
3545 }
3546 
3547 void MacroAssembler::testptr(Register dst, Register src) {
3548   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
3549 }
3550 
3551 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3552 void MacroAssembler::tlab_allocate(Register thread, Register obj,
3553                                    Register var_size_in_bytes,
3554                                    int con_size_in_bytes,
3555                                    Register t1,
3556                                    Register t2,
3557                                    Label& slow_case) {
3558   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3559   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3560 }
3561 
3562 // Defines obj, preserves var_size_in_bytes
3563 void MacroAssembler::eden_allocate(Register thread, Register obj,
3564                                    Register var_size_in_bytes,
3565                                    int con_size_in_bytes,
3566                                    Register t1,
3567                                    Label& slow_case) {
3568   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3569   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3570 }
3571 
3572 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3573 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3574   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3575   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3576   Label done;
3577 
3578   testptr(length_in_bytes, length_in_bytes);
3579   jcc(Assembler::zero, done);
3580 
3581   // initialize topmost word, divide index by 2, check if odd and test if zero
3582   // note: for the remaining code to work, index must be a multiple of BytesPerWord
3583 #ifdef ASSERT
3584   {
3585     Label L;
3586     testptr(length_in_bytes, BytesPerWord - 1);
3587     jcc(Assembler::zero, L);
3588     stop("length must be a multiple of BytesPerWord");
3589     bind(L);
3590   }
3591 #endif
3592   Register index = length_in_bytes;
3593   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
3594   if (UseIncDec) {
3595     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
3596   } else {
3597     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
3598     shrptr(index, 1);
3599   }
3600 #ifndef _LP64
3601   // index could have not been a multiple of 8 (i.e., bit 2 was set)
3602   {
3603     Label even;
3604     // note: if index was a multiple of 8, then it cannot
3605     //       be 0 now otherwise it must have been 0 before
3606     //       => if it is even, we don't need to check for 0 again
3607     jcc(Assembler::carryClear, even);
3608     // clear topmost word (no jump would be needed if conditional assignment worked here)
3609     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3610     // index could be 0 now, must check again
3611     jcc(Assembler::zero, done);
3612     bind(even);
3613   }
3614 #endif // !_LP64
3615   // initialize remaining object fields: index is a multiple of 2 now
3616   {
3617     Label loop;
3618     bind(loop);
3619     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
3620     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
3621     decrement(index);
3622     jcc(Assembler::notZero, loop);
3623   }
3624 
3625   bind(done);
3626 }
3627 
3628 // Look up the method for a megamorphic invokeinterface call.
3629 // The target method is determined by <intf_klass, itable_index>.
3630 // The receiver klass is in recv_klass.
3631 // On success, the result will be in method_result, and execution falls through.
3632 // On failure, execution transfers to the given label.
3633 void MacroAssembler::lookup_interface_method(Register recv_klass,
3634                                              Register intf_klass,
3635                                              RegisterOrConstant itable_index,
3636                                              Register method_result,
3637                                              Register scan_temp,
3638                                              Label& L_no_such_interface,
3639                                              bool return_method) {
3640   assert_different_registers(recv_klass, intf_klass, scan_temp);
3641   assert_different_registers(method_result, intf_klass, scan_temp);
3642   assert(recv_klass != method_result || !return_method,
3643          "recv_klass can be destroyed when method isn't needed");
3644 
3645   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
3646          "caller must use same register for non-constant itable index as for method");
3647 
3648   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
3649   int vtable_base = in_bytes(Klass::vtable_start_offset());
3650   int itentry_off = itableMethodEntry::method_offset_in_bytes();
3651   int scan_step   = itableOffsetEntry::size() * wordSize;
3652   int vte_size    = vtableEntry::size_in_bytes();
3653   Address::ScaleFactor times_vte_scale = Address::times_ptr;
3654   assert(vte_size == wordSize, "else adjust times_vte_scale");
3655 
3656   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
3657 
3658   // %%% Could store the aligned, prescaled offset in the klassoop.
3659   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
3660 
3661   if (return_method) {
3662     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
3663     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
3664     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
3665   }
3666 
3667   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
3668   //   if (scan->interface() == intf) {
3669   //     result = (klass + scan->offset() + itable_index);
3670   //   }
3671   // }
3672   Label search, found_method;
3673 
3674   for (int peel = 1; peel >= 0; peel--) {
3675     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
3676     cmpptr(intf_klass, method_result);
3677 
3678     if (peel) {
3679       jccb(Assembler::equal, found_method);
3680     } else {
3681       jccb(Assembler::notEqual, search);
3682       // (invert the test to fall through to found_method...)
3683     }
3684 
3685     if (!peel)  break;
3686 
3687     bind(search);
3688 
3689     // Check that the previous entry is non-null.  A null entry means that
3690     // the receiver class doesn't implement the interface, and wasn't the
3691     // same as when the caller was compiled.
3692     testptr(method_result, method_result);
3693     jcc(Assembler::zero, L_no_such_interface);
3694     addptr(scan_temp, scan_step);
3695   }
3696 
3697   bind(found_method);
3698 
3699   if (return_method) {
3700     // Got a hit.
3701     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
3702     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
3703   }
3704 }
3705 
3706 
3707 // virtual method calling
3708 void MacroAssembler::lookup_virtual_method(Register recv_klass,
3709                                            RegisterOrConstant vtable_index,
3710                                            Register method_result) {
3711   const int base = in_bytes(Klass::vtable_start_offset());
3712   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
3713   Address vtable_entry_addr(recv_klass,
3714                             vtable_index, Address::times_ptr,
3715                             base + vtableEntry::method_offset_in_bytes());
3716   movptr(method_result, vtable_entry_addr);
3717 }
3718 
3719 
3720 void MacroAssembler::check_klass_subtype(Register sub_klass,
3721                            Register super_klass,
3722                            Register temp_reg,
3723                            Label& L_success) {
3724   Label L_failure;
3725   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
3726   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
3727   bind(L_failure);
3728 }
3729 
3730 
3731 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3732                                                    Register super_klass,
3733                                                    Register temp_reg,
3734                                                    Label* L_success,
3735                                                    Label* L_failure,
3736                                                    Label* L_slow_path,
3737                                         RegisterOrConstant super_check_offset) {
3738   assert_different_registers(sub_klass, super_klass, temp_reg);
3739   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
3740   if (super_check_offset.is_register()) {
3741     assert_different_registers(sub_klass, super_klass,
3742                                super_check_offset.as_register());
3743   } else if (must_load_sco) {
3744     assert(temp_reg != noreg, "supply either a temp or a register offset");
3745   }
3746 
3747   Label L_fallthrough;
3748   int label_nulls = 0;
3749   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
3750   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
3751   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
3752   assert(label_nulls <= 1, "at most one NULL in the batch");
3753 
3754   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3755   int sco_offset = in_bytes(Klass::super_check_offset_offset());
3756   Address super_check_offset_addr(super_klass, sco_offset);
3757 
3758   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
3759   // range of a jccb.  If this routine grows larger, reconsider at
3760   // least some of these.
3761 #define local_jcc(assembler_cond, label)                                \
3762   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
3763   else                             jcc( assembler_cond, label) /*omit semi*/
3764 
3765   // Hacked jmp, which may only be used just before L_fallthrough.
3766 #define final_jmp(label)                                                \
3767   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
3768   else                            jmp(label)                /*omit semi*/
3769 
3770   // If the pointers are equal, we are done (e.g., String[] elements).
3771   // This self-check enables sharing of secondary supertype arrays among
3772   // non-primary types such as array-of-interface.  Otherwise, each such
3773   // type would need its own customized SSA.
3774   // We move this check to the front of the fast path because many
3775   // type checks are in fact trivially successful in this manner,
3776   // so we get a nicely predicted branch right at the start of the check.
3777   cmpptr(sub_klass, super_klass);
3778   local_jcc(Assembler::equal, *L_success);
3779 
3780   // Check the supertype display:
3781   if (must_load_sco) {
3782     // Positive movl does right thing on LP64.
3783     movl(temp_reg, super_check_offset_addr);
3784     super_check_offset = RegisterOrConstant(temp_reg);
3785   }
3786   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
3787   cmpptr(super_klass, super_check_addr); // load displayed supertype
3788 
3789   // This check has worked decisively for primary supers.
3790   // Secondary supers are sought in the super_cache ('super_cache_addr').
3791   // (Secondary supers are interfaces and very deeply nested subtypes.)
3792   // This works in the same check above because of a tricky aliasing
3793   // between the super_cache and the primary super display elements.
3794   // (The 'super_check_addr' can address either, as the case requires.)
3795   // Note that the cache is updated below if it does not help us find
3796   // what we need immediately.
3797   // So if it was a primary super, we can just fail immediately.
3798   // Otherwise, it's the slow path for us (no success at this point).
3799 
3800   if (super_check_offset.is_register()) {
3801     local_jcc(Assembler::equal, *L_success);
3802     cmpl(super_check_offset.as_register(), sc_offset);
3803     if (L_failure == &L_fallthrough) {
3804       local_jcc(Assembler::equal, *L_slow_path);
3805     } else {
3806       local_jcc(Assembler::notEqual, *L_failure);
3807       final_jmp(*L_slow_path);
3808     }
3809   } else if (super_check_offset.as_constant() == sc_offset) {
3810     // Need a slow path; fast failure is impossible.
3811     if (L_slow_path == &L_fallthrough) {
3812       local_jcc(Assembler::equal, *L_success);
3813     } else {
3814       local_jcc(Assembler::notEqual, *L_slow_path);
3815       final_jmp(*L_success);
3816     }
3817   } else {
3818     // No slow path; it's a fast decision.
3819     if (L_failure == &L_fallthrough) {
3820       local_jcc(Assembler::equal, *L_success);
3821     } else {
3822       local_jcc(Assembler::notEqual, *L_failure);
3823       final_jmp(*L_success);
3824     }
3825   }
3826 
3827   bind(L_fallthrough);
3828 
3829 #undef local_jcc
3830 #undef final_jmp
3831 }
3832 
3833 
3834 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3835                                                    Register super_klass,
3836                                                    Register temp_reg,
3837                                                    Register temp2_reg,
3838                                                    Label* L_success,
3839                                                    Label* L_failure,
3840                                                    bool set_cond_codes) {
3841   assert_different_registers(sub_klass, super_klass, temp_reg);
3842   if (temp2_reg != noreg)
3843     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
3844 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
3845 
3846   Label L_fallthrough;
3847   int label_nulls = 0;
3848   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
3849   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
3850   assert(label_nulls <= 1, "at most one NULL in the batch");
3851 
3852   // a couple of useful fields in sub_klass:
3853   int ss_offset = in_bytes(Klass::secondary_supers_offset());
3854   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3855   Address secondary_supers_addr(sub_klass, ss_offset);
3856   Address super_cache_addr(     sub_klass, sc_offset);
3857 
3858   // Do a linear scan of the secondary super-klass chain.
3859   // This code is rarely used, so simplicity is a virtue here.
3860   // The repne_scan instruction uses fixed registers, which we must spill.
3861   // Don't worry too much about pre-existing connections with the input regs.
3862 
3863   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
3864   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
3865 
3866   // Get super_klass value into rax (even if it was in rdi or rcx).
3867   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
3868   if (super_klass != rax || UseCompressedOops) {
3869     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
3870     mov(rax, super_klass);
3871   }
3872   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
3873   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
3874 
3875 #ifndef PRODUCT
3876   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
3877   ExternalAddress pst_counter_addr((address) pst_counter);
3878   NOT_LP64(  incrementl(pst_counter_addr) );
3879   LP64_ONLY( lea(rcx, pst_counter_addr) );
3880   LP64_ONLY( incrementl(Address(rcx, 0)) );
3881 #endif //PRODUCT
3882 
3883   // We will consult the secondary-super array.
3884   movptr(rdi, secondary_supers_addr);
3885   // Load the array length.  (Positive movl does right thing on LP64.)
3886   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
3887   // Skip to start of data.
3888   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
3889 
3890   // Scan RCX words at [RDI] for an occurrence of RAX.
3891   // Set NZ/Z based on last compare.
3892   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
3893   // not change flags (only scas instruction which is repeated sets flags).
3894   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
3895 
3896     testptr(rax,rax); // Set Z = 0
3897     repne_scan();
3898 
3899   // Unspill the temp. registers:
3900   if (pushed_rdi)  pop(rdi);
3901   if (pushed_rcx)  pop(rcx);
3902   if (pushed_rax)  pop(rax);
3903 
3904   if (set_cond_codes) {
3905     // Special hack for the AD files:  rdi is guaranteed non-zero.
3906     assert(!pushed_rdi, "rdi must be left non-NULL");
3907     // Also, the condition codes are properly set Z/NZ on succeed/failure.
3908   }
3909 
3910   if (L_failure == &L_fallthrough)
3911         jccb(Assembler::notEqual, *L_failure);
3912   else  jcc(Assembler::notEqual, *L_failure);
3913 
3914   // Success.  Cache the super we found and proceed in triumph.
3915   movptr(super_cache_addr, super_klass);
3916 
3917   if (L_success != &L_fallthrough) {
3918     jmp(*L_success);
3919   }
3920 
3921 #undef IS_A_TEMP
3922 
3923   bind(L_fallthrough);
3924 }
3925 
3926 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
3927   assert(L_fast_path != NULL || L_slow_path != NULL, "at least one is required");
3928 
3929   Label L_fallthrough;
3930   if (L_fast_path == NULL) {
3931     L_fast_path = &L_fallthrough;
3932   } else if (L_slow_path == NULL) {
3933     L_slow_path = &L_fallthrough;
3934   }
3935 
3936   // Fast path check: class is fully initialized
3937   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
3938   jcc(Assembler::equal, *L_fast_path);
3939 
3940   // Fast path check: current thread is initializer thread
3941   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
3942   if (L_slow_path == &L_fallthrough) {
3943     jcc(Assembler::equal, *L_fast_path);
3944     bind(*L_slow_path);
3945   } else if (L_fast_path == &L_fallthrough) {
3946     jcc(Assembler::notEqual, *L_slow_path);
3947     bind(*L_fast_path);
3948   } else {
3949     Unimplemented();
3950   }
3951 }
3952 
3953 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
3954   if (VM_Version::supports_cmov()) {
3955     cmovl(cc, dst, src);
3956   } else {
3957     Label L;
3958     jccb(negate_condition(cc), L);
3959     movl(dst, src);
3960     bind(L);
3961   }
3962 }
3963 
3964 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
3965   if (VM_Version::supports_cmov()) {
3966     cmovl(cc, dst, src);
3967   } else {
3968     Label L;
3969     jccb(negate_condition(cc), L);
3970     movl(dst, src);
3971     bind(L);
3972   }
3973 }
3974 
3975 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
3976   if (!VerifyOops) return;
3977 
3978   // Pass register number to verify_oop_subroutine
3979   const char* b = NULL;
3980   {
3981     ResourceMark rm;
3982     stringStream ss;
3983     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
3984     b = code_string(ss.as_string());
3985   }
3986   BLOCK_COMMENT("verify_oop {");
3987 #ifdef _LP64
3988   push(rscratch1);                    // save r10, trashed by movptr()
3989 #endif
3990   push(rax);                          // save rax,
3991   push(reg);                          // pass register argument
3992   ExternalAddress buffer((address) b);
3993   // avoid using pushptr, as it modifies scratch registers
3994   // and our contract is not to modify anything
3995   movptr(rax, buffer.addr());
3996   push(rax);
3997   // call indirectly to solve generation ordering problem
3998   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
3999   call(rax);
4000   // Caller pops the arguments (oop, message) and restores rax, r10
4001   BLOCK_COMMENT("} verify_oop");
4002 }
4003 
4004 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
4005   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
4006     vpternlogd(dst, 0xFF, dst, dst, vector_len);
4007   } else {
4008     assert(UseAVX > 0, "");
4009     vpcmpeqb(dst, dst, dst, vector_len);
4010   }
4011 }
4012 
4013 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
4014                                          int extra_slot_offset) {
4015   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4016   int stackElementSize = Interpreter::stackElementSize;
4017   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4018 #ifdef ASSERT
4019   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4020   assert(offset1 - offset == stackElementSize, "correct arithmetic");
4021 #endif
4022   Register             scale_reg    = noreg;
4023   Address::ScaleFactor scale_factor = Address::no_scale;
4024   if (arg_slot.is_constant()) {
4025     offset += arg_slot.as_constant() * stackElementSize;
4026   } else {
4027     scale_reg    = arg_slot.as_register();
4028     scale_factor = Address::times(stackElementSize);
4029   }
4030   offset += wordSize;           // return PC is on stack
4031   return Address(rsp, scale_reg, scale_factor, offset);
4032 }
4033 
4034 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
4035   if (!VerifyOops) return;
4036 
4037   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
4038   // Pass register number to verify_oop_subroutine
4039   const char* b = NULL;
4040   {
4041     ResourceMark rm;
4042     stringStream ss;
4043     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
4044     b = code_string(ss.as_string());
4045   }
4046 #ifdef _LP64
4047   push(rscratch1);                    // save r10, trashed by movptr()
4048 #endif
4049   push(rax);                          // save rax,
4050   // addr may contain rsp so we will have to adjust it based on the push
4051   // we just did (and on 64 bit we do two pushes)
4052   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4053   // stores rax into addr which is backwards of what was intended.
4054   if (addr.uses(rsp)) {
4055     lea(rax, addr);
4056     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
4057   } else {
4058     pushptr(addr);
4059   }
4060 
4061   ExternalAddress buffer((address) b);
4062   // pass msg argument
4063   // avoid using pushptr, as it modifies scratch registers
4064   // and our contract is not to modify anything
4065   movptr(rax, buffer.addr());
4066   push(rax);
4067 
4068   // call indirectly to solve generation ordering problem
4069   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4070   call(rax);
4071   // Caller pops the arguments (addr, message) and restores rax, r10.
4072 }
4073 
4074 void MacroAssembler::verify_tlab() {
4075 #ifdef ASSERT
4076   if (UseTLAB && VerifyOops) {
4077     Label next, ok;
4078     Register t1 = rsi;
4079     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
4080 
4081     push(t1);
4082     NOT_LP64(push(thread_reg));
4083     NOT_LP64(get_thread(thread_reg));
4084 
4085     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
4086     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
4087     jcc(Assembler::aboveEqual, next);
4088     STOP("assert(top >= start)");
4089     should_not_reach_here();
4090 
4091     bind(next);
4092     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
4093     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
4094     jcc(Assembler::aboveEqual, ok);
4095     STOP("assert(top <= end)");
4096     should_not_reach_here();
4097 
4098     bind(ok);
4099     NOT_LP64(pop(thread_reg));
4100     pop(t1);
4101   }
4102 #endif
4103 }
4104 
4105 class ControlWord {
4106  public:
4107   int32_t _value;
4108 
4109   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
4110   int  precision_control() const       { return  (_value >>  8) & 3      ; }
4111   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4112   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4113   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4114   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4115   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4116   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4117 
4118   void print() const {
4119     // rounding control
4120     const char* rc;
4121     switch (rounding_control()) {
4122       case 0: rc = "round near"; break;
4123       case 1: rc = "round down"; break;
4124       case 2: rc = "round up  "; break;
4125       case 3: rc = "chop      "; break;
4126       default:
4127         rc = NULL; // silence compiler warnings
4128         fatal("Unknown rounding control: %d", rounding_control());
4129     };
4130     // precision control
4131     const char* pc;
4132     switch (precision_control()) {
4133       case 0: pc = "24 bits "; break;
4134       case 1: pc = "reserved"; break;
4135       case 2: pc = "53 bits "; break;
4136       case 3: pc = "64 bits "; break;
4137       default:
4138         pc = NULL; // silence compiler warnings
4139         fatal("Unknown precision control: %d", precision_control());
4140     };
4141     // flags
4142     char f[9];
4143     f[0] = ' ';
4144     f[1] = ' ';
4145     f[2] = (precision   ()) ? 'P' : 'p';
4146     f[3] = (underflow   ()) ? 'U' : 'u';
4147     f[4] = (overflow    ()) ? 'O' : 'o';
4148     f[5] = (zero_divide ()) ? 'Z' : 'z';
4149     f[6] = (denormalized()) ? 'D' : 'd';
4150     f[7] = (invalid     ()) ? 'I' : 'i';
4151     f[8] = '\x0';
4152     // output
4153     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
4154   }
4155 
4156 };
4157 
4158 class StatusWord {
4159  public:
4160   int32_t _value;
4161 
4162   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
4163   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
4164   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
4165   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
4166   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
4167   int  top() const                     { return  (_value >> 11) & 7      ; }
4168   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
4169   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
4170   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4171   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4172   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4173   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4174   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4175   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4176 
4177   void print() const {
4178     // condition codes
4179     char c[5];
4180     c[0] = (C3()) ? '3' : '-';
4181     c[1] = (C2()) ? '2' : '-';
4182     c[2] = (C1()) ? '1' : '-';
4183     c[3] = (C0()) ? '0' : '-';
4184     c[4] = '\x0';
4185     // flags
4186     char f[9];
4187     f[0] = (error_status()) ? 'E' : '-';
4188     f[1] = (stack_fault ()) ? 'S' : '-';
4189     f[2] = (precision   ()) ? 'P' : '-';
4190     f[3] = (underflow   ()) ? 'U' : '-';
4191     f[4] = (overflow    ()) ? 'O' : '-';
4192     f[5] = (zero_divide ()) ? 'Z' : '-';
4193     f[6] = (denormalized()) ? 'D' : '-';
4194     f[7] = (invalid     ()) ? 'I' : '-';
4195     f[8] = '\x0';
4196     // output
4197     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
4198   }
4199 
4200 };
4201 
4202 class TagWord {
4203  public:
4204   int32_t _value;
4205 
4206   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
4207 
4208   void print() const {
4209     printf("%04x", _value & 0xFFFF);
4210   }
4211 
4212 };
4213 
4214 class FPU_Register {
4215  public:
4216   int32_t _m0;
4217   int32_t _m1;
4218   int16_t _ex;
4219 
4220   bool is_indefinite() const           {
4221     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
4222   }
4223 
4224   void print() const {
4225     char  sign = (_ex < 0) ? '-' : '+';
4226     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
4227     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
4228   };
4229 
4230 };
4231 
4232 class FPU_State {
4233  public:
4234   enum {
4235     register_size       = 10,
4236     number_of_registers =  8,
4237     register_mask       =  7
4238   };
4239 
4240   ControlWord  _control_word;
4241   StatusWord   _status_word;
4242   TagWord      _tag_word;
4243   int32_t      _error_offset;
4244   int32_t      _error_selector;
4245   int32_t      _data_offset;
4246   int32_t      _data_selector;
4247   int8_t       _register[register_size * number_of_registers];
4248 
4249   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
4250   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
4251 
4252   const char* tag_as_string(int tag) const {
4253     switch (tag) {
4254       case 0: return "valid";
4255       case 1: return "zero";
4256       case 2: return "special";
4257       case 3: return "empty";
4258     }
4259     ShouldNotReachHere();
4260     return NULL;
4261   }
4262 
4263   void print() const {
4264     // print computation registers
4265     { int t = _status_word.top();
4266       for (int i = 0; i < number_of_registers; i++) {
4267         int j = (i - t) & register_mask;
4268         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
4269         st(j)->print();
4270         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
4271       }
4272     }
4273     printf("\n");
4274     // print control registers
4275     printf("ctrl = "); _control_word.print(); printf("\n");
4276     printf("stat = "); _status_word .print(); printf("\n");
4277     printf("tags = "); _tag_word    .print(); printf("\n");
4278   }
4279 
4280 };
4281 
4282 class Flag_Register {
4283  public:
4284   int32_t _value;
4285 
4286   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
4287   bool direction() const               { return ((_value >> 10) & 1) != 0; }
4288   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
4289   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
4290   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
4291   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
4292   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
4293 
4294   void print() const {
4295     // flags
4296     char f[8];
4297     f[0] = (overflow       ()) ? 'O' : '-';
4298     f[1] = (direction      ()) ? 'D' : '-';
4299     f[2] = (sign           ()) ? 'S' : '-';
4300     f[3] = (zero           ()) ? 'Z' : '-';
4301     f[4] = (auxiliary_carry()) ? 'A' : '-';
4302     f[5] = (parity         ()) ? 'P' : '-';
4303     f[6] = (carry          ()) ? 'C' : '-';
4304     f[7] = '\x0';
4305     // output
4306     printf("%08x  flags = %s", _value, f);
4307   }
4308 
4309 };
4310 
4311 class IU_Register {
4312  public:
4313   int32_t _value;
4314 
4315   void print() const {
4316     printf("%08x  %11d", _value, _value);
4317   }
4318 
4319 };
4320 
4321 class IU_State {
4322  public:
4323   Flag_Register _eflags;
4324   IU_Register   _rdi;
4325   IU_Register   _rsi;
4326   IU_Register   _rbp;
4327   IU_Register   _rsp;
4328   IU_Register   _rbx;
4329   IU_Register   _rdx;
4330   IU_Register   _rcx;
4331   IU_Register   _rax;
4332 
4333   void print() const {
4334     // computation registers
4335     printf("rax,  = "); _rax.print(); printf("\n");
4336     printf("rbx,  = "); _rbx.print(); printf("\n");
4337     printf("rcx  = "); _rcx.print(); printf("\n");
4338     printf("rdx  = "); _rdx.print(); printf("\n");
4339     printf("rdi  = "); _rdi.print(); printf("\n");
4340     printf("rsi  = "); _rsi.print(); printf("\n");
4341     printf("rbp,  = "); _rbp.print(); printf("\n");
4342     printf("rsp  = "); _rsp.print(); printf("\n");
4343     printf("\n");
4344     // control registers
4345     printf("flgs = "); _eflags.print(); printf("\n");
4346   }
4347 };
4348 
4349 
4350 class CPU_State {
4351  public:
4352   FPU_State _fpu_state;
4353   IU_State  _iu_state;
4354 
4355   void print() const {
4356     printf("--------------------------------------------------\n");
4357     _iu_state .print();
4358     printf("\n");
4359     _fpu_state.print();
4360     printf("--------------------------------------------------\n");
4361   }
4362 
4363 };
4364 
4365 
4366 static void _print_CPU_state(CPU_State* state) {
4367   state->print();
4368 };
4369 
4370 
4371 void MacroAssembler::print_CPU_state() {
4372   push_CPU_state();
4373   push(rsp);                // pass CPU state
4374   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
4375   addptr(rsp, wordSize);       // discard argument
4376   pop_CPU_state();
4377 }
4378 
4379 
4380 #ifndef _LP64
4381 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
4382   static int counter = 0;
4383   FPU_State* fs = &state->_fpu_state;
4384   counter++;
4385   // For leaf calls, only verify that the top few elements remain empty.
4386   // We only need 1 empty at the top for C2 code.
4387   if( stack_depth < 0 ) {
4388     if( fs->tag_for_st(7) != 3 ) {
4389       printf("FPR7 not empty\n");
4390       state->print();
4391       assert(false, "error");
4392       return false;
4393     }
4394     return true;                // All other stack states do not matter
4395   }
4396 
4397   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
4398          "bad FPU control word");
4399 
4400   // compute stack depth
4401   int i = 0;
4402   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
4403   int d = i;
4404   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
4405   // verify findings
4406   if (i != FPU_State::number_of_registers) {
4407     // stack not contiguous
4408     printf("%s: stack not contiguous at ST%d\n", s, i);
4409     state->print();
4410     assert(false, "error");
4411     return false;
4412   }
4413   // check if computed stack depth corresponds to expected stack depth
4414   if (stack_depth < 0) {
4415     // expected stack depth is -stack_depth or less
4416     if (d > -stack_depth) {
4417       // too many elements on the stack
4418       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
4419       state->print();
4420       assert(false, "error");
4421       return false;
4422     }
4423   } else {
4424     // expected stack depth is stack_depth
4425     if (d != stack_depth) {
4426       // wrong stack depth
4427       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
4428       state->print();
4429       assert(false, "error");
4430       return false;
4431     }
4432   }
4433   // everything is cool
4434   return true;
4435 }
4436 
4437 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
4438   if (!VerifyFPU) return;
4439   push_CPU_state();
4440   push(rsp);                // pass CPU state
4441   ExternalAddress msg((address) s);
4442   // pass message string s
4443   pushptr(msg.addr());
4444   push(stack_depth);        // pass stack depth
4445   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
4446   addptr(rsp, 3 * wordSize);   // discard arguments
4447   // check for error
4448   { Label L;
4449     testl(rax, rax);
4450     jcc(Assembler::notZero, L);
4451     int3();                  // break if error condition
4452     bind(L);
4453   }
4454   pop_CPU_state();
4455 }
4456 #endif // _LP64
4457 
4458 void MacroAssembler::restore_cpu_control_state_after_jni() {
4459   // Either restore the MXCSR register after returning from the JNI Call
4460   // or verify that it wasn't changed (with -Xcheck:jni flag).
4461   if (VM_Version::supports_sse()) {
4462     if (RestoreMXCSROnJNICalls) {
4463       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()));
4464     } else if (CheckJNICalls) {
4465       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
4466     }
4467   }
4468   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
4469   vzeroupper();
4470   // Reset k1 to 0xffff.
4471 
4472 #ifdef COMPILER2
4473   if (PostLoopMultiversioning && VM_Version::supports_evex()) {
4474     push(rcx);
4475     movl(rcx, 0xffff);
4476     kmovwl(k1, rcx);
4477     pop(rcx);
4478   }
4479 #endif // COMPILER2
4480 
4481 #ifndef _LP64
4482   // Either restore the x87 floating pointer control word after returning
4483   // from the JNI call or verify that it wasn't changed.
4484   if (CheckJNICalls) {
4485     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
4486   }
4487 #endif // _LP64
4488 }
4489 
4490 // ((OopHandle)result).resolve();
4491 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
4492   assert_different_registers(result, tmp);
4493 
4494   // Only 64 bit platforms support GCs that require a tmp register
4495   // Only IN_HEAP loads require a thread_tmp register
4496   // OopHandle::resolve is an indirection like jobject.
4497   access_load_at(T_OBJECT, IN_NATIVE,
4498                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
4499 }
4500 
4501 // ((WeakHandle)result).resolve();
4502 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
4503   assert_different_registers(rresult, rtmp);
4504   Label resolved;
4505 
4506   // A null weak handle resolves to null.
4507   cmpptr(rresult, 0);
4508   jcc(Assembler::equal, resolved);
4509 
4510   // Only 64 bit platforms support GCs that require a tmp register
4511   // Only IN_HEAP loads require a thread_tmp register
4512   // WeakHandle::resolve is an indirection like jweak.
4513   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4514                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
4515   bind(resolved);
4516 }
4517 
4518 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4519   // get mirror
4520   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4521   load_method_holder(mirror, method);
4522   movptr(mirror, Address(mirror, mirror_offset));
4523   resolve_oop_handle(mirror, tmp);
4524 }
4525 
4526 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4527   load_method_holder(rresult, rmethod);
4528   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4529 }
4530 
4531 void MacroAssembler::load_method_holder(Register holder, Register method) {
4532   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4533   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4534   movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4535 }
4536 
4537 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
4538   assert_different_registers(src, tmp);
4539   assert_different_registers(dst, tmp);
4540 #ifdef _LP64
4541   if (UseCompressedClassPointers) {
4542     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4543     decode_klass_not_null(dst, tmp);
4544   } else
4545 #endif
4546     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4547 }
4548 
4549 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4550   assert_different_registers(src, tmp);
4551   assert_different_registers(dst, tmp);
4552 #ifdef _LP64
4553   if (UseCompressedClassPointers) {
4554     encode_klass_not_null(src, tmp);
4555     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4556   } else
4557 #endif
4558     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4559 }
4560 
4561 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4562                                     Register tmp1, Register thread_tmp) {
4563   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4564   decorators = AccessInternal::decorator_fixup(decorators);
4565   bool as_raw = (decorators & AS_RAW) != 0;
4566   if (as_raw) {
4567     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4568   } else {
4569     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4570   }
4571 }
4572 
4573 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4574                                      Register tmp1, Register tmp2) {
4575   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4576   decorators = AccessInternal::decorator_fixup(decorators);
4577   bool as_raw = (decorators & AS_RAW) != 0;
4578   if (as_raw) {
4579     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
4580   } else {
4581     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
4582   }
4583 }
4584 
4585 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4586                                    Register thread_tmp, DecoratorSet decorators) {
4587   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4588 }
4589 
4590 // Doesn't do verfication, generates fixed size code
4591 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4592                                             Register thread_tmp, DecoratorSet decorators) {
4593   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4594 }
4595 
4596 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4597                                     Register tmp2, DecoratorSet decorators) {
4598   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
4599 }
4600 
4601 // Used for storing NULLs.
4602 void MacroAssembler::store_heap_oop_null(Address dst) {
4603   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
4604 }
4605 
4606 #ifdef _LP64
4607 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4608   if (UseCompressedClassPointers) {
4609     // Store to klass gap in destination
4610     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
4611   }
4612 }
4613 
4614 #ifdef ASSERT
4615 void MacroAssembler::verify_heapbase(const char* msg) {
4616   assert (UseCompressedOops, "should be compressed");
4617   assert (Universe::heap() != NULL, "java heap should be initialized");
4618   if (CheckCompressedOops) {
4619     Label ok;
4620     push(rscratch1); // cmpptr trashes rscratch1
4621     cmpptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
4622     jcc(Assembler::equal, ok);
4623     STOP(msg);
4624     bind(ok);
4625     pop(rscratch1);
4626   }
4627 }
4628 #endif
4629 
4630 // Algorithm must match oop.inline.hpp encode_heap_oop.
4631 void MacroAssembler::encode_heap_oop(Register r) {
4632 #ifdef ASSERT
4633   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
4634 #endif
4635   verify_oop_msg(r, "broken oop in encode_heap_oop");
4636   if (CompressedOops::base() == NULL) {
4637     if (CompressedOops::shift() != 0) {
4638       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4639       shrq(r, LogMinObjAlignmentInBytes);
4640     }
4641     return;
4642   }
4643   testq(r, r);
4644   cmovq(Assembler::equal, r, r12_heapbase);
4645   subq(r, r12_heapbase);
4646   shrq(r, LogMinObjAlignmentInBytes);
4647 }
4648 
4649 void MacroAssembler::encode_heap_oop_not_null(Register r) {
4650 #ifdef ASSERT
4651   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
4652   if (CheckCompressedOops) {
4653     Label ok;
4654     testq(r, r);
4655     jcc(Assembler::notEqual, ok);
4656     STOP("null oop passed to encode_heap_oop_not_null");
4657     bind(ok);
4658   }
4659 #endif
4660   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
4661   if (CompressedOops::base() != NULL) {
4662     subq(r, r12_heapbase);
4663   }
4664   if (CompressedOops::shift() != 0) {
4665     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4666     shrq(r, LogMinObjAlignmentInBytes);
4667   }
4668 }
4669 
4670 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
4671 #ifdef ASSERT
4672   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
4673   if (CheckCompressedOops) {
4674     Label ok;
4675     testq(src, src);
4676     jcc(Assembler::notEqual, ok);
4677     STOP("null oop passed to encode_heap_oop_not_null2");
4678     bind(ok);
4679   }
4680 #endif
4681   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
4682   if (dst != src) {
4683     movq(dst, src);
4684   }
4685   if (CompressedOops::base() != NULL) {
4686     subq(dst, r12_heapbase);
4687   }
4688   if (CompressedOops::shift() != 0) {
4689     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4690     shrq(dst, LogMinObjAlignmentInBytes);
4691   }
4692 }
4693 
4694 void  MacroAssembler::decode_heap_oop(Register r) {
4695 #ifdef ASSERT
4696   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
4697 #endif
4698   if (CompressedOops::base() == NULL) {
4699     if (CompressedOops::shift() != 0) {
4700       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4701       shlq(r, LogMinObjAlignmentInBytes);
4702     }
4703   } else {
4704     Label done;
4705     shlq(r, LogMinObjAlignmentInBytes);
4706     jccb(Assembler::equal, done);
4707     addq(r, r12_heapbase);
4708     bind(done);
4709   }
4710   verify_oop_msg(r, "broken oop in decode_heap_oop");
4711 }
4712 
4713 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
4714   // Note: it will change flags
4715   assert (UseCompressedOops, "should only be used for compressed headers");
4716   assert (Universe::heap() != NULL, "java heap should be initialized");
4717   // Cannot assert, unverified entry point counts instructions (see .ad file)
4718   // vtableStubs also counts instructions in pd_code_size_limit.
4719   // Also do not verify_oop as this is called by verify_oop.
4720   if (CompressedOops::shift() != 0) {
4721     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4722     shlq(r, LogMinObjAlignmentInBytes);
4723     if (CompressedOops::base() != NULL) {
4724       addq(r, r12_heapbase);
4725     }
4726   } else {
4727     assert (CompressedOops::base() == NULL, "sanity");
4728   }
4729 }
4730 
4731 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
4732   // Note: it will change flags
4733   assert (UseCompressedOops, "should only be used for compressed headers");
4734   assert (Universe::heap() != NULL, "java heap should be initialized");
4735   // Cannot assert, unverified entry point counts instructions (see .ad file)
4736   // vtableStubs also counts instructions in pd_code_size_limit.
4737   // Also do not verify_oop as this is called by verify_oop.
4738   if (CompressedOops::shift() != 0) {
4739     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4740     if (LogMinObjAlignmentInBytes == Address::times_8) {
4741       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
4742     } else {
4743       if (dst != src) {
4744         movq(dst, src);
4745       }
4746       shlq(dst, LogMinObjAlignmentInBytes);
4747       if (CompressedOops::base() != NULL) {
4748         addq(dst, r12_heapbase);
4749       }
4750     }
4751   } else {
4752     assert (CompressedOops::base() == NULL, "sanity");
4753     if (dst != src) {
4754       movq(dst, src);
4755     }
4756   }
4757 }
4758 
4759 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
4760   assert_different_registers(r, tmp);
4761   if (CompressedKlassPointers::base() != NULL) {
4762     mov64(tmp, (int64_t)CompressedKlassPointers::base());
4763     subq(r, tmp);
4764   }
4765   if (CompressedKlassPointers::shift() != 0) {
4766     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
4767     shrq(r, LogKlassAlignmentInBytes);
4768   }
4769 }
4770 
4771 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
4772   assert_different_registers(src, dst);
4773   if (CompressedKlassPointers::base() != NULL) {
4774     mov64(dst, -(int64_t)CompressedKlassPointers::base());
4775     addq(dst, src);
4776   } else {
4777     movptr(dst, src);
4778   }
4779   if (CompressedKlassPointers::shift() != 0) {
4780     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
4781     shrq(dst, LogKlassAlignmentInBytes);
4782   }
4783 }
4784 
4785 // !!! If the instructions that get generated here change then function
4786 // instr_size_for_decode_klass_not_null() needs to get updated.
4787 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
4788   assert_different_registers(r, tmp);
4789   // Note: it will change flags
4790   assert(UseCompressedClassPointers, "should only be used for compressed headers");
4791   // Cannot assert, unverified entry point counts instructions (see .ad file)
4792   // vtableStubs also counts instructions in pd_code_size_limit.
4793   // Also do not verify_oop as this is called by verify_oop.
4794   if (CompressedKlassPointers::shift() != 0) {
4795     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
4796     shlq(r, LogKlassAlignmentInBytes);
4797   }
4798   if (CompressedKlassPointers::base() != NULL) {
4799     mov64(tmp, (int64_t)CompressedKlassPointers::base());
4800     addq(r, tmp);
4801   }
4802 }
4803 
4804 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
4805   assert_different_registers(src, dst);
4806   // Note: it will change flags
4807   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4808   // Cannot assert, unverified entry point counts instructions (see .ad file)
4809   // vtableStubs also counts instructions in pd_code_size_limit.
4810   // Also do not verify_oop as this is called by verify_oop.
4811 
4812   if (CompressedKlassPointers::base() == NULL &&
4813       CompressedKlassPointers::shift() == 0) {
4814     // The best case scenario is that there is no base or shift. Then it is already
4815     // a pointer that needs nothing but a register rename.
4816     movl(dst, src);
4817   } else {
4818     if (CompressedKlassPointers::base() != NULL) {
4819       mov64(dst, (int64_t)CompressedKlassPointers::base());
4820     } else {
4821       xorq(dst, dst);
4822     }
4823     if (CompressedKlassPointers::shift() != 0) {
4824       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
4825       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
4826       leaq(dst, Address(dst, src, Address::times_8, 0));
4827     } else {
4828       addq(dst, src);
4829     }
4830   }
4831 }
4832 
4833 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
4834   assert (UseCompressedOops, "should only be used for compressed headers");
4835   assert (Universe::heap() != NULL, "java heap should be initialized");
4836   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4837   int oop_index = oop_recorder()->find_index(obj);
4838   RelocationHolder rspec = oop_Relocation::spec(oop_index);
4839   mov_narrow_oop(dst, oop_index, rspec);
4840 }
4841 
4842 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
4843   assert (UseCompressedOops, "should only be used for compressed headers");
4844   assert (Universe::heap() != NULL, "java heap should be initialized");
4845   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4846   int oop_index = oop_recorder()->find_index(obj);
4847   RelocationHolder rspec = oop_Relocation::spec(oop_index);
4848   mov_narrow_oop(dst, oop_index, rspec);
4849 }
4850 
4851 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
4852   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4853   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4854   int klass_index = oop_recorder()->find_index(k);
4855   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
4856   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
4857 }
4858 
4859 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
4860   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4861   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4862   int klass_index = oop_recorder()->find_index(k);
4863   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
4864   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
4865 }
4866 
4867 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
4868   assert (UseCompressedOops, "should only be used for compressed headers");
4869   assert (Universe::heap() != NULL, "java heap should be initialized");
4870   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4871   int oop_index = oop_recorder()->find_index(obj);
4872   RelocationHolder rspec = oop_Relocation::spec(oop_index);
4873   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
4874 }
4875 
4876 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
4877   assert (UseCompressedOops, "should only be used for compressed headers");
4878   assert (Universe::heap() != NULL, "java heap should be initialized");
4879   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4880   int oop_index = oop_recorder()->find_index(obj);
4881   RelocationHolder rspec = oop_Relocation::spec(oop_index);
4882   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
4883 }
4884 
4885 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
4886   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4887   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4888   int klass_index = oop_recorder()->find_index(k);
4889   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
4890   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
4891 }
4892 
4893 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
4894   assert (UseCompressedClassPointers, "should only be used for compressed headers");
4895   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4896   int klass_index = oop_recorder()->find_index(k);
4897   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
4898   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
4899 }
4900 
4901 void MacroAssembler::reinit_heapbase() {
4902   if (UseCompressedOops) {
4903     if (Universe::heap() != NULL) {
4904       if (CompressedOops::base() == NULL) {
4905         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
4906       } else {
4907         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
4908       }
4909     } else {
4910       movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
4911     }
4912   }
4913 }
4914 
4915 #endif // _LP64
4916 
4917 // C2 compiled method's prolog code.
4918 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
4919 
4920   // WARNING: Initial instruction MUST be 5 bytes or longer so that
4921   // NativeJump::patch_verified_entry will be able to patch out the entry
4922   // code safely. The push to verify stack depth is ok at 5 bytes,
4923   // the frame allocation can be either 3 or 6 bytes. So if we don't do
4924   // stack bang then we must use the 6 byte frame allocation even if
4925   // we have no frame. :-(
4926   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
4927 
4928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
4929   // Remove word for return addr
4930   framesize -= wordSize;
4931   stack_bang_size -= wordSize;
4932 
4933   // Calls to C2R adapters often do not accept exceptional returns.
4934   // We require that their callers must bang for them.  But be careful, because
4935   // some VM calls (such as call site linkage) can use several kilobytes of
4936   // stack.  But the stack safety zone should account for that.
4937   // See bugs 4446381, 4468289, 4497237.
4938   if (stack_bang_size > 0) {
4939     generate_stack_overflow_check(stack_bang_size);
4940 
4941     // We always push rbp, so that on return to interpreter rbp, will be
4942     // restored correctly and we can correct the stack.
4943     push(rbp);
4944     // Save caller's stack pointer into RBP if the frame pointer is preserved.
4945     if (PreserveFramePointer) {
4946       mov(rbp, rsp);
4947     }
4948     // Remove word for ebp
4949     framesize -= wordSize;
4950 
4951     // Create frame
4952     if (framesize) {
4953       subptr(rsp, framesize);
4954     }
4955   } else {
4956     // Create frame (force generation of a 4 byte immediate value)
4957     subptr_imm32(rsp, framesize);
4958 
4959     // Save RBP register now.
4960     framesize -= wordSize;
4961     movptr(Address(rsp, framesize), rbp);
4962     // Save caller's stack pointer into RBP if the frame pointer is preserved.
4963     if (PreserveFramePointer) {
4964       movptr(rbp, rsp);
4965       if (framesize > 0) {
4966         addptr(rbp, framesize);
4967       }
4968     }
4969   }
4970 
4971   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
4972     framesize -= wordSize;
4973     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4974   }
4975 
4976 #ifndef _LP64
4977   // If method sets FPU control word do it now
4978   if (fp_mode_24b) {
4979     fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
4980   }
4981   if (UseSSE >= 2 && VerifyFPU) {
4982     verify_FPU(0, "FPU stack must be clean on entry");
4983   }
4984 #endif
4985 
4986 #ifdef ASSERT
4987   if (VerifyStackAtCalls) {
4988     Label L;
4989     push(rax);
4990     mov(rax, rsp);
4991     andptr(rax, StackAlignmentInBytes-1);
4992     cmpptr(rax, StackAlignmentInBytes-wordSize);
4993     pop(rax);
4994     jcc(Assembler::equal, L);
4995     STOP("Stack is not properly aligned!");
4996     bind(L);
4997   }
4998 #endif
4999 
5000   if (!is_stub) {
5001     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5002     bs->nmethod_entry_barrier(this);
5003   }
5004 }
5005 
5006 #if COMPILER2_OR_JVMCI
5007 
5008 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5009 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5010   // cnt - number of qwords (8-byte words).
5011   // base - start address, qword aligned.
5012   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5013   bool use64byteVector = MaxVectorSize == 64 && AVX3Threshold == 0;
5014   if (use64byteVector) {
5015     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5016   } else if (MaxVectorSize >= 32) {
5017     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5018   } else {
5019     pxor(xtmp, xtmp);
5020   }
5021   jmp(L_zero_64_bytes);
5022 
5023   BIND(L_loop);
5024   if (MaxVectorSize >= 32) {
5025     fill64_avx(base, 0, xtmp, use64byteVector);
5026   } else {
5027     movdqu(Address(base,  0), xtmp);
5028     movdqu(Address(base, 16), xtmp);
5029     movdqu(Address(base, 32), xtmp);
5030     movdqu(Address(base, 48), xtmp);
5031   }
5032   addptr(base, 64);
5033 
5034   BIND(L_zero_64_bytes);
5035   subptr(cnt, 8);
5036   jccb(Assembler::greaterEqual, L_loop);
5037 
5038   // Copy trailing 64 bytes
5039   if (use64byteVector) {
5040     addptr(cnt, 8);
5041     jccb(Assembler::equal, L_end);
5042     fill64_masked_avx(3, base, 0, xtmp, mask, cnt, rtmp, true);
5043     jmp(L_end);
5044   } else {
5045     addptr(cnt, 4);
5046     jccb(Assembler::less, L_tail);
5047     if (MaxVectorSize >= 32) {
5048       vmovdqu(Address(base, 0), xtmp);
5049     } else {
5050       movdqu(Address(base,  0), xtmp);
5051       movdqu(Address(base, 16), xtmp);
5052     }
5053   }
5054   addptr(base, 32);
5055   subptr(cnt, 4);
5056 
5057   BIND(L_tail);
5058   addptr(cnt, 4);
5059   jccb(Assembler::lessEqual, L_end);
5060   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
5061     fill32_masked_avx(3, base, 0, xtmp, mask, cnt, rtmp);
5062   } else {
5063     decrement(cnt);
5064 
5065     BIND(L_sloop);
5066     movq(Address(base, 0), xtmp);
5067     addptr(base, 8);
5068     decrement(cnt);
5069     jccb(Assembler::greaterEqual, L_sloop);
5070   }
5071   BIND(L_end);
5072 }
5073 
5074 // Clearing constant sized memory using YMM/ZMM registers.
5075 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5076   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
5077   bool use64byteVector = MaxVectorSize > 32 && AVX3Threshold == 0;
5078 
5079   int vector64_count = (cnt & (~0x7)) >> 3;
5080   cnt = cnt & 0x7;
5081 
5082   // 64 byte initialization loop.
5083   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
5084   for (int i = 0; i < vector64_count; i++) {
5085     fill64_avx(base, i * 64, xtmp, use64byteVector);
5086   }
5087 
5088   // Clear remaining 64 byte tail.
5089   int disp = vector64_count * 64;
5090   if (cnt) {
5091     switch (cnt) {
5092       case 1:
5093         movq(Address(base, disp), xtmp);
5094         break;
5095       case 2:
5096         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_128bit);
5097         break;
5098       case 3:
5099         movl(rtmp, 0x7);
5100         kmovwl(mask, rtmp);
5101         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_256bit);
5102         break;
5103       case 4:
5104         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
5105         break;
5106       case 5:
5107         if (use64byteVector) {
5108           movl(rtmp, 0x1F);
5109           kmovwl(mask, rtmp);
5110           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
5111         } else {
5112           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
5113           movq(Address(base, disp + 32), xtmp);
5114         }
5115         break;
5116       case 6:
5117         if (use64byteVector) {
5118           movl(rtmp, 0x3F);
5119           kmovwl(mask, rtmp);
5120           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
5121         } else {
5122           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
5123           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, Assembler::AVX_128bit);
5124         }
5125         break;
5126       case 7:
5127         if (use64byteVector) {
5128           movl(rtmp, 0x7F);
5129           kmovwl(mask, rtmp);
5130           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
5131         } else {
5132           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
5133           movl(rtmp, 0x7);
5134           kmovwl(mask, rtmp);
5135           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, Assembler::AVX_256bit);
5136         }
5137         break;
5138       default:
5139         fatal("Unexpected length : %d\n",cnt);
5140         break;
5141     }
5142   }
5143 }
5144 
5145 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
5146                                bool is_large, KRegister mask) {
5147   // cnt      - number of qwords (8-byte words).
5148   // base     - start address, qword aligned.
5149   // is_large - if optimizers know cnt is larger than InitArrayShortSize
5150   assert(base==rdi, "base register must be edi for rep stos");
5151   assert(tmp==rax,   "tmp register must be eax for rep stos");
5152   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
5153   assert(InitArrayShortSize % BytesPerLong == 0,
5154     "InitArrayShortSize should be the multiple of BytesPerLong");
5155 
5156   Label DONE;
5157   if (!is_large || !UseXMMForObjInit) {
5158     xorptr(tmp, tmp);
5159   }
5160 
5161   if (!is_large) {
5162     Label LOOP, LONG;
5163     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
5164     jccb(Assembler::greater, LONG);
5165 
5166     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
5167 
5168     decrement(cnt);
5169     jccb(Assembler::negative, DONE); // Zero length
5170 
5171     // Use individual pointer-sized stores for small counts:
5172     BIND(LOOP);
5173     movptr(Address(base, cnt, Address::times_ptr), tmp);
5174     decrement(cnt);
5175     jccb(Assembler::greaterEqual, LOOP);
5176     jmpb(DONE);
5177 
5178     BIND(LONG);
5179   }
5180 
5181   // Use longer rep-prefixed ops for non-small counts:
5182   if (UseFastStosb) {
5183     shlptr(cnt, 3); // convert to number of bytes
5184     rep_stosb();
5185   } else if (UseXMMForObjInit) {
5186     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
5187   } else {
5188     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
5189     rep_stos();
5190   }
5191 
5192   BIND(DONE);
5193 }
5194 
5195 #endif //COMPILER2_OR_JVMCI
5196 
5197 
5198 void MacroAssembler::generate_fill(BasicType t, bool aligned,
5199                                    Register to, Register value, Register count,
5200                                    Register rtmp, XMMRegister xtmp) {
5201   ShortBranchVerifier sbv(this);
5202   assert_different_registers(to, value, count, rtmp);
5203   Label L_exit;
5204   Label L_fill_2_bytes, L_fill_4_bytes;
5205 
5206   int shift = -1;
5207   switch (t) {
5208     case T_BYTE:
5209       shift = 2;
5210       break;
5211     case T_SHORT:
5212       shift = 1;
5213       break;
5214     case T_INT:
5215       shift = 0;
5216       break;
5217     default: ShouldNotReachHere();
5218   }
5219 
5220   if (t == T_BYTE) {
5221     andl(value, 0xff);
5222     movl(rtmp, value);
5223     shll(rtmp, 8);
5224     orl(value, rtmp);
5225   }
5226   if (t == T_SHORT) {
5227     andl(value, 0xffff);
5228   }
5229   if (t == T_BYTE || t == T_SHORT) {
5230     movl(rtmp, value);
5231     shll(rtmp, 16);
5232     orl(value, rtmp);
5233   }
5234 
5235   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
5236   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
5237   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
5238     Label L_skip_align2;
5239     // align source address at 4 bytes address boundary
5240     if (t == T_BYTE) {
5241       Label L_skip_align1;
5242       // One byte misalignment happens only for byte arrays
5243       testptr(to, 1);
5244       jccb(Assembler::zero, L_skip_align1);
5245       movb(Address(to, 0), value);
5246       increment(to);
5247       decrement(count);
5248       BIND(L_skip_align1);
5249     }
5250     // Two bytes misalignment happens only for byte and short (char) arrays
5251     testptr(to, 2);
5252     jccb(Assembler::zero, L_skip_align2);
5253     movw(Address(to, 0), value);
5254     addptr(to, 2);
5255     subl(count, 1<<(shift-1));
5256     BIND(L_skip_align2);
5257   }
5258   if (UseSSE < 2) {
5259     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
5260     // Fill 32-byte chunks
5261     subl(count, 8 << shift);
5262     jcc(Assembler::less, L_check_fill_8_bytes);
5263     align(16);
5264 
5265     BIND(L_fill_32_bytes_loop);
5266 
5267     for (int i = 0; i < 32; i += 4) {
5268       movl(Address(to, i), value);
5269     }
5270 
5271     addptr(to, 32);
5272     subl(count, 8 << shift);
5273     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
5274     BIND(L_check_fill_8_bytes);
5275     addl(count, 8 << shift);
5276     jccb(Assembler::zero, L_exit);
5277     jmpb(L_fill_8_bytes);
5278 
5279     //
5280     // length is too short, just fill qwords
5281     //
5282     BIND(L_fill_8_bytes_loop);
5283     movl(Address(to, 0), value);
5284     movl(Address(to, 4), value);
5285     addptr(to, 8);
5286     BIND(L_fill_8_bytes);
5287     subl(count, 1 << (shift + 1));
5288     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
5289     // fall through to fill 4 bytes
5290   } else {
5291     Label L_fill_32_bytes;
5292     if (!UseUnalignedLoadStores) {
5293       // align to 8 bytes, we know we are 4 byte aligned to start
5294       testptr(to, 4);
5295       jccb(Assembler::zero, L_fill_32_bytes);
5296       movl(Address(to, 0), value);
5297       addptr(to, 4);
5298       subl(count, 1<<shift);
5299     }
5300     BIND(L_fill_32_bytes);
5301     {
5302       assert( UseSSE >= 2, "supported cpu only" );
5303       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
5304       movdl(xtmp, value);
5305       if (UseAVX >= 2 && UseUnalignedLoadStores) {
5306         Label L_check_fill_32_bytes;
5307         if (UseAVX > 2) {
5308           // Fill 64-byte chunks
5309           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
5310 
5311           // If number of bytes to fill < AVX3Threshold, perform fill using AVX2
5312           cmpl(count, AVX3Threshold);
5313           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
5314 
5315           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
5316 
5317           subl(count, 16 << shift);
5318           jccb(Assembler::less, L_check_fill_32_bytes);
5319           align(16);
5320 
5321           BIND(L_fill_64_bytes_loop_avx3);
5322           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
5323           addptr(to, 64);
5324           subl(count, 16 << shift);
5325           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
5326           jmpb(L_check_fill_32_bytes);
5327 
5328           BIND(L_check_fill_64_bytes_avx2);
5329         }
5330         // Fill 64-byte chunks
5331         Label L_fill_64_bytes_loop;
5332         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
5333 
5334         subl(count, 16 << shift);
5335         jcc(Assembler::less, L_check_fill_32_bytes);
5336         align(16);
5337 
5338         BIND(L_fill_64_bytes_loop);
5339         vmovdqu(Address(to, 0), xtmp);
5340         vmovdqu(Address(to, 32), xtmp);
5341         addptr(to, 64);
5342         subl(count, 16 << shift);
5343         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
5344 
5345         BIND(L_check_fill_32_bytes);
5346         addl(count, 8 << shift);
5347         jccb(Assembler::less, L_check_fill_8_bytes);
5348         vmovdqu(Address(to, 0), xtmp);
5349         addptr(to, 32);
5350         subl(count, 8 << shift);
5351 
5352         BIND(L_check_fill_8_bytes);
5353         // clean upper bits of YMM registers
5354         movdl(xtmp, value);
5355         pshufd(xtmp, xtmp, 0);
5356       } else {
5357         // Fill 32-byte chunks
5358         pshufd(xtmp, xtmp, 0);
5359 
5360         subl(count, 8 << shift);
5361         jcc(Assembler::less, L_check_fill_8_bytes);
5362         align(16);
5363 
5364         BIND(L_fill_32_bytes_loop);
5365 
5366         if (UseUnalignedLoadStores) {
5367           movdqu(Address(to, 0), xtmp);
5368           movdqu(Address(to, 16), xtmp);
5369         } else {
5370           movq(Address(to, 0), xtmp);
5371           movq(Address(to, 8), xtmp);
5372           movq(Address(to, 16), xtmp);
5373           movq(Address(to, 24), xtmp);
5374         }
5375 
5376         addptr(to, 32);
5377         subl(count, 8 << shift);
5378         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
5379 
5380         BIND(L_check_fill_8_bytes);
5381       }
5382       addl(count, 8 << shift);
5383       jccb(Assembler::zero, L_exit);
5384       jmpb(L_fill_8_bytes);
5385 
5386       //
5387       // length is too short, just fill qwords
5388       //
5389       BIND(L_fill_8_bytes_loop);
5390       movq(Address(to, 0), xtmp);
5391       addptr(to, 8);
5392       BIND(L_fill_8_bytes);
5393       subl(count, 1 << (shift + 1));
5394       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
5395     }
5396   }
5397   // fill trailing 4 bytes
5398   BIND(L_fill_4_bytes);
5399   testl(count, 1<<shift);
5400   jccb(Assembler::zero, L_fill_2_bytes);
5401   movl(Address(to, 0), value);
5402   if (t == T_BYTE || t == T_SHORT) {
5403     Label L_fill_byte;
5404     addptr(to, 4);
5405     BIND(L_fill_2_bytes);
5406     // fill trailing 2 bytes
5407     testl(count, 1<<(shift-1));
5408     jccb(Assembler::zero, L_fill_byte);
5409     movw(Address(to, 0), value);
5410     if (t == T_BYTE) {
5411       addptr(to, 2);
5412       BIND(L_fill_byte);
5413       // fill trailing byte
5414       testl(count, 1);
5415       jccb(Assembler::zero, L_exit);
5416       movb(Address(to, 0), value);
5417     } else {
5418       BIND(L_fill_byte);
5419     }
5420   } else {
5421     BIND(L_fill_2_bytes);
5422   }
5423   BIND(L_exit);
5424 }
5425 
5426 // encode char[] to byte[] in ISO_8859_1 or ASCII
5427    //@IntrinsicCandidate
5428    //private static int implEncodeISOArray(byte[] sa, int sp,
5429    //byte[] da, int dp, int len) {
5430    //  int i = 0;
5431    //  for (; i < len; i++) {
5432    //    char c = StringUTF16.getChar(sa, sp++);
5433    //    if (c > '\u00FF')
5434    //      break;
5435    //    da[dp++] = (byte)c;
5436    //  }
5437    //  return i;
5438    //}
5439    //
5440    //@IntrinsicCandidate
5441    //private static int implEncodeAsciiArray(char[] sa, int sp,
5442    //    byte[] da, int dp, int len) {
5443    //  int i = 0;
5444    //  for (; i < len; i++) {
5445    //    char c = sa[sp++];
5446    //    if (c >= '\u0080')
5447    //      break;
5448    //    da[dp++] = (byte)c;
5449    //  }
5450    //  return i;
5451    //}
5452 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
5453   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
5454   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
5455   Register tmp5, Register result, bool ascii) {
5456 
5457   // rsi: src
5458   // rdi: dst
5459   // rdx: len
5460   // rcx: tmp5
5461   // rax: result
5462   ShortBranchVerifier sbv(this);
5463   assert_different_registers(src, dst, len, tmp5, result);
5464   Label L_done, L_copy_1_char, L_copy_1_char_exit;
5465 
5466   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
5467   int short_mask = ascii ? 0xff80 : 0xff00;
5468 
5469   // set result
5470   xorl(result, result);
5471   // check for zero length
5472   testl(len, len);
5473   jcc(Assembler::zero, L_done);
5474 
5475   movl(result, len);
5476 
5477   // Setup pointers
5478   lea(src, Address(src, len, Address::times_2)); // char[]
5479   lea(dst, Address(dst, len, Address::times_1)); // byte[]
5480   negptr(len);
5481 
5482   if (UseSSE42Intrinsics || UseAVX >= 2) {
5483     Label L_copy_8_chars, L_copy_8_chars_exit;
5484     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
5485 
5486     if (UseAVX >= 2) {
5487       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
5488       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
5489       movdl(tmp1Reg, tmp5);
5490       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
5491       jmp(L_chars_32_check);
5492 
5493       bind(L_copy_32_chars);
5494       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
5495       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
5496       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
5497       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
5498       jccb(Assembler::notZero, L_copy_32_chars_exit);
5499       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
5500       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
5501       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
5502 
5503       bind(L_chars_32_check);
5504       addptr(len, 32);
5505       jcc(Assembler::lessEqual, L_copy_32_chars);
5506 
5507       bind(L_copy_32_chars_exit);
5508       subptr(len, 16);
5509       jccb(Assembler::greater, L_copy_16_chars_exit);
5510 
5511     } else if (UseSSE42Intrinsics) {
5512       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
5513       movdl(tmp1Reg, tmp5);
5514       pshufd(tmp1Reg, tmp1Reg, 0);
5515       jmpb(L_chars_16_check);
5516     }
5517 
5518     bind(L_copy_16_chars);
5519     if (UseAVX >= 2) {
5520       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
5521       vptest(tmp2Reg, tmp1Reg);
5522       jcc(Assembler::notZero, L_copy_16_chars_exit);
5523       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
5524       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
5525     } else {
5526       if (UseAVX > 0) {
5527         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
5528         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
5529         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
5530       } else {
5531         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
5532         por(tmp2Reg, tmp3Reg);
5533         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
5534         por(tmp2Reg, tmp4Reg);
5535       }
5536       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
5537       jccb(Assembler::notZero, L_copy_16_chars_exit);
5538       packuswb(tmp3Reg, tmp4Reg);
5539     }
5540     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
5541 
5542     bind(L_chars_16_check);
5543     addptr(len, 16);
5544     jcc(Assembler::lessEqual, L_copy_16_chars);
5545 
5546     bind(L_copy_16_chars_exit);
5547     if (UseAVX >= 2) {
5548       // clean upper bits of YMM registers
5549       vpxor(tmp2Reg, tmp2Reg);
5550       vpxor(tmp3Reg, tmp3Reg);
5551       vpxor(tmp4Reg, tmp4Reg);
5552       movdl(tmp1Reg, tmp5);
5553       pshufd(tmp1Reg, tmp1Reg, 0);
5554     }
5555     subptr(len, 8);
5556     jccb(Assembler::greater, L_copy_8_chars_exit);
5557 
5558     bind(L_copy_8_chars);
5559     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
5560     ptest(tmp3Reg, tmp1Reg);
5561     jccb(Assembler::notZero, L_copy_8_chars_exit);
5562     packuswb(tmp3Reg, tmp1Reg);
5563     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
5564     addptr(len, 8);
5565     jccb(Assembler::lessEqual, L_copy_8_chars);
5566 
5567     bind(L_copy_8_chars_exit);
5568     subptr(len, 8);
5569     jccb(Assembler::zero, L_done);
5570   }
5571 
5572   bind(L_copy_1_char);
5573   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
5574   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
5575   jccb(Assembler::notZero, L_copy_1_char_exit);
5576   movb(Address(dst, len, Address::times_1, 0), tmp5);
5577   addptr(len, 1);
5578   jccb(Assembler::less, L_copy_1_char);
5579 
5580   bind(L_copy_1_char_exit);
5581   addptr(result, len); // len is negative count of not processed elements
5582 
5583   bind(L_done);
5584 }
5585 
5586 #ifdef _LP64
5587 /**
5588  * Helper for multiply_to_len().
5589  */
5590 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
5591   addq(dest_lo, src1);
5592   adcq(dest_hi, 0);
5593   addq(dest_lo, src2);
5594   adcq(dest_hi, 0);
5595 }
5596 
5597 /**
5598  * Multiply 64 bit by 64 bit first loop.
5599  */
5600 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
5601                                            Register y, Register y_idx, Register z,
5602                                            Register carry, Register product,
5603                                            Register idx, Register kdx) {
5604   //
5605   //  jlong carry, x[], y[], z[];
5606   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
5607   //    huge_128 product = y[idx] * x[xstart] + carry;
5608   //    z[kdx] = (jlong)product;
5609   //    carry  = (jlong)(product >>> 64);
5610   //  }
5611   //  z[xstart] = carry;
5612   //
5613 
5614   Label L_first_loop, L_first_loop_exit;
5615   Label L_one_x, L_one_y, L_multiply;
5616 
5617   decrementl(xstart);
5618   jcc(Assembler::negative, L_one_x);
5619 
5620   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
5621   rorq(x_xstart, 32); // convert big-endian to little-endian
5622 
5623   bind(L_first_loop);
5624   decrementl(idx);
5625   jcc(Assembler::negative, L_first_loop_exit);
5626   decrementl(idx);
5627   jcc(Assembler::negative, L_one_y);
5628   movq(y_idx, Address(y, idx, Address::times_4,  0));
5629   rorq(y_idx, 32); // convert big-endian to little-endian
5630   bind(L_multiply);
5631   movq(product, x_xstart);
5632   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
5633   addq(product, carry);
5634   adcq(rdx, 0);
5635   subl(kdx, 2);
5636   movl(Address(z, kdx, Address::times_4,  4), product);
5637   shrq(product, 32);
5638   movl(Address(z, kdx, Address::times_4,  0), product);
5639   movq(carry, rdx);
5640   jmp(L_first_loop);
5641 
5642   bind(L_one_y);
5643   movl(y_idx, Address(y,  0));
5644   jmp(L_multiply);
5645 
5646   bind(L_one_x);
5647   movl(x_xstart, Address(x,  0));
5648   jmp(L_first_loop);
5649 
5650   bind(L_first_loop_exit);
5651 }
5652 
5653 /**
5654  * Multiply 64 bit by 64 bit and add 128 bit.
5655  */
5656 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
5657                                             Register yz_idx, Register idx,
5658                                             Register carry, Register product, int offset) {
5659   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
5660   //     z[kdx] = (jlong)product;
5661 
5662   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
5663   rorq(yz_idx, 32); // convert big-endian to little-endian
5664   movq(product, x_xstart);
5665   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
5666   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
5667   rorq(yz_idx, 32); // convert big-endian to little-endian
5668 
5669   add2_with_carry(rdx, product, carry, yz_idx);
5670 
5671   movl(Address(z, idx, Address::times_4,  offset+4), product);
5672   shrq(product, 32);
5673   movl(Address(z, idx, Address::times_4,  offset), product);
5674 
5675 }
5676 
5677 /**
5678  * Multiply 128 bit by 128 bit. Unrolled inner loop.
5679  */
5680 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
5681                                              Register yz_idx, Register idx, Register jdx,
5682                                              Register carry, Register product,
5683                                              Register carry2) {
5684   //   jlong carry, x[], y[], z[];
5685   //   int kdx = ystart+1;
5686   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
5687   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
5688   //     z[kdx+idx+1] = (jlong)product;
5689   //     jlong carry2  = (jlong)(product >>> 64);
5690   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
5691   //     z[kdx+idx] = (jlong)product;
5692   //     carry  = (jlong)(product >>> 64);
5693   //   }
5694   //   idx += 2;
5695   //   if (idx > 0) {
5696   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
5697   //     z[kdx+idx] = (jlong)product;
5698   //     carry  = (jlong)(product >>> 64);
5699   //   }
5700   //
5701 
5702   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
5703 
5704   movl(jdx, idx);
5705   andl(jdx, 0xFFFFFFFC);
5706   shrl(jdx, 2);
5707 
5708   bind(L_third_loop);
5709   subl(jdx, 1);
5710   jcc(Assembler::negative, L_third_loop_exit);
5711   subl(idx, 4);
5712 
5713   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
5714   movq(carry2, rdx);
5715 
5716   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
5717   movq(carry, rdx);
5718   jmp(L_third_loop);
5719 
5720   bind (L_third_loop_exit);
5721 
5722   andl (idx, 0x3);
5723   jcc(Assembler::zero, L_post_third_loop_done);
5724 
5725   Label L_check_1;
5726   subl(idx, 2);
5727   jcc(Assembler::negative, L_check_1);
5728 
5729   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
5730   movq(carry, rdx);
5731 
5732   bind (L_check_1);
5733   addl (idx, 0x2);
5734   andl (idx, 0x1);
5735   subl(idx, 1);
5736   jcc(Assembler::negative, L_post_third_loop_done);
5737 
5738   movl(yz_idx, Address(y, idx, Address::times_4,  0));
5739   movq(product, x_xstart);
5740   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
5741   movl(yz_idx, Address(z, idx, Address::times_4,  0));
5742 
5743   add2_with_carry(rdx, product, yz_idx, carry);
5744 
5745   movl(Address(z, idx, Address::times_4,  0), product);
5746   shrq(product, 32);
5747 
5748   shlq(rdx, 32);
5749   orq(product, rdx);
5750   movq(carry, product);
5751 
5752   bind(L_post_third_loop_done);
5753 }
5754 
5755 /**
5756  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
5757  *
5758  */
5759 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
5760                                                   Register carry, Register carry2,
5761                                                   Register idx, Register jdx,
5762                                                   Register yz_idx1, Register yz_idx2,
5763                                                   Register tmp, Register tmp3, Register tmp4) {
5764   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
5765 
5766   //   jlong carry, x[], y[], z[];
5767   //   int kdx = ystart+1;
5768   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
5769   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
5770   //     jlong carry2  = (jlong)(tmp3 >>> 64);
5771   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
5772   //     carry  = (jlong)(tmp4 >>> 64);
5773   //     z[kdx+idx+1] = (jlong)tmp3;
5774   //     z[kdx+idx] = (jlong)tmp4;
5775   //   }
5776   //   idx += 2;
5777   //   if (idx > 0) {
5778   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
5779   //     z[kdx+idx] = (jlong)yz_idx1;
5780   //     carry  = (jlong)(yz_idx1 >>> 64);
5781   //   }
5782   //
5783 
5784   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
5785 
5786   movl(jdx, idx);
5787   andl(jdx, 0xFFFFFFFC);
5788   shrl(jdx, 2);
5789 
5790   bind(L_third_loop);
5791   subl(jdx, 1);
5792   jcc(Assembler::negative, L_third_loop_exit);
5793   subl(idx, 4);
5794 
5795   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
5796   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
5797   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
5798   rorxq(yz_idx2, yz_idx2, 32);
5799 
5800   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
5801   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
5802 
5803   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
5804   rorxq(yz_idx1, yz_idx1, 32);
5805   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
5806   rorxq(yz_idx2, yz_idx2, 32);
5807 
5808   if (VM_Version::supports_adx()) {
5809     adcxq(tmp3, carry);
5810     adoxq(tmp3, yz_idx1);
5811 
5812     adcxq(tmp4, tmp);
5813     adoxq(tmp4, yz_idx2);
5814 
5815     movl(carry, 0); // does not affect flags
5816     adcxq(carry2, carry);
5817     adoxq(carry2, carry);
5818   } else {
5819     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
5820     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
5821   }
5822   movq(carry, carry2);
5823 
5824   movl(Address(z, idx, Address::times_4, 12), tmp3);
5825   shrq(tmp3, 32);
5826   movl(Address(z, idx, Address::times_4,  8), tmp3);
5827 
5828   movl(Address(z, idx, Address::times_4,  4), tmp4);
5829   shrq(tmp4, 32);
5830   movl(Address(z, idx, Address::times_4,  0), tmp4);
5831 
5832   jmp(L_third_loop);
5833 
5834   bind (L_third_loop_exit);
5835 
5836   andl (idx, 0x3);
5837   jcc(Assembler::zero, L_post_third_loop_done);
5838 
5839   Label L_check_1;
5840   subl(idx, 2);
5841   jcc(Assembler::negative, L_check_1);
5842 
5843   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
5844   rorxq(yz_idx1, yz_idx1, 32);
5845   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
5846   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
5847   rorxq(yz_idx2, yz_idx2, 32);
5848 
5849   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
5850 
5851   movl(Address(z, idx, Address::times_4,  4), tmp3);
5852   shrq(tmp3, 32);
5853   movl(Address(z, idx, Address::times_4,  0), tmp3);
5854   movq(carry, tmp4);
5855 
5856   bind (L_check_1);
5857   addl (idx, 0x2);
5858   andl (idx, 0x1);
5859   subl(idx, 1);
5860   jcc(Assembler::negative, L_post_third_loop_done);
5861   movl(tmp4, Address(y, idx, Address::times_4,  0));
5862   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
5863   movl(tmp4, Address(z, idx, Address::times_4,  0));
5864 
5865   add2_with_carry(carry2, tmp3, tmp4, carry);
5866 
5867   movl(Address(z, idx, Address::times_4,  0), tmp3);
5868   shrq(tmp3, 32);
5869 
5870   shlq(carry2, 32);
5871   orq(tmp3, carry2);
5872   movq(carry, tmp3);
5873 
5874   bind(L_post_third_loop_done);
5875 }
5876 
5877 /**
5878  * Code for BigInteger::multiplyToLen() instrinsic.
5879  *
5880  * rdi: x
5881  * rax: xlen
5882  * rsi: y
5883  * rcx: ylen
5884  * r8:  z
5885  * r11: zlen
5886  * r12: tmp1
5887  * r13: tmp2
5888  * r14: tmp3
5889  * r15: tmp4
5890  * rbx: tmp5
5891  *
5892  */
5893 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
5894                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
5895   ShortBranchVerifier sbv(this);
5896   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
5897 
5898   push(tmp1);
5899   push(tmp2);
5900   push(tmp3);
5901   push(tmp4);
5902   push(tmp5);
5903 
5904   push(xlen);
5905   push(zlen);
5906 
5907   const Register idx = tmp1;
5908   const Register kdx = tmp2;
5909   const Register xstart = tmp3;
5910 
5911   const Register y_idx = tmp4;
5912   const Register carry = tmp5;
5913   const Register product  = xlen;
5914   const Register x_xstart = zlen;  // reuse register
5915 
5916   // First Loop.
5917   //
5918   //  final static long LONG_MASK = 0xffffffffL;
5919   //  int xstart = xlen - 1;
5920   //  int ystart = ylen - 1;
5921   //  long carry = 0;
5922   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
5923   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
5924   //    z[kdx] = (int)product;
5925   //    carry = product >>> 32;
5926   //  }
5927   //  z[xstart] = (int)carry;
5928   //
5929 
5930   movl(idx, ylen);      // idx = ylen;
5931   movl(kdx, zlen);      // kdx = xlen+ylen;
5932   xorq(carry, carry);   // carry = 0;
5933 
5934   Label L_done;
5935 
5936   movl(xstart, xlen);
5937   decrementl(xstart);
5938   jcc(Assembler::negative, L_done);
5939 
5940   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
5941 
5942   Label L_second_loop;
5943   testl(kdx, kdx);
5944   jcc(Assembler::zero, L_second_loop);
5945 
5946   Label L_carry;
5947   subl(kdx, 1);
5948   jcc(Assembler::zero, L_carry);
5949 
5950   movl(Address(z, kdx, Address::times_4,  0), carry);
5951   shrq(carry, 32);
5952   subl(kdx, 1);
5953 
5954   bind(L_carry);
5955   movl(Address(z, kdx, Address::times_4,  0), carry);
5956 
5957   // Second and third (nested) loops.
5958   //
5959   // for (int i = xstart-1; i >= 0; i--) { // Second loop
5960   //   carry = 0;
5961   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
5962   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
5963   //                    (z[k] & LONG_MASK) + carry;
5964   //     z[k] = (int)product;
5965   //     carry = product >>> 32;
5966   //   }
5967   //   z[i] = (int)carry;
5968   // }
5969   //
5970   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
5971 
5972   const Register jdx = tmp1;
5973 
5974   bind(L_second_loop);
5975   xorl(carry, carry);    // carry = 0;
5976   movl(jdx, ylen);       // j = ystart+1
5977 
5978   subl(xstart, 1);       // i = xstart-1;
5979   jcc(Assembler::negative, L_done);
5980 
5981   push (z);
5982 
5983   Label L_last_x;
5984   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
5985   subl(xstart, 1);       // i = xstart-1;
5986   jcc(Assembler::negative, L_last_x);
5987 
5988   if (UseBMI2Instructions) {
5989     movq(rdx,  Address(x, xstart, Address::times_4,  0));
5990     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
5991   } else {
5992     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
5993     rorq(x_xstart, 32);  // convert big-endian to little-endian
5994   }
5995 
5996   Label L_third_loop_prologue;
5997   bind(L_third_loop_prologue);
5998 
5999   push (x);
6000   push (xstart);
6001   push (ylen);
6002 
6003 
6004   if (UseBMI2Instructions) {
6005     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
6006   } else { // !UseBMI2Instructions
6007     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
6008   }
6009 
6010   pop(ylen);
6011   pop(xlen);
6012   pop(x);
6013   pop(z);
6014 
6015   movl(tmp3, xlen);
6016   addl(tmp3, 1);
6017   movl(Address(z, tmp3, Address::times_4,  0), carry);
6018   subl(tmp3, 1);
6019   jccb(Assembler::negative, L_done);
6020 
6021   shrq(carry, 32);
6022   movl(Address(z, tmp3, Address::times_4,  0), carry);
6023   jmp(L_second_loop);
6024 
6025   // Next infrequent code is moved outside loops.
6026   bind(L_last_x);
6027   if (UseBMI2Instructions) {
6028     movl(rdx, Address(x,  0));
6029   } else {
6030     movl(x_xstart, Address(x,  0));
6031   }
6032   jmp(L_third_loop_prologue);
6033 
6034   bind(L_done);
6035 
6036   pop(zlen);
6037   pop(xlen);
6038 
6039   pop(tmp5);
6040   pop(tmp4);
6041   pop(tmp3);
6042   pop(tmp2);
6043   pop(tmp1);
6044 }
6045 
6046 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
6047   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
6048   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
6049   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
6050   Label VECTOR8_TAIL, VECTOR4_TAIL;
6051   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
6052   Label SAME_TILL_END, DONE;
6053   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
6054 
6055   //scale is in rcx in both Win64 and Unix
6056   ShortBranchVerifier sbv(this);
6057 
6058   shlq(length);
6059   xorq(result, result);
6060 
6061   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
6062       VM_Version::supports_avx512vlbw()) {
6063     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
6064 
6065     cmpq(length, 64);
6066     jcc(Assembler::less, VECTOR32_TAIL);
6067 
6068     movq(tmp1, length);
6069     andq(tmp1, 0x3F);      // tail count
6070     andq(length, ~(0x3F)); //vector count
6071 
6072     bind(VECTOR64_LOOP);
6073     // AVX512 code to compare 64 byte vectors.
6074     evmovdqub(rymm0, Address(obja, result), false, Assembler::AVX_512bit);
6075     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
6076     kortestql(k7, k7);
6077     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
6078     addq(result, 64);
6079     subq(length, 64);
6080     jccb(Assembler::notZero, VECTOR64_LOOP);
6081 
6082     //bind(VECTOR64_TAIL);
6083     testq(tmp1, tmp1);
6084     jcc(Assembler::zero, SAME_TILL_END);
6085 
6086     //bind(VECTOR64_TAIL);
6087     // AVX512 code to compare upto 63 byte vectors.
6088     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
6089     shlxq(tmp2, tmp2, tmp1);
6090     notq(tmp2);
6091     kmovql(k3, tmp2);
6092 
6093     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
6094     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
6095 
6096     ktestql(k7, k3);
6097     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
6098 
6099     bind(VECTOR64_NOT_EQUAL);
6100     kmovql(tmp1, k7);
6101     notq(tmp1);
6102     tzcntq(tmp1, tmp1);
6103     addq(result, tmp1);
6104     shrq(result);
6105     jmp(DONE);
6106     bind(VECTOR32_TAIL);
6107   }
6108 
6109   cmpq(length, 8);
6110   jcc(Assembler::equal, VECTOR8_LOOP);
6111   jcc(Assembler::less, VECTOR4_TAIL);
6112 
6113   if (UseAVX >= 2) {
6114     Label VECTOR16_TAIL, VECTOR32_LOOP;
6115 
6116     cmpq(length, 16);
6117     jcc(Assembler::equal, VECTOR16_LOOP);
6118     jcc(Assembler::less, VECTOR8_LOOP);
6119 
6120     cmpq(length, 32);
6121     jccb(Assembler::less, VECTOR16_TAIL);
6122 
6123     subq(length, 32);
6124     bind(VECTOR32_LOOP);
6125     vmovdqu(rymm0, Address(obja, result));
6126     vmovdqu(rymm1, Address(objb, result));
6127     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
6128     vptest(rymm2, rymm2);
6129     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
6130     addq(result, 32);
6131     subq(length, 32);
6132     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
6133     addq(length, 32);
6134     jcc(Assembler::equal, SAME_TILL_END);
6135     //falling through if less than 32 bytes left //close the branch here.
6136 
6137     bind(VECTOR16_TAIL);
6138     cmpq(length, 16);
6139     jccb(Assembler::less, VECTOR8_TAIL);
6140     bind(VECTOR16_LOOP);
6141     movdqu(rymm0, Address(obja, result));
6142     movdqu(rymm1, Address(objb, result));
6143     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
6144     ptest(rymm2, rymm2);
6145     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6146     addq(result, 16);
6147     subq(length, 16);
6148     jcc(Assembler::equal, SAME_TILL_END);
6149     //falling through if less than 16 bytes left
6150   } else {//regular intrinsics
6151 
6152     cmpq(length, 16);
6153     jccb(Assembler::less, VECTOR8_TAIL);
6154 
6155     subq(length, 16);
6156     bind(VECTOR16_LOOP);
6157     movdqu(rymm0, Address(obja, result));
6158     movdqu(rymm1, Address(objb, result));
6159     pxor(rymm0, rymm1);
6160     ptest(rymm0, rymm0);
6161     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6162     addq(result, 16);
6163     subq(length, 16);
6164     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
6165     addq(length, 16);
6166     jcc(Assembler::equal, SAME_TILL_END);
6167     //falling through if less than 16 bytes left
6168   }
6169 
6170   bind(VECTOR8_TAIL);
6171   cmpq(length, 8);
6172   jccb(Assembler::less, VECTOR4_TAIL);
6173   bind(VECTOR8_LOOP);
6174   movq(tmp1, Address(obja, result));
6175   movq(tmp2, Address(objb, result));
6176   xorq(tmp1, tmp2);
6177   testq(tmp1, tmp1);
6178   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
6179   addq(result, 8);
6180   subq(length, 8);
6181   jcc(Assembler::equal, SAME_TILL_END);
6182   //falling through if less than 8 bytes left
6183 
6184   bind(VECTOR4_TAIL);
6185   cmpq(length, 4);
6186   jccb(Assembler::less, BYTES_TAIL);
6187   bind(VECTOR4_LOOP);
6188   movl(tmp1, Address(obja, result));
6189   xorl(tmp1, Address(objb, result));
6190   testl(tmp1, tmp1);
6191   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
6192   addq(result, 4);
6193   subq(length, 4);
6194   jcc(Assembler::equal, SAME_TILL_END);
6195   //falling through if less than 4 bytes left
6196 
6197   bind(BYTES_TAIL);
6198   bind(BYTES_LOOP);
6199   load_unsigned_byte(tmp1, Address(obja, result));
6200   load_unsigned_byte(tmp2, Address(objb, result));
6201   xorl(tmp1, tmp2);
6202   testl(tmp1, tmp1);
6203   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6204   decq(length);
6205   jcc(Assembler::zero, SAME_TILL_END);
6206   incq(result);
6207   load_unsigned_byte(tmp1, Address(obja, result));
6208   load_unsigned_byte(tmp2, Address(objb, result));
6209   xorl(tmp1, tmp2);
6210   testl(tmp1, tmp1);
6211   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6212   decq(length);
6213   jcc(Assembler::zero, SAME_TILL_END);
6214   incq(result);
6215   load_unsigned_byte(tmp1, Address(obja, result));
6216   load_unsigned_byte(tmp2, Address(objb, result));
6217   xorl(tmp1, tmp2);
6218   testl(tmp1, tmp1);
6219   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6220   jmp(SAME_TILL_END);
6221 
6222   if (UseAVX >= 2) {
6223     bind(VECTOR32_NOT_EQUAL);
6224     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
6225     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
6226     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
6227     vpmovmskb(tmp1, rymm0);
6228     bsfq(tmp1, tmp1);
6229     addq(result, tmp1);
6230     shrq(result);
6231     jmp(DONE);
6232   }
6233 
6234   bind(VECTOR16_NOT_EQUAL);
6235   if (UseAVX >= 2) {
6236     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
6237     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
6238     pxor(rymm0, rymm2);
6239   } else {
6240     pcmpeqb(rymm2, rymm2);
6241     pxor(rymm0, rymm1);
6242     pcmpeqb(rymm0, rymm1);
6243     pxor(rymm0, rymm2);
6244   }
6245   pmovmskb(tmp1, rymm0);
6246   bsfq(tmp1, tmp1);
6247   addq(result, tmp1);
6248   shrq(result);
6249   jmpb(DONE);
6250 
6251   bind(VECTOR8_NOT_EQUAL);
6252   bind(VECTOR4_NOT_EQUAL);
6253   bsfq(tmp1, tmp1);
6254   shrq(tmp1, 3);
6255   addq(result, tmp1);
6256   bind(BYTES_NOT_EQUAL);
6257   shrq(result);
6258   jmpb(DONE);
6259 
6260   bind(SAME_TILL_END);
6261   mov64(result, -1);
6262 
6263   bind(DONE);
6264 }
6265 
6266 //Helper functions for square_to_len()
6267 
6268 /**
6269  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
6270  * Preserves x and z and modifies rest of the registers.
6271  */
6272 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6273   // Perform square and right shift by 1
6274   // Handle odd xlen case first, then for even xlen do the following
6275   // jlong carry = 0;
6276   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
6277   //     huge_128 product = x[j:j+1] * x[j:j+1];
6278   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
6279   //     z[i+2:i+3] = (jlong)(product >>> 1);
6280   //     carry = (jlong)product;
6281   // }
6282 
6283   xorq(tmp5, tmp5);     // carry
6284   xorq(rdxReg, rdxReg);
6285   xorl(tmp1, tmp1);     // index for x
6286   xorl(tmp4, tmp4);     // index for z
6287 
6288   Label L_first_loop, L_first_loop_exit;
6289 
6290   testl(xlen, 1);
6291   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
6292 
6293   // Square and right shift by 1 the odd element using 32 bit multiply
6294   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
6295   imulq(raxReg, raxReg);
6296   shrq(raxReg, 1);
6297   adcq(tmp5, 0);
6298   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
6299   incrementl(tmp1);
6300   addl(tmp4, 2);
6301 
6302   // Square and  right shift by 1 the rest using 64 bit multiply
6303   bind(L_first_loop);
6304   cmpptr(tmp1, xlen);
6305   jccb(Assembler::equal, L_first_loop_exit);
6306 
6307   // Square
6308   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
6309   rorq(raxReg, 32);    // convert big-endian to little-endian
6310   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
6311 
6312   // Right shift by 1 and save carry
6313   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
6314   rcrq(rdxReg, 1);
6315   rcrq(raxReg, 1);
6316   adcq(tmp5, 0);
6317 
6318   // Store result in z
6319   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
6320   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
6321 
6322   // Update indices for x and z
6323   addl(tmp1, 2);
6324   addl(tmp4, 4);
6325   jmp(L_first_loop);
6326 
6327   bind(L_first_loop_exit);
6328 }
6329 
6330 
6331 /**
6332  * Perform the following multiply add operation using BMI2 instructions
6333  * carry:sum = sum + op1*op2 + carry
6334  * op2 should be in rdx
6335  * op2 is preserved, all other registers are modified
6336  */
6337 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
6338   // assert op2 is rdx
6339   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
6340   addq(sum, carry);
6341   adcq(tmp2, 0);
6342   addq(sum, op1);
6343   adcq(tmp2, 0);
6344   movq(carry, tmp2);
6345 }
6346 
6347 /**
6348  * Perform the following multiply add operation:
6349  * carry:sum = sum + op1*op2 + carry
6350  * Preserves op1, op2 and modifies rest of registers
6351  */
6352 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
6353   // rdx:rax = op1 * op2
6354   movq(raxReg, op2);
6355   mulq(op1);
6356 
6357   //  rdx:rax = sum + carry + rdx:rax
6358   addq(sum, carry);
6359   adcq(rdxReg, 0);
6360   addq(sum, raxReg);
6361   adcq(rdxReg, 0);
6362 
6363   // carry:sum = rdx:sum
6364   movq(carry, rdxReg);
6365 }
6366 
6367 /**
6368  * Add 64 bit long carry into z[] with carry propogation.
6369  * Preserves z and carry register values and modifies rest of registers.
6370  *
6371  */
6372 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
6373   Label L_fourth_loop, L_fourth_loop_exit;
6374 
6375   movl(tmp1, 1);
6376   subl(zlen, 2);
6377   addq(Address(z, zlen, Address::times_4, 0), carry);
6378 
6379   bind(L_fourth_loop);
6380   jccb(Assembler::carryClear, L_fourth_loop_exit);
6381   subl(zlen, 2);
6382   jccb(Assembler::negative, L_fourth_loop_exit);
6383   addq(Address(z, zlen, Address::times_4, 0), tmp1);
6384   jmp(L_fourth_loop);
6385   bind(L_fourth_loop_exit);
6386 }
6387 
6388 /**
6389  * Shift z[] left by 1 bit.
6390  * Preserves x, len, z and zlen registers and modifies rest of the registers.
6391  *
6392  */
6393 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
6394 
6395   Label L_fifth_loop, L_fifth_loop_exit;
6396 
6397   // Fifth loop
6398   // Perform primitiveLeftShift(z, zlen, 1)
6399 
6400   const Register prev_carry = tmp1;
6401   const Register new_carry = tmp4;
6402   const Register value = tmp2;
6403   const Register zidx = tmp3;
6404 
6405   // int zidx, carry;
6406   // long value;
6407   // carry = 0;
6408   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
6409   //    (carry:value)  = (z[i] << 1) | carry ;
6410   //    z[i] = value;
6411   // }
6412 
6413   movl(zidx, zlen);
6414   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
6415 
6416   bind(L_fifth_loop);
6417   decl(zidx);  // Use decl to preserve carry flag
6418   decl(zidx);
6419   jccb(Assembler::negative, L_fifth_loop_exit);
6420 
6421   if (UseBMI2Instructions) {
6422      movq(value, Address(z, zidx, Address::times_4, 0));
6423      rclq(value, 1);
6424      rorxq(value, value, 32);
6425      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
6426   }
6427   else {
6428     // clear new_carry
6429     xorl(new_carry, new_carry);
6430 
6431     // Shift z[i] by 1, or in previous carry and save new carry
6432     movq(value, Address(z, zidx, Address::times_4, 0));
6433     shlq(value, 1);
6434     adcl(new_carry, 0);
6435 
6436     orq(value, prev_carry);
6437     rorq(value, 0x20);
6438     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
6439 
6440     // Set previous carry = new carry
6441     movl(prev_carry, new_carry);
6442   }
6443   jmp(L_fifth_loop);
6444 
6445   bind(L_fifth_loop_exit);
6446 }
6447 
6448 
6449 /**
6450  * Code for BigInteger::squareToLen() intrinsic
6451  *
6452  * rdi: x
6453  * rsi: len
6454  * r8:  z
6455  * rcx: zlen
6456  * r12: tmp1
6457  * r13: tmp2
6458  * r14: tmp3
6459  * r15: tmp4
6460  * rbx: tmp5
6461  *
6462  */
6463 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6464 
6465   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
6466   push(tmp1);
6467   push(tmp2);
6468   push(tmp3);
6469   push(tmp4);
6470   push(tmp5);
6471 
6472   // First loop
6473   // Store the squares, right shifted one bit (i.e., divided by 2).
6474   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
6475 
6476   // Add in off-diagonal sums.
6477   //
6478   // Second, third (nested) and fourth loops.
6479   // zlen +=2;
6480   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
6481   //    carry = 0;
6482   //    long op2 = x[xidx:xidx+1];
6483   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
6484   //       k -= 2;
6485   //       long op1 = x[j:j+1];
6486   //       long sum = z[k:k+1];
6487   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
6488   //       z[k:k+1] = sum;
6489   //    }
6490   //    add_one_64(z, k, carry, tmp_regs);
6491   // }
6492 
6493   const Register carry = tmp5;
6494   const Register sum = tmp3;
6495   const Register op1 = tmp4;
6496   Register op2 = tmp2;
6497 
6498   push(zlen);
6499   push(len);
6500   addl(zlen,2);
6501   bind(L_second_loop);
6502   xorq(carry, carry);
6503   subl(zlen, 4);
6504   subl(len, 2);
6505   push(zlen);
6506   push(len);
6507   cmpl(len, 0);
6508   jccb(Assembler::lessEqual, L_second_loop_exit);
6509 
6510   // Multiply an array by one 64 bit long.
6511   if (UseBMI2Instructions) {
6512     op2 = rdxReg;
6513     movq(op2, Address(x, len, Address::times_4,  0));
6514     rorxq(op2, op2, 32);
6515   }
6516   else {
6517     movq(op2, Address(x, len, Address::times_4,  0));
6518     rorq(op2, 32);
6519   }
6520 
6521   bind(L_third_loop);
6522   decrementl(len);
6523   jccb(Assembler::negative, L_third_loop_exit);
6524   decrementl(len);
6525   jccb(Assembler::negative, L_last_x);
6526 
6527   movq(op1, Address(x, len, Address::times_4,  0));
6528   rorq(op1, 32);
6529 
6530   bind(L_multiply);
6531   subl(zlen, 2);
6532   movq(sum, Address(z, zlen, Address::times_4,  0));
6533 
6534   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
6535   if (UseBMI2Instructions) {
6536     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
6537   }
6538   else {
6539     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
6540   }
6541 
6542   movq(Address(z, zlen, Address::times_4, 0), sum);
6543 
6544   jmp(L_third_loop);
6545   bind(L_third_loop_exit);
6546 
6547   // Fourth loop
6548   // Add 64 bit long carry into z with carry propogation.
6549   // Uses offsetted zlen.
6550   add_one_64(z, zlen, carry, tmp1);
6551 
6552   pop(len);
6553   pop(zlen);
6554   jmp(L_second_loop);
6555 
6556   // Next infrequent code is moved outside loops.
6557   bind(L_last_x);
6558   movl(op1, Address(x, 0));
6559   jmp(L_multiply);
6560 
6561   bind(L_second_loop_exit);
6562   pop(len);
6563   pop(zlen);
6564   pop(len);
6565   pop(zlen);
6566 
6567   // Fifth loop
6568   // Shift z left 1 bit.
6569   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
6570 
6571   // z[zlen-1] |= x[len-1] & 1;
6572   movl(tmp3, Address(x, len, Address::times_4, -4));
6573   andl(tmp3, 1);
6574   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
6575 
6576   pop(tmp5);
6577   pop(tmp4);
6578   pop(tmp3);
6579   pop(tmp2);
6580   pop(tmp1);
6581 }
6582 
6583 /**
6584  * Helper function for mul_add()
6585  * Multiply the in[] by int k and add to out[] starting at offset offs using
6586  * 128 bit by 32 bit multiply and return the carry in tmp5.
6587  * Only quad int aligned length of in[] is operated on in this function.
6588  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
6589  * This function preserves out, in and k registers.
6590  * len and offset point to the appropriate index in "in" & "out" correspondingly
6591  * tmp5 has the carry.
6592  * other registers are temporary and are modified.
6593  *
6594  */
6595 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
6596   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
6597   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6598 
6599   Label L_first_loop, L_first_loop_exit;
6600 
6601   movl(tmp1, len);
6602   shrl(tmp1, 2);
6603 
6604   bind(L_first_loop);
6605   subl(tmp1, 1);
6606   jccb(Assembler::negative, L_first_loop_exit);
6607 
6608   subl(len, 4);
6609   subl(offset, 4);
6610 
6611   Register op2 = tmp2;
6612   const Register sum = tmp3;
6613   const Register op1 = tmp4;
6614   const Register carry = tmp5;
6615 
6616   if (UseBMI2Instructions) {
6617     op2 = rdxReg;
6618   }
6619 
6620   movq(op1, Address(in, len, Address::times_4,  8));
6621   rorq(op1, 32);
6622   movq(sum, Address(out, offset, Address::times_4,  8));
6623   rorq(sum, 32);
6624   if (UseBMI2Instructions) {
6625     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
6626   }
6627   else {
6628     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
6629   }
6630   // Store back in big endian from little endian
6631   rorq(sum, 0x20);
6632   movq(Address(out, offset, Address::times_4,  8), sum);
6633 
6634   movq(op1, Address(in, len, Address::times_4,  0));
6635   rorq(op1, 32);
6636   movq(sum, Address(out, offset, Address::times_4,  0));
6637   rorq(sum, 32);
6638   if (UseBMI2Instructions) {
6639     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
6640   }
6641   else {
6642     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
6643   }
6644   // Store back in big endian from little endian
6645   rorq(sum, 0x20);
6646   movq(Address(out, offset, Address::times_4,  0), sum);
6647 
6648   jmp(L_first_loop);
6649   bind(L_first_loop_exit);
6650 }
6651 
6652 /**
6653  * Code for BigInteger::mulAdd() intrinsic
6654  *
6655  * rdi: out
6656  * rsi: in
6657  * r11: offs (out.length - offset)
6658  * rcx: len
6659  * r8:  k
6660  * r12: tmp1
6661  * r13: tmp2
6662  * r14: tmp3
6663  * r15: tmp4
6664  * rbx: tmp5
6665  * Multiply the in[] by word k and add to out[], return the carry in rax
6666  */
6667 void MacroAssembler::mul_add(Register out, Register in, Register offs,
6668    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
6669    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6670 
6671   Label L_carry, L_last_in, L_done;
6672 
6673 // carry = 0;
6674 // for (int j=len-1; j >= 0; j--) {
6675 //    long product = (in[j] & LONG_MASK) * kLong +
6676 //                   (out[offs] & LONG_MASK) + carry;
6677 //    out[offs--] = (int)product;
6678 //    carry = product >>> 32;
6679 // }
6680 //
6681   push(tmp1);
6682   push(tmp2);
6683   push(tmp3);
6684   push(tmp4);
6685   push(tmp5);
6686 
6687   Register op2 = tmp2;
6688   const Register sum = tmp3;
6689   const Register op1 = tmp4;
6690   const Register carry =  tmp5;
6691 
6692   if (UseBMI2Instructions) {
6693     op2 = rdxReg;
6694     movl(op2, k);
6695   }
6696   else {
6697     movl(op2, k);
6698   }
6699 
6700   xorq(carry, carry);
6701 
6702   //First loop
6703 
6704   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
6705   //The carry is in tmp5
6706   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
6707 
6708   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
6709   decrementl(len);
6710   jccb(Assembler::negative, L_carry);
6711   decrementl(len);
6712   jccb(Assembler::negative, L_last_in);
6713 
6714   movq(op1, Address(in, len, Address::times_4,  0));
6715   rorq(op1, 32);
6716 
6717   subl(offs, 2);
6718   movq(sum, Address(out, offs, Address::times_4,  0));
6719   rorq(sum, 32);
6720 
6721   if (UseBMI2Instructions) {
6722     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
6723   }
6724   else {
6725     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
6726   }
6727 
6728   // Store back in big endian from little endian
6729   rorq(sum, 0x20);
6730   movq(Address(out, offs, Address::times_4,  0), sum);
6731 
6732   testl(len, len);
6733   jccb(Assembler::zero, L_carry);
6734 
6735   //Multiply the last in[] entry, if any
6736   bind(L_last_in);
6737   movl(op1, Address(in, 0));
6738   movl(sum, Address(out, offs, Address::times_4,  -4));
6739 
6740   movl(raxReg, k);
6741   mull(op1); //tmp4 * eax -> edx:eax
6742   addl(sum, carry);
6743   adcl(rdxReg, 0);
6744   addl(sum, raxReg);
6745   adcl(rdxReg, 0);
6746   movl(carry, rdxReg);
6747 
6748   movl(Address(out, offs, Address::times_4,  -4), sum);
6749 
6750   bind(L_carry);
6751   //return tmp5/carry as carry in rax
6752   movl(rax, carry);
6753 
6754   bind(L_done);
6755   pop(tmp5);
6756   pop(tmp4);
6757   pop(tmp3);
6758   pop(tmp2);
6759   pop(tmp1);
6760 }
6761 #endif
6762 
6763 /**
6764  * Emits code to update CRC-32 with a byte value according to constants in table
6765  *
6766  * @param [in,out]crc   Register containing the crc.
6767  * @param [in]val       Register containing the byte to fold into the CRC.
6768  * @param [in]table     Register containing the table of crc constants.
6769  *
6770  * uint32_t crc;
6771  * val = crc_table[(val ^ crc) & 0xFF];
6772  * crc = val ^ (crc >> 8);
6773  *
6774  */
6775 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
6776   xorl(val, crc);
6777   andl(val, 0xFF);
6778   shrl(crc, 8); // unsigned shift
6779   xorl(crc, Address(table, val, Address::times_4, 0));
6780 }
6781 
6782 /**
6783  * Fold 128-bit data chunk
6784  */
6785 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
6786   if (UseAVX > 0) {
6787     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
6788     vpclmulldq(xcrc, xK, xcrc); // [63:0]
6789     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
6790     pxor(xcrc, xtmp);
6791   } else {
6792     movdqa(xtmp, xcrc);
6793     pclmulhdq(xtmp, xK);   // [123:64]
6794     pclmulldq(xcrc, xK);   // [63:0]
6795     pxor(xcrc, xtmp);
6796     movdqu(xtmp, Address(buf, offset));
6797     pxor(xcrc, xtmp);
6798   }
6799 }
6800 
6801 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
6802   if (UseAVX > 0) {
6803     vpclmulhdq(xtmp, xK, xcrc);
6804     vpclmulldq(xcrc, xK, xcrc);
6805     pxor(xcrc, xbuf);
6806     pxor(xcrc, xtmp);
6807   } else {
6808     movdqa(xtmp, xcrc);
6809     pclmulhdq(xtmp, xK);
6810     pclmulldq(xcrc, xK);
6811     pxor(xcrc, xbuf);
6812     pxor(xcrc, xtmp);
6813   }
6814 }
6815 
6816 /**
6817  * 8-bit folds to compute 32-bit CRC
6818  *
6819  * uint64_t xcrc;
6820  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
6821  */
6822 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
6823   movdl(tmp, xcrc);
6824   andl(tmp, 0xFF);
6825   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
6826   psrldq(xcrc, 1); // unsigned shift one byte
6827   pxor(xcrc, xtmp);
6828 }
6829 
6830 /**
6831  * uint32_t crc;
6832  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
6833  */
6834 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
6835   movl(tmp, crc);
6836   andl(tmp, 0xFF);
6837   shrl(crc, 8);
6838   xorl(crc, Address(table, tmp, Address::times_4, 0));
6839 }
6840 
6841 /**
6842  * @param crc   register containing existing CRC (32-bit)
6843  * @param buf   register pointing to input byte buffer (byte*)
6844  * @param len   register containing number of bytes
6845  * @param table register that will contain address of CRC table
6846  * @param tmp   scratch register
6847  */
6848 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
6849   assert_different_registers(crc, buf, len, table, tmp, rax);
6850 
6851   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
6852   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
6853 
6854   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
6855   // context for the registers used, where all instructions below are using 128-bit mode
6856   // On EVEX without VL and BW, these instructions will all be AVX.
6857   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
6858   notl(crc); // ~crc
6859   cmpl(len, 16);
6860   jcc(Assembler::less, L_tail);
6861 
6862   // Align buffer to 16 bytes
6863   movl(tmp, buf);
6864   andl(tmp, 0xF);
6865   jccb(Assembler::zero, L_aligned);
6866   subl(tmp,  16);
6867   addl(len, tmp);
6868 
6869   align(4);
6870   BIND(L_align_loop);
6871   movsbl(rax, Address(buf, 0)); // load byte with sign extension
6872   update_byte_crc32(crc, rax, table);
6873   increment(buf);
6874   incrementl(tmp);
6875   jccb(Assembler::less, L_align_loop);
6876 
6877   BIND(L_aligned);
6878   movl(tmp, len); // save
6879   shrl(len, 4);
6880   jcc(Assembler::zero, L_tail_restore);
6881 
6882   // Fold crc into first bytes of vector
6883   movdqa(xmm1, Address(buf, 0));
6884   movdl(rax, xmm1);
6885   xorl(crc, rax);
6886   if (VM_Version::supports_sse4_1()) {
6887     pinsrd(xmm1, crc, 0);
6888   } else {
6889     pinsrw(xmm1, crc, 0);
6890     shrl(crc, 16);
6891     pinsrw(xmm1, crc, 1);
6892   }
6893   addptr(buf, 16);
6894   subl(len, 4); // len > 0
6895   jcc(Assembler::less, L_fold_tail);
6896 
6897   movdqa(xmm2, Address(buf,  0));
6898   movdqa(xmm3, Address(buf, 16));
6899   movdqa(xmm4, Address(buf, 32));
6900   addptr(buf, 48);
6901   subl(len, 3);
6902   jcc(Assembler::lessEqual, L_fold_512b);
6903 
6904   // Fold total 512 bits of polynomial on each iteration,
6905   // 128 bits per each of 4 parallel streams.
6906   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
6907 
6908   align(32);
6909   BIND(L_fold_512b_loop);
6910   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
6911   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
6912   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
6913   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
6914   addptr(buf, 64);
6915   subl(len, 4);
6916   jcc(Assembler::greater, L_fold_512b_loop);
6917 
6918   // Fold 512 bits to 128 bits.
6919   BIND(L_fold_512b);
6920   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
6921   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
6922   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
6923   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
6924 
6925   // Fold the rest of 128 bits data chunks
6926   BIND(L_fold_tail);
6927   addl(len, 3);
6928   jccb(Assembler::lessEqual, L_fold_128b);
6929   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
6930 
6931   BIND(L_fold_tail_loop);
6932   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
6933   addptr(buf, 16);
6934   decrementl(len);
6935   jccb(Assembler::greater, L_fold_tail_loop);
6936 
6937   // Fold 128 bits in xmm1 down into 32 bits in crc register.
6938   BIND(L_fold_128b);
6939   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
6940   if (UseAVX > 0) {
6941     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
6942     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
6943     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
6944   } else {
6945     movdqa(xmm2, xmm0);
6946     pclmulqdq(xmm2, xmm1, 0x1);
6947     movdqa(xmm3, xmm0);
6948     pand(xmm3, xmm2);
6949     pclmulqdq(xmm0, xmm3, 0x1);
6950   }
6951   psrldq(xmm1, 8);
6952   psrldq(xmm2, 4);
6953   pxor(xmm0, xmm1);
6954   pxor(xmm0, xmm2);
6955 
6956   // 8 8-bit folds to compute 32-bit CRC.
6957   for (int j = 0; j < 4; j++) {
6958     fold_8bit_crc32(xmm0, table, xmm1, rax);
6959   }
6960   movdl(crc, xmm0); // mov 32 bits to general register
6961   for (int j = 0; j < 4; j++) {
6962     fold_8bit_crc32(crc, table, rax);
6963   }
6964 
6965   BIND(L_tail_restore);
6966   movl(len, tmp); // restore
6967   BIND(L_tail);
6968   andl(len, 0xf);
6969   jccb(Assembler::zero, L_exit);
6970 
6971   // Fold the rest of bytes
6972   align(4);
6973   BIND(L_tail_loop);
6974   movsbl(rax, Address(buf, 0)); // load byte with sign extension
6975   update_byte_crc32(crc, rax, table);
6976   increment(buf);
6977   decrementl(len);
6978   jccb(Assembler::greater, L_tail_loop);
6979 
6980   BIND(L_exit);
6981   notl(crc); // ~c
6982 }
6983 
6984 #ifdef _LP64
6985 // Helper function for AVX 512 CRC32
6986 // Fold 512-bit data chunks
6987 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
6988                                              Register pos, int offset) {
6989   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
6990   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
6991   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
6992   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
6993   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
6994 }
6995 
6996 // Helper function for AVX 512 CRC32
6997 // Compute CRC32 for < 256B buffers
6998 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register key, Register pos,
6999                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
7000                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
7001 
7002   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
7003   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
7004   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
7005 
7006   // check if there is enough buffer to be able to fold 16B at a time
7007   cmpl(len, 32);
7008   jcc(Assembler::less, L_less_than_32);
7009 
7010   // if there is, load the constants
7011   movdqu(xmm10, Address(key, 1 * 16));    //rk1 and rk2 in xmm10
7012   movdl(xmm0, crc);                        // get the initial crc value
7013   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7014   pxor(xmm7, xmm0);
7015 
7016   // update the buffer pointer
7017   addl(pos, 16);
7018   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
7019   subl(len, 32);
7020   jmp(L_16B_reduction_loop);
7021 
7022   bind(L_less_than_32);
7023   //mov initial crc to the return value. this is necessary for zero - length buffers.
7024   movl(rax, crc);
7025   testl(len, len);
7026   jcc(Assembler::equal, L_cleanup);
7027 
7028   movdl(xmm0, crc);                        //get the initial crc value
7029 
7030   cmpl(len, 16);
7031   jcc(Assembler::equal, L_exact_16_left);
7032   jcc(Assembler::less, L_less_than_16_left);
7033 
7034   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7035   pxor(xmm7, xmm0);                       //xor the initial crc value
7036   addl(pos, 16);
7037   subl(len, 16);
7038   movdqu(xmm10, Address(key, 1 * 16));    // rk1 and rk2 in xmm10
7039   jmp(L_get_last_two_xmms);
7040 
7041   bind(L_less_than_16_left);
7042   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
7043   pxor(xmm1, xmm1);
7044   movptr(tmp1, rsp);
7045   movdqu(Address(tmp1, 0 * 16), xmm1);
7046 
7047   cmpl(len, 4);
7048   jcc(Assembler::less, L_only_less_than_4);
7049 
7050   //backup the counter value
7051   movl(tmp2, len);
7052   cmpl(len, 8);
7053   jcc(Assembler::less, L_less_than_8_left);
7054 
7055   //load 8 Bytes
7056   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
7057   movq(Address(tmp1, 0 * 16), rax);
7058   addptr(tmp1, 8);
7059   subl(len, 8);
7060   addl(pos, 8);
7061 
7062   bind(L_less_than_8_left);
7063   cmpl(len, 4);
7064   jcc(Assembler::less, L_less_than_4_left);
7065 
7066   //load 4 Bytes
7067   movl(rax, Address(buf, pos, Address::times_1, 0));
7068   movl(Address(tmp1, 0 * 16), rax);
7069   addptr(tmp1, 4);
7070   subl(len, 4);
7071   addl(pos, 4);
7072 
7073   bind(L_less_than_4_left);
7074   cmpl(len, 2);
7075   jcc(Assembler::less, L_less_than_2_left);
7076 
7077   // load 2 Bytes
7078   movw(rax, Address(buf, pos, Address::times_1, 0));
7079   movl(Address(tmp1, 0 * 16), rax);
7080   addptr(tmp1, 2);
7081   subl(len, 2);
7082   addl(pos, 2);
7083 
7084   bind(L_less_than_2_left);
7085   cmpl(len, 1);
7086   jcc(Assembler::less, L_zero_left);
7087 
7088   // load 1 Byte
7089   movb(rax, Address(buf, pos, Address::times_1, 0));
7090   movb(Address(tmp1, 0 * 16), rax);
7091 
7092   bind(L_zero_left);
7093   movdqu(xmm7, Address(rsp, 0));
7094   pxor(xmm7, xmm0);                       //xor the initial crc value
7095 
7096   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7097   movdqu(xmm0, Address(rax, tmp2));
7098   pshufb(xmm7, xmm0);
7099   jmp(L_128_done);
7100 
7101   bind(L_exact_16_left);
7102   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
7103   pxor(xmm7, xmm0);                       //xor the initial crc value
7104   jmp(L_128_done);
7105 
7106   bind(L_only_less_than_4);
7107   cmpl(len, 3);
7108   jcc(Assembler::less, L_only_less_than_3);
7109 
7110   // load 3 Bytes
7111   movb(rax, Address(buf, pos, Address::times_1, 0));
7112   movb(Address(tmp1, 0), rax);
7113 
7114   movb(rax, Address(buf, pos, Address::times_1, 1));
7115   movb(Address(tmp1, 1), rax);
7116 
7117   movb(rax, Address(buf, pos, Address::times_1, 2));
7118   movb(Address(tmp1, 2), rax);
7119 
7120   movdqu(xmm7, Address(rsp, 0));
7121   pxor(xmm7, xmm0);                     //xor the initial crc value
7122 
7123   pslldq(xmm7, 0x5);
7124   jmp(L_barrett);
7125   bind(L_only_less_than_3);
7126   cmpl(len, 2);
7127   jcc(Assembler::less, L_only_less_than_2);
7128 
7129   // load 2 Bytes
7130   movb(rax, Address(buf, pos, Address::times_1, 0));
7131   movb(Address(tmp1, 0), rax);
7132 
7133   movb(rax, Address(buf, pos, Address::times_1, 1));
7134   movb(Address(tmp1, 1), rax);
7135 
7136   movdqu(xmm7, Address(rsp, 0));
7137   pxor(xmm7, xmm0);                     //xor the initial crc value
7138 
7139   pslldq(xmm7, 0x6);
7140   jmp(L_barrett);
7141 
7142   bind(L_only_less_than_2);
7143   //load 1 Byte
7144   movb(rax, Address(buf, pos, Address::times_1, 0));
7145   movb(Address(tmp1, 0), rax);
7146 
7147   movdqu(xmm7, Address(rsp, 0));
7148   pxor(xmm7, xmm0);                     //xor the initial crc value
7149 
7150   pslldq(xmm7, 0x7);
7151 }
7152 
7153 /**
7154 * Compute CRC32 using AVX512 instructions
7155 * param crc   register containing existing CRC (32-bit)
7156 * param buf   register pointing to input byte buffer (byte*)
7157 * param len   register containing number of bytes
7158 * param tmp1  scratch register
7159 * param tmp2  scratch register
7160 * return rax  result register
7161 */
7162 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register key, Register tmp1, Register tmp2) {
7163   assert_different_registers(crc, buf, len, key, tmp1, tmp2, rax);
7164 
7165   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
7166   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
7167   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
7168   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
7169   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
7170 
7171   const Register pos = r12;
7172   push(r12);
7173   subptr(rsp, 16 * 2 + 8);
7174 
7175   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
7176   // context for the registers used, where all instructions below are using 128-bit mode
7177   // On EVEX without VL and BW, these instructions will all be AVX.
7178   lea(key, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr()));
7179   notl(crc);
7180   movl(pos, 0);
7181 
7182   // check if smaller than 256B
7183   cmpl(len, 256);
7184   jcc(Assembler::less, L_less_than_256);
7185 
7186   // load the initial crc value
7187   movdl(xmm10, crc);
7188 
7189   // receive the initial 64B data, xor the initial crc value
7190   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
7191   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
7192   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
7193   evbroadcasti32x4(xmm10, Address(key, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
7194 
7195   subl(len, 256);
7196   cmpl(len, 256);
7197   jcc(Assembler::less, L_fold_128_B_loop);
7198 
7199   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
7200   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
7201   evbroadcasti32x4(xmm16, Address(key, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
7202   subl(len, 256);
7203 
7204   bind(L_fold_256_B_loop);
7205   addl(pos, 256);
7206   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
7207   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
7208   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
7209   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
7210 
7211   subl(len, 256);
7212   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
7213 
7214   // Fold 256 into 128
7215   addl(pos, 256);
7216   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
7217   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
7218   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
7219 
7220   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
7221   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
7222   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
7223 
7224   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
7225   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
7226 
7227   addl(len, 128);
7228   jmp(L_fold_128_B_register);
7229 
7230   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
7231   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
7232 
7233   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
7234   bind(L_fold_128_B_loop);
7235   addl(pos, 128);
7236   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
7237   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
7238 
7239   subl(len, 128);
7240   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
7241 
7242   addl(pos, 128);
7243 
7244   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
7245   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
7246   bind(L_fold_128_B_register);
7247   evmovdquq(xmm16, Address(key, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
7248   evmovdquq(xmm11, Address(key, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
7249   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
7250   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
7251   // save last that has no multiplicand
7252   vextracti64x2(xmm7, xmm4, 3);
7253 
7254   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
7255   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
7256   // Needed later in reduction loop
7257   movdqu(xmm10, Address(key, 1 * 16));
7258   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
7259   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
7260 
7261   // Swap 1,0,3,2 - 01 00 11 10
7262   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
7263   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
7264   vextracti128(xmm5, xmm8, 1);
7265   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
7266 
7267   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
7268   // instead of a cmp instruction, we use the negative flag with the jl instruction
7269   addl(len, 128 - 16);
7270   jcc(Assembler::less, L_final_reduction_for_128);
7271 
7272   bind(L_16B_reduction_loop);
7273   vpclmulqdq(xmm8, xmm7, xmm10, 0x1);
7274   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7275   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7276   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
7277   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7278   addl(pos, 16);
7279   subl(len, 16);
7280   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
7281 
7282   bind(L_final_reduction_for_128);
7283   addl(len, 16);
7284   jcc(Assembler::equal, L_128_done);
7285 
7286   bind(L_get_last_two_xmms);
7287   movdqu(xmm2, xmm7);
7288   addl(pos, len);
7289   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
7290   subl(pos, len);
7291 
7292   // get rid of the extra data that was loaded before
7293   // load the shift constant
7294   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7295   movdqu(xmm0, Address(rax, len));
7296   addl(rax, len);
7297 
7298   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7299   //Change mask to 512
7300   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
7301   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
7302 
7303   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
7304   vpclmulqdq(xmm8, xmm7, xmm10, 0x1);
7305   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7306   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7307   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
7308 
7309   bind(L_128_done);
7310   // compute crc of a 128-bit value
7311   movdqu(xmm10, Address(key, 3 * 16));
7312   movdqu(xmm0, xmm7);
7313 
7314   // 64b fold
7315   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
7316   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
7317   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7318 
7319   // 32b fold
7320   movdqu(xmm0, xmm7);
7321   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
7322   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7323   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7324   jmp(L_barrett);
7325 
7326   bind(L_less_than_256);
7327   kernel_crc32_avx512_256B(crc, buf, len, key, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
7328 
7329   //barrett reduction
7330   bind(L_barrett);
7331   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
7332   movdqu(xmm1, xmm7);
7333   movdqu(xmm2, xmm7);
7334   movdqu(xmm10, Address(key, 4 * 16));
7335 
7336   pclmulqdq(xmm7, xmm10, 0x0);
7337   pxor(xmm7, xmm2);
7338   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
7339   movdqu(xmm2, xmm7);
7340   pclmulqdq(xmm7, xmm10, 0x10);
7341   pxor(xmm7, xmm2);
7342   pxor(xmm7, xmm1);
7343   pextrd(crc, xmm7, 2);
7344 
7345   bind(L_cleanup);
7346   notl(crc); // ~c
7347   addptr(rsp, 16 * 2 + 8);
7348   pop(r12);
7349 }
7350 
7351 // S. Gueron / Information Processing Letters 112 (2012) 184
7352 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
7353 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
7354 // Output: the 64-bit carry-less product of B * CONST
7355 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
7356                                      Register tmp1, Register tmp2, Register tmp3) {
7357   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
7358   if (n > 0) {
7359     addq(tmp3, n * 256 * 8);
7360   }
7361   //    Q1 = TABLEExt[n][B & 0xFF];
7362   movl(tmp1, in);
7363   andl(tmp1, 0x000000FF);
7364   shll(tmp1, 3);
7365   addq(tmp1, tmp3);
7366   movq(tmp1, Address(tmp1, 0));
7367 
7368   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
7369   movl(tmp2, in);
7370   shrl(tmp2, 8);
7371   andl(tmp2, 0x000000FF);
7372   shll(tmp2, 3);
7373   addq(tmp2, tmp3);
7374   movq(tmp2, Address(tmp2, 0));
7375 
7376   shlq(tmp2, 8);
7377   xorq(tmp1, tmp2);
7378 
7379   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
7380   movl(tmp2, in);
7381   shrl(tmp2, 16);
7382   andl(tmp2, 0x000000FF);
7383   shll(tmp2, 3);
7384   addq(tmp2, tmp3);
7385   movq(tmp2, Address(tmp2, 0));
7386 
7387   shlq(tmp2, 16);
7388   xorq(tmp1, tmp2);
7389 
7390   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
7391   shrl(in, 24);
7392   andl(in, 0x000000FF);
7393   shll(in, 3);
7394   addq(in, tmp3);
7395   movq(in, Address(in, 0));
7396 
7397   shlq(in, 24);
7398   xorq(in, tmp1);
7399   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
7400 }
7401 
7402 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
7403                                       Register in_out,
7404                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
7405                                       XMMRegister w_xtmp2,
7406                                       Register tmp1,
7407                                       Register n_tmp2, Register n_tmp3) {
7408   if (is_pclmulqdq_supported) {
7409     movdl(w_xtmp1, in_out); // modified blindly
7410 
7411     movl(tmp1, const_or_pre_comp_const_index);
7412     movdl(w_xtmp2, tmp1);
7413     pclmulqdq(w_xtmp1, w_xtmp2, 0);
7414 
7415     movdq(in_out, w_xtmp1);
7416   } else {
7417     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
7418   }
7419 }
7420 
7421 // Recombination Alternative 2: No bit-reflections
7422 // T1 = (CRC_A * U1) << 1
7423 // T2 = (CRC_B * U2) << 1
7424 // C1 = T1 >> 32
7425 // C2 = T2 >> 32
7426 // T1 = T1 & 0xFFFFFFFF
7427 // T2 = T2 & 0xFFFFFFFF
7428 // T1 = CRC32(0, T1)
7429 // T2 = CRC32(0, T2)
7430 // C1 = C1 ^ T1
7431 // C2 = C2 ^ T2
7432 // CRC = C1 ^ C2 ^ CRC_C
7433 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
7434                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7435                                      Register tmp1, Register tmp2,
7436                                      Register n_tmp3) {
7437   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
7438   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
7439   shlq(in_out, 1);
7440   movl(tmp1, in_out);
7441   shrq(in_out, 32);
7442   xorl(tmp2, tmp2);
7443   crc32(tmp2, tmp1, 4);
7444   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
7445   shlq(in1, 1);
7446   movl(tmp1, in1);
7447   shrq(in1, 32);
7448   xorl(tmp2, tmp2);
7449   crc32(tmp2, tmp1, 4);
7450   xorl(in1, tmp2);
7451   xorl(in_out, in1);
7452   xorl(in_out, in2);
7453 }
7454 
7455 // Set N to predefined value
7456 // Subtract from a lenght of a buffer
7457 // execute in a loop:
7458 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
7459 // for i = 1 to N do
7460 //  CRC_A = CRC32(CRC_A, A[i])
7461 //  CRC_B = CRC32(CRC_B, B[i])
7462 //  CRC_C = CRC32(CRC_C, C[i])
7463 // end for
7464 // Recombine
7465 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
7466                                        Register in_out1, Register in_out2, Register in_out3,
7467                                        Register tmp1, Register tmp2, Register tmp3,
7468                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7469                                        Register tmp4, Register tmp5,
7470                                        Register n_tmp6) {
7471   Label L_processPartitions;
7472   Label L_processPartition;
7473   Label L_exit;
7474 
7475   bind(L_processPartitions);
7476   cmpl(in_out1, 3 * size);
7477   jcc(Assembler::less, L_exit);
7478     xorl(tmp1, tmp1);
7479     xorl(tmp2, tmp2);
7480     movq(tmp3, in_out2);
7481     addq(tmp3, size);
7482 
7483     bind(L_processPartition);
7484       crc32(in_out3, Address(in_out2, 0), 8);
7485       crc32(tmp1, Address(in_out2, size), 8);
7486       crc32(tmp2, Address(in_out2, size * 2), 8);
7487       addq(in_out2, 8);
7488       cmpq(in_out2, tmp3);
7489       jcc(Assembler::less, L_processPartition);
7490     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
7491             w_xtmp1, w_xtmp2, w_xtmp3,
7492             tmp4, tmp5,
7493             n_tmp6);
7494     addq(in_out2, 2 * size);
7495     subl(in_out1, 3 * size);
7496     jmp(L_processPartitions);
7497 
7498   bind(L_exit);
7499 }
7500 #else
7501 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
7502                                      Register tmp1, Register tmp2, Register tmp3,
7503                                      XMMRegister xtmp1, XMMRegister xtmp2) {
7504   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
7505   if (n > 0) {
7506     addl(tmp3, n * 256 * 8);
7507   }
7508   //    Q1 = TABLEExt[n][B & 0xFF];
7509   movl(tmp1, in_out);
7510   andl(tmp1, 0x000000FF);
7511   shll(tmp1, 3);
7512   addl(tmp1, tmp3);
7513   movq(xtmp1, Address(tmp1, 0));
7514 
7515   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
7516   movl(tmp2, in_out);
7517   shrl(tmp2, 8);
7518   andl(tmp2, 0x000000FF);
7519   shll(tmp2, 3);
7520   addl(tmp2, tmp3);
7521   movq(xtmp2, Address(tmp2, 0));
7522 
7523   psllq(xtmp2, 8);
7524   pxor(xtmp1, xtmp2);
7525 
7526   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
7527   movl(tmp2, in_out);
7528   shrl(tmp2, 16);
7529   andl(tmp2, 0x000000FF);
7530   shll(tmp2, 3);
7531   addl(tmp2, tmp3);
7532   movq(xtmp2, Address(tmp2, 0));
7533 
7534   psllq(xtmp2, 16);
7535   pxor(xtmp1, xtmp2);
7536 
7537   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
7538   shrl(in_out, 24);
7539   andl(in_out, 0x000000FF);
7540   shll(in_out, 3);
7541   addl(in_out, tmp3);
7542   movq(xtmp2, Address(in_out, 0));
7543 
7544   psllq(xtmp2, 24);
7545   pxor(xtmp1, xtmp2); // Result in CXMM
7546   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
7547 }
7548 
7549 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
7550                                       Register in_out,
7551                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
7552                                       XMMRegister w_xtmp2,
7553                                       Register tmp1,
7554                                       Register n_tmp2, Register n_tmp3) {
7555   if (is_pclmulqdq_supported) {
7556     movdl(w_xtmp1, in_out);
7557 
7558     movl(tmp1, const_or_pre_comp_const_index);
7559     movdl(w_xtmp2, tmp1);
7560     pclmulqdq(w_xtmp1, w_xtmp2, 0);
7561     // Keep result in XMM since GPR is 32 bit in length
7562   } else {
7563     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
7564   }
7565 }
7566 
7567 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
7568                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7569                                      Register tmp1, Register tmp2,
7570                                      Register n_tmp3) {
7571   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
7572   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
7573 
7574   psllq(w_xtmp1, 1);
7575   movdl(tmp1, w_xtmp1);
7576   psrlq(w_xtmp1, 32);
7577   movdl(in_out, w_xtmp1);
7578 
7579   xorl(tmp2, tmp2);
7580   crc32(tmp2, tmp1, 4);
7581   xorl(in_out, tmp2);
7582 
7583   psllq(w_xtmp2, 1);
7584   movdl(tmp1, w_xtmp2);
7585   psrlq(w_xtmp2, 32);
7586   movdl(in1, w_xtmp2);
7587 
7588   xorl(tmp2, tmp2);
7589   crc32(tmp2, tmp1, 4);
7590   xorl(in1, tmp2);
7591   xorl(in_out, in1);
7592   xorl(in_out, in2);
7593 }
7594 
7595 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
7596                                        Register in_out1, Register in_out2, Register in_out3,
7597                                        Register tmp1, Register tmp2, Register tmp3,
7598                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7599                                        Register tmp4, Register tmp5,
7600                                        Register n_tmp6) {
7601   Label L_processPartitions;
7602   Label L_processPartition;
7603   Label L_exit;
7604 
7605   bind(L_processPartitions);
7606   cmpl(in_out1, 3 * size);
7607   jcc(Assembler::less, L_exit);
7608     xorl(tmp1, tmp1);
7609     xorl(tmp2, tmp2);
7610     movl(tmp3, in_out2);
7611     addl(tmp3, size);
7612 
7613     bind(L_processPartition);
7614       crc32(in_out3, Address(in_out2, 0), 4);
7615       crc32(tmp1, Address(in_out2, size), 4);
7616       crc32(tmp2, Address(in_out2, size*2), 4);
7617       crc32(in_out3, Address(in_out2, 0+4), 4);
7618       crc32(tmp1, Address(in_out2, size+4), 4);
7619       crc32(tmp2, Address(in_out2, size*2+4), 4);
7620       addl(in_out2, 8);
7621       cmpl(in_out2, tmp3);
7622       jcc(Assembler::less, L_processPartition);
7623 
7624         push(tmp3);
7625         push(in_out1);
7626         push(in_out2);
7627         tmp4 = tmp3;
7628         tmp5 = in_out1;
7629         n_tmp6 = in_out2;
7630 
7631       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
7632             w_xtmp1, w_xtmp2, w_xtmp3,
7633             tmp4, tmp5,
7634             n_tmp6);
7635 
7636         pop(in_out2);
7637         pop(in_out1);
7638         pop(tmp3);
7639 
7640     addl(in_out2, 2 * size);
7641     subl(in_out1, 3 * size);
7642     jmp(L_processPartitions);
7643 
7644   bind(L_exit);
7645 }
7646 #endif //LP64
7647 
7648 #ifdef _LP64
7649 // Algorithm 2: Pipelined usage of the CRC32 instruction.
7650 // Input: A buffer I of L bytes.
7651 // Output: the CRC32C value of the buffer.
7652 // Notations:
7653 // Write L = 24N + r, with N = floor (L/24).
7654 // r = L mod 24 (0 <= r < 24).
7655 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
7656 // N quadwords, and R consists of r bytes.
7657 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
7658 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
7659 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
7660 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
7661 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
7662                                           Register tmp1, Register tmp2, Register tmp3,
7663                                           Register tmp4, Register tmp5, Register tmp6,
7664                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7665                                           bool is_pclmulqdq_supported) {
7666   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
7667   Label L_wordByWord;
7668   Label L_byteByByteProlog;
7669   Label L_byteByByte;
7670   Label L_exit;
7671 
7672   if (is_pclmulqdq_supported ) {
7673     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
7674     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
7675 
7676     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
7677     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
7678 
7679     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
7680     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
7681     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
7682   } else {
7683     const_or_pre_comp_const_index[0] = 1;
7684     const_or_pre_comp_const_index[1] = 0;
7685 
7686     const_or_pre_comp_const_index[2] = 3;
7687     const_or_pre_comp_const_index[3] = 2;
7688 
7689     const_or_pre_comp_const_index[4] = 5;
7690     const_or_pre_comp_const_index[5] = 4;
7691    }
7692   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
7693                     in2, in1, in_out,
7694                     tmp1, tmp2, tmp3,
7695                     w_xtmp1, w_xtmp2, w_xtmp3,
7696                     tmp4, tmp5,
7697                     tmp6);
7698   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
7699                     in2, in1, in_out,
7700                     tmp1, tmp2, tmp3,
7701                     w_xtmp1, w_xtmp2, w_xtmp3,
7702                     tmp4, tmp5,
7703                     tmp6);
7704   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
7705                     in2, in1, in_out,
7706                     tmp1, tmp2, tmp3,
7707                     w_xtmp1, w_xtmp2, w_xtmp3,
7708                     tmp4, tmp5,
7709                     tmp6);
7710   movl(tmp1, in2);
7711   andl(tmp1, 0x00000007);
7712   negl(tmp1);
7713   addl(tmp1, in2);
7714   addq(tmp1, in1);
7715 
7716   BIND(L_wordByWord);
7717   cmpq(in1, tmp1);
7718   jcc(Assembler::greaterEqual, L_byteByByteProlog);
7719     crc32(in_out, Address(in1, 0), 4);
7720     addq(in1, 4);
7721     jmp(L_wordByWord);
7722 
7723   BIND(L_byteByByteProlog);
7724   andl(in2, 0x00000007);
7725   movl(tmp2, 1);
7726 
7727   BIND(L_byteByByte);
7728   cmpl(tmp2, in2);
7729   jccb(Assembler::greater, L_exit);
7730     crc32(in_out, Address(in1, 0), 1);
7731     incq(in1);
7732     incl(tmp2);
7733     jmp(L_byteByByte);
7734 
7735   BIND(L_exit);
7736 }
7737 #else
7738 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
7739                                           Register tmp1, Register  tmp2, Register tmp3,
7740                                           Register tmp4, Register  tmp5, Register tmp6,
7741                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
7742                                           bool is_pclmulqdq_supported) {
7743   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
7744   Label L_wordByWord;
7745   Label L_byteByByteProlog;
7746   Label L_byteByByte;
7747   Label L_exit;
7748 
7749   if (is_pclmulqdq_supported) {
7750     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
7751     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
7752 
7753     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
7754     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
7755 
7756     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
7757     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
7758   } else {
7759     const_or_pre_comp_const_index[0] = 1;
7760     const_or_pre_comp_const_index[1] = 0;
7761 
7762     const_or_pre_comp_const_index[2] = 3;
7763     const_or_pre_comp_const_index[3] = 2;
7764 
7765     const_or_pre_comp_const_index[4] = 5;
7766     const_or_pre_comp_const_index[5] = 4;
7767   }
7768   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
7769                     in2, in1, in_out,
7770                     tmp1, tmp2, tmp3,
7771                     w_xtmp1, w_xtmp2, w_xtmp3,
7772                     tmp4, tmp5,
7773                     tmp6);
7774   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
7775                     in2, in1, in_out,
7776                     tmp1, tmp2, tmp3,
7777                     w_xtmp1, w_xtmp2, w_xtmp3,
7778                     tmp4, tmp5,
7779                     tmp6);
7780   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
7781                     in2, in1, in_out,
7782                     tmp1, tmp2, tmp3,
7783                     w_xtmp1, w_xtmp2, w_xtmp3,
7784                     tmp4, tmp5,
7785                     tmp6);
7786   movl(tmp1, in2);
7787   andl(tmp1, 0x00000007);
7788   negl(tmp1);
7789   addl(tmp1, in2);
7790   addl(tmp1, in1);
7791 
7792   BIND(L_wordByWord);
7793   cmpl(in1, tmp1);
7794   jcc(Assembler::greaterEqual, L_byteByByteProlog);
7795     crc32(in_out, Address(in1,0), 4);
7796     addl(in1, 4);
7797     jmp(L_wordByWord);
7798 
7799   BIND(L_byteByByteProlog);
7800   andl(in2, 0x00000007);
7801   movl(tmp2, 1);
7802 
7803   BIND(L_byteByByte);
7804   cmpl(tmp2, in2);
7805   jccb(Assembler::greater, L_exit);
7806     movb(tmp1, Address(in1, 0));
7807     crc32(in_out, tmp1, 1);
7808     incl(in1);
7809     incl(tmp2);
7810     jmp(L_byteByByte);
7811 
7812   BIND(L_exit);
7813 }
7814 #endif // LP64
7815 #undef BIND
7816 #undef BLOCK_COMMENT
7817 
7818 // Compress char[] array to byte[].
7819 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
7820 //   @IntrinsicCandidate
7821 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
7822 //     for (int i = 0; i < len; i++) {
7823 //       int c = src[srcOff++];
7824 //       if (c >>> 8 != 0) {
7825 //         return 0;
7826 //       }
7827 //       dst[dstOff++] = (byte)c;
7828 //     }
7829 //     return len;
7830 //   }
7831 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
7832   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
7833   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
7834   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
7835   Label copy_chars_loop, return_length, return_zero, done;
7836 
7837   // rsi: src
7838   // rdi: dst
7839   // rdx: len
7840   // rcx: tmp5
7841   // rax: result
7842 
7843   // rsi holds start addr of source char[] to be compressed
7844   // rdi holds start addr of destination byte[]
7845   // rdx holds length
7846 
7847   assert(len != result, "");
7848 
7849   // save length for return
7850   push(len);
7851 
7852   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
7853     VM_Version::supports_avx512vlbw() &&
7854     VM_Version::supports_bmi2()) {
7855 
7856     Label copy_32_loop, copy_loop_tail, below_threshold;
7857 
7858     // alignment
7859     Label post_alignment;
7860 
7861     // if length of the string is less than 16, handle it in an old fashioned way
7862     testl(len, -32);
7863     jcc(Assembler::zero, below_threshold);
7864 
7865     // First check whether a character is compressable ( <= 0xFF).
7866     // Create mask to test for Unicode chars inside zmm vector
7867     movl(result, 0x00FF);
7868     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
7869 
7870     testl(len, -64);
7871     jcc(Assembler::zero, post_alignment);
7872 
7873     movl(tmp5, dst);
7874     andl(tmp5, (32 - 1));
7875     negl(tmp5);
7876     andl(tmp5, (32 - 1));
7877 
7878     // bail out when there is nothing to be done
7879     testl(tmp5, 0xFFFFFFFF);
7880     jcc(Assembler::zero, post_alignment);
7881 
7882     // ~(~0 << len), where len is the # of remaining elements to process
7883     movl(result, 0xFFFFFFFF);
7884     shlxl(result, result, tmp5);
7885     notl(result);
7886     kmovdl(mask2, result);
7887 
7888     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
7889     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
7890     ktestd(mask1, mask2);
7891     jcc(Assembler::carryClear, return_zero);
7892 
7893     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
7894 
7895     addptr(src, tmp5);
7896     addptr(src, tmp5);
7897     addptr(dst, tmp5);
7898     subl(len, tmp5);
7899 
7900     bind(post_alignment);
7901     // end of alignment
7902 
7903     movl(tmp5, len);
7904     andl(tmp5, (32 - 1));    // tail count (in chars)
7905     andl(len, ~(32 - 1));    // vector count (in chars)
7906     jcc(Assembler::zero, copy_loop_tail);
7907 
7908     lea(src, Address(src, len, Address::times_2));
7909     lea(dst, Address(dst, len, Address::times_1));
7910     negptr(len);
7911 
7912     bind(copy_32_loop);
7913     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), /*merge*/ false, Assembler::AVX_512bit);
7914     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
7915     kortestdl(mask1, mask1);
7916     jcc(Assembler::carryClear, return_zero);
7917 
7918     // All elements in current processed chunk are valid candidates for
7919     // compression. Write a truncated byte elements to the memory.
7920     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
7921     addptr(len, 32);
7922     jcc(Assembler::notZero, copy_32_loop);
7923 
7924     bind(copy_loop_tail);
7925     // bail out when there is nothing to be done
7926     testl(tmp5, 0xFFFFFFFF);
7927     jcc(Assembler::zero, return_length);
7928 
7929     movl(len, tmp5);
7930 
7931     // ~(~0 << len), where len is the # of remaining elements to process
7932     movl(result, 0xFFFFFFFF);
7933     shlxl(result, result, len);
7934     notl(result);
7935 
7936     kmovdl(mask2, result);
7937 
7938     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
7939     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
7940     ktestd(mask1, mask2);
7941     jcc(Assembler::carryClear, return_zero);
7942 
7943     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
7944     jmp(return_length);
7945 
7946     bind(below_threshold);
7947   }
7948 
7949   if (UseSSE42Intrinsics) {
7950     Label copy_32_loop, copy_16, copy_tail;
7951 
7952     movl(result, len);
7953 
7954     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
7955 
7956     // vectored compression
7957     andl(len, 0xfffffff0);    // vector count (in chars)
7958     andl(result, 0x0000000f);    // tail count (in chars)
7959     testl(len, len);
7960     jcc(Assembler::zero, copy_16);
7961 
7962     // compress 16 chars per iter
7963     movdl(tmp1Reg, tmp5);
7964     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
7965     pxor(tmp4Reg, tmp4Reg);
7966 
7967     lea(src, Address(src, len, Address::times_2));
7968     lea(dst, Address(dst, len, Address::times_1));
7969     negptr(len);
7970 
7971     bind(copy_32_loop);
7972     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
7973     por(tmp4Reg, tmp2Reg);
7974     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
7975     por(tmp4Reg, tmp3Reg);
7976     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
7977     jcc(Assembler::notZero, return_zero);
7978     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
7979     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
7980     addptr(len, 16);
7981     jcc(Assembler::notZero, copy_32_loop);
7982 
7983     // compress next vector of 8 chars (if any)
7984     bind(copy_16);
7985     movl(len, result);
7986     andl(len, 0xfffffff8);    // vector count (in chars)
7987     andl(result, 0x00000007);    // tail count (in chars)
7988     testl(len, len);
7989     jccb(Assembler::zero, copy_tail);
7990 
7991     movdl(tmp1Reg, tmp5);
7992     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
7993     pxor(tmp3Reg, tmp3Reg);
7994 
7995     movdqu(tmp2Reg, Address(src, 0));
7996     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
7997     jccb(Assembler::notZero, return_zero);
7998     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
7999     movq(Address(dst, 0), tmp2Reg);
8000     addptr(src, 16);
8001     addptr(dst, 8);
8002 
8003     bind(copy_tail);
8004     movl(len, result);
8005   }
8006   // compress 1 char per iter
8007   testl(len, len);
8008   jccb(Assembler::zero, return_length);
8009   lea(src, Address(src, len, Address::times_2));
8010   lea(dst, Address(dst, len, Address::times_1));
8011   negptr(len);
8012 
8013   bind(copy_chars_loop);
8014   load_unsigned_short(result, Address(src, len, Address::times_2));
8015   testl(result, 0xff00);      // check if Unicode char
8016   jccb(Assembler::notZero, return_zero);
8017   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
8018   increment(len);
8019   jcc(Assembler::notZero, copy_chars_loop);
8020 
8021   // if compression succeeded, return length
8022   bind(return_length);
8023   pop(result);
8024   jmpb(done);
8025 
8026   // if compression failed, return 0
8027   bind(return_zero);
8028   xorl(result, result);
8029   addptr(rsp, wordSize);
8030 
8031   bind(done);
8032 }
8033 
8034 // Inflate byte[] array to char[].
8035 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
8036 //   @IntrinsicCandidate
8037 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
8038 //     for (int i = 0; i < len; i++) {
8039 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
8040 //     }
8041 //   }
8042 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
8043   XMMRegister tmp1, Register tmp2, KRegister mask) {
8044   Label copy_chars_loop, done, below_threshold, avx3_threshold;
8045   // rsi: src
8046   // rdi: dst
8047   // rdx: len
8048   // rcx: tmp2
8049 
8050   // rsi holds start addr of source byte[] to be inflated
8051   // rdi holds start addr of destination char[]
8052   // rdx holds length
8053   assert_different_registers(src, dst, len, tmp2);
8054   movl(tmp2, len);
8055   if ((UseAVX > 2) && // AVX512
8056     VM_Version::supports_avx512vlbw() &&
8057     VM_Version::supports_bmi2()) {
8058 
8059     Label copy_32_loop, copy_tail;
8060     Register tmp3_aliased = len;
8061 
8062     // if length of the string is less than 16, handle it in an old fashioned way
8063     testl(len, -16);
8064     jcc(Assembler::zero, below_threshold);
8065 
8066     testl(len, -1 * AVX3Threshold);
8067     jcc(Assembler::zero, avx3_threshold);
8068 
8069     // In order to use only one arithmetic operation for the main loop we use
8070     // this pre-calculation
8071     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
8072     andl(len, -32);     // vector count
8073     jccb(Assembler::zero, copy_tail);
8074 
8075     lea(src, Address(src, len, Address::times_1));
8076     lea(dst, Address(dst, len, Address::times_2));
8077     negptr(len);
8078 
8079 
8080     // inflate 32 chars per iter
8081     bind(copy_32_loop);
8082     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
8083     evmovdquw(Address(dst, len, Address::times_2), tmp1, /*merge*/ false, Assembler::AVX_512bit);
8084     addptr(len, 32);
8085     jcc(Assembler::notZero, copy_32_loop);
8086 
8087     bind(copy_tail);
8088     // bail out when there is nothing to be done
8089     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
8090     jcc(Assembler::zero, done);
8091 
8092     // ~(~0 << length), where length is the # of remaining elements to process
8093     movl(tmp3_aliased, -1);
8094     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
8095     notl(tmp3_aliased);
8096     kmovdl(mask, tmp3_aliased);
8097     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
8098     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
8099 
8100     jmp(done);
8101     bind(avx3_threshold);
8102   }
8103   if (UseSSE42Intrinsics) {
8104     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
8105 
8106     if (UseAVX > 1) {
8107       andl(tmp2, (16 - 1));
8108       andl(len, -16);
8109       jccb(Assembler::zero, copy_new_tail);
8110     } else {
8111       andl(tmp2, 0x00000007);   // tail count (in chars)
8112       andl(len, 0xfffffff8);    // vector count (in chars)
8113       jccb(Assembler::zero, copy_tail);
8114     }
8115 
8116     // vectored inflation
8117     lea(src, Address(src, len, Address::times_1));
8118     lea(dst, Address(dst, len, Address::times_2));
8119     negptr(len);
8120 
8121     if (UseAVX > 1) {
8122       bind(copy_16_loop);
8123       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
8124       vmovdqu(Address(dst, len, Address::times_2), tmp1);
8125       addptr(len, 16);
8126       jcc(Assembler::notZero, copy_16_loop);
8127 
8128       bind(below_threshold);
8129       bind(copy_new_tail);
8130       movl(len, tmp2);
8131       andl(tmp2, 0x00000007);
8132       andl(len, 0xFFFFFFF8);
8133       jccb(Assembler::zero, copy_tail);
8134 
8135       pmovzxbw(tmp1, Address(src, 0));
8136       movdqu(Address(dst, 0), tmp1);
8137       addptr(src, 8);
8138       addptr(dst, 2 * 8);
8139 
8140       jmp(copy_tail, true);
8141     }
8142 
8143     // inflate 8 chars per iter
8144     bind(copy_8_loop);
8145     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
8146     movdqu(Address(dst, len, Address::times_2), tmp1);
8147     addptr(len, 8);
8148     jcc(Assembler::notZero, copy_8_loop);
8149 
8150     bind(copy_tail);
8151     movl(len, tmp2);
8152 
8153     cmpl(len, 4);
8154     jccb(Assembler::less, copy_bytes);
8155 
8156     movdl(tmp1, Address(src, 0));  // load 4 byte chars
8157     pmovzxbw(tmp1, tmp1);
8158     movq(Address(dst, 0), tmp1);
8159     subptr(len, 4);
8160     addptr(src, 4);
8161     addptr(dst, 8);
8162 
8163     bind(copy_bytes);
8164   } else {
8165     bind(below_threshold);
8166   }
8167 
8168   testl(len, len);
8169   jccb(Assembler::zero, done);
8170   lea(src, Address(src, len, Address::times_1));
8171   lea(dst, Address(dst, len, Address::times_2));
8172   negptr(len);
8173 
8174   // inflate 1 char per iter
8175   bind(copy_chars_loop);
8176   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
8177   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
8178   increment(len);
8179   jcc(Assembler::notZero, copy_chars_loop);
8180 
8181   bind(done);
8182 }
8183 
8184 
8185 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len) {
8186   switch(type) {
8187     case T_BYTE:
8188     case T_BOOLEAN:
8189       evmovdqub(dst, kmask, src, false, vector_len);
8190       break;
8191     case T_CHAR:
8192     case T_SHORT:
8193       evmovdquw(dst, kmask, src, false, vector_len);
8194       break;
8195     case T_INT:
8196     case T_FLOAT:
8197       evmovdqul(dst, kmask, src, false, vector_len);
8198       break;
8199     case T_LONG:
8200     case T_DOUBLE:
8201       evmovdquq(dst, kmask, src, false, vector_len);
8202       break;
8203     default:
8204       fatal("Unexpected type argument %s", type2name(type));
8205       break;
8206   }
8207 }
8208 
8209 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len) {
8210   switch(type) {
8211     case T_BYTE:
8212     case T_BOOLEAN:
8213       evmovdqub(dst, kmask, src, true, vector_len);
8214       break;
8215     case T_CHAR:
8216     case T_SHORT:
8217       evmovdquw(dst, kmask, src, true, vector_len);
8218       break;
8219     case T_INT:
8220     case T_FLOAT:
8221       evmovdqul(dst, kmask, src, true, vector_len);
8222       break;
8223     case T_LONG:
8224     case T_DOUBLE:
8225       evmovdquq(dst, kmask, src, true, vector_len);
8226       break;
8227     default:
8228       fatal("Unexpected type argument %s", type2name(type));
8229       break;
8230   }
8231 }
8232 
8233 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
8234   switch(masklen) {
8235     case 2:
8236        knotbl(dst, src);
8237        movl(rtmp, 3);
8238        kmovbl(ktmp, rtmp);
8239        kandbl(dst, ktmp, dst);
8240        break;
8241     case 4:
8242        knotbl(dst, src);
8243        movl(rtmp, 15);
8244        kmovbl(ktmp, rtmp);
8245        kandbl(dst, ktmp, dst);
8246        break;
8247     case 8:
8248        knotbl(dst, src);
8249        break;
8250     case 16:
8251        knotwl(dst, src);
8252        break;
8253     case 32:
8254        knotdl(dst, src);
8255        break;
8256     case 64:
8257        knotql(dst, src);
8258        break;
8259     default:
8260       fatal("Unexpected vector length %d", masklen);
8261       break;
8262   }
8263 }
8264 
8265 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8266   switch(type) {
8267     case T_BOOLEAN:
8268     case T_BYTE:
8269        kandbl(dst, src1, src2);
8270        break;
8271     case T_CHAR:
8272     case T_SHORT:
8273        kandwl(dst, src1, src2);
8274        break;
8275     case T_INT:
8276     case T_FLOAT:
8277        kanddl(dst, src1, src2);
8278        break;
8279     case T_LONG:
8280     case T_DOUBLE:
8281        kandql(dst, src1, src2);
8282        break;
8283     default:
8284       fatal("Unexpected type argument %s", type2name(type));
8285       break;
8286   }
8287 }
8288 
8289 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8290   switch(type) {
8291     case T_BOOLEAN:
8292     case T_BYTE:
8293        korbl(dst, src1, src2);
8294        break;
8295     case T_CHAR:
8296     case T_SHORT:
8297        korwl(dst, src1, src2);
8298        break;
8299     case T_INT:
8300     case T_FLOAT:
8301        kordl(dst, src1, src2);
8302        break;
8303     case T_LONG:
8304     case T_DOUBLE:
8305        korql(dst, src1, src2);
8306        break;
8307     default:
8308       fatal("Unexpected type argument %s", type2name(type));
8309       break;
8310   }
8311 }
8312 
8313 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8314   switch(type) {
8315     case T_BOOLEAN:
8316     case T_BYTE:
8317        kxorbl(dst, src1, src2);
8318        break;
8319     case T_CHAR:
8320     case T_SHORT:
8321        kxorwl(dst, src1, src2);
8322        break;
8323     case T_INT:
8324     case T_FLOAT:
8325        kxordl(dst, src1, src2);
8326        break;
8327     case T_LONG:
8328     case T_DOUBLE:
8329        kxorql(dst, src1, src2);
8330        break;
8331     default:
8332       fatal("Unexpected type argument %s", type2name(type));
8333       break;
8334   }
8335 }
8336 
8337 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8338   switch(type) {
8339     case T_BOOLEAN:
8340     case T_BYTE:
8341       evpermb(dst, mask, nds, src, merge, vector_len); break;
8342     case T_CHAR:
8343     case T_SHORT:
8344       evpermw(dst, mask, nds, src, merge, vector_len); break;
8345     case T_INT:
8346     case T_FLOAT:
8347       evpermd(dst, mask, nds, src, merge, vector_len); break;
8348     case T_LONG:
8349     case T_DOUBLE:
8350       evpermq(dst, mask, nds, src, merge, vector_len); break;
8351     default:
8352       fatal("Unexpected type argument %s", type2name(type)); break;
8353   }
8354 }
8355 
8356 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8357   switch(type) {
8358     case T_BOOLEAN:
8359     case T_BYTE:
8360       evpermb(dst, mask, nds, src, merge, vector_len); break;
8361     case T_CHAR:
8362     case T_SHORT:
8363       evpermw(dst, mask, nds, src, merge, vector_len); break;
8364     case T_INT:
8365     case T_FLOAT:
8366       evpermd(dst, mask, nds, src, merge, vector_len); break;
8367     case T_LONG:
8368     case T_DOUBLE:
8369       evpermq(dst, mask, nds, src, merge, vector_len); break;
8370     default:
8371       fatal("Unexpected type argument %s", type2name(type)); break;
8372   }
8373 }
8374 
8375 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8376   switch(type) {
8377     case T_BYTE:
8378       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8379     case T_SHORT:
8380       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8381     case T_INT:
8382       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8383     case T_LONG:
8384       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8385     default:
8386       fatal("Unexpected type argument %s", type2name(type)); break;
8387   }
8388 }
8389 
8390 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8391   switch(type) {
8392     case T_BYTE:
8393       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8394     case T_SHORT:
8395       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8396     case T_INT:
8397       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8398     case T_LONG:
8399       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8400     default:
8401       fatal("Unexpected type argument %s", type2name(type)); break;
8402   }
8403 }
8404 
8405 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8406   switch(type) {
8407     case T_BYTE:
8408       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8409     case T_SHORT:
8410       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8411     case T_INT:
8412       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8413     case T_LONG:
8414       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8415     default:
8416       fatal("Unexpected type argument %s", type2name(type)); break;
8417   }
8418 }
8419 
8420 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8421   switch(type) {
8422     case T_BYTE:
8423       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8424     case T_SHORT:
8425       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8426     case T_INT:
8427       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8428     case T_LONG:
8429       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8430     default:
8431       fatal("Unexpected type argument %s", type2name(type)); break;
8432   }
8433 }
8434 
8435 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8436   switch(type) {
8437     case T_INT:
8438       evpxord(dst, mask, nds, src, merge, vector_len); break;
8439     case T_LONG:
8440       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8441     default:
8442       fatal("Unexpected type argument %s", type2name(type)); break;
8443   }
8444 }
8445 
8446 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8447   switch(type) {
8448     case T_INT:
8449       evpxord(dst, mask, nds, src, merge, vector_len); break;
8450     case T_LONG:
8451       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8452     default:
8453       fatal("Unexpected type argument %s", type2name(type)); break;
8454   }
8455 }
8456 
8457 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8458   switch(type) {
8459     case T_INT:
8460       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8461     case T_LONG:
8462       evporq(dst, mask, nds, src, merge, vector_len); break;
8463     default:
8464       fatal("Unexpected type argument %s", type2name(type)); break;
8465   }
8466 }
8467 
8468 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8469   switch(type) {
8470     case T_INT:
8471       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8472     case T_LONG:
8473       evporq(dst, mask, nds, src, merge, vector_len); break;
8474     default:
8475       fatal("Unexpected type argument %s", type2name(type)); break;
8476   }
8477 }
8478 
8479 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8480   switch(type) {
8481     case T_INT:
8482       evpandd(dst, mask, nds, src, merge, vector_len); break;
8483     case T_LONG:
8484       evpandq(dst, mask, nds, src, merge, vector_len); break;
8485     default:
8486       fatal("Unexpected type argument %s", type2name(type)); break;
8487   }
8488 }
8489 
8490 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8491   switch(type) {
8492     case T_INT:
8493       evpandd(dst, mask, nds, src, merge, vector_len); break;
8494     case T_LONG:
8495       evpandq(dst, mask, nds, src, merge, vector_len); break;
8496     default:
8497       fatal("Unexpected type argument %s", type2name(type)); break;
8498   }
8499 }
8500 
8501 void MacroAssembler::anytrue(Register dst, uint masklen, KRegister src1, KRegister src2) {
8502    masklen = masklen < 8 ? 8 : masklen;
8503    ktest(masklen, src1, src2);
8504    setb(Assembler::notZero, dst);
8505    movzbl(dst, dst);
8506 }
8507 
8508 void MacroAssembler::alltrue(Register dst, uint masklen, KRegister src1, KRegister src2, KRegister kscratch) {
8509   if (masklen < 8) {
8510     knotbl(kscratch, src2);
8511     kortestbl(src1, kscratch);
8512     setb(Assembler::carrySet, dst);
8513     movzbl(dst, dst);
8514   } else {
8515     ktest(masklen, src1, src2);
8516     setb(Assembler::carrySet, dst);
8517     movzbl(dst, dst);
8518   }
8519 }
8520 
8521 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
8522   switch(masklen) {
8523     case 8:
8524        kortestbl(src1, src2);
8525        break;
8526     case 16:
8527        kortestwl(src1, src2);
8528        break;
8529     case 32:
8530        kortestdl(src1, src2);
8531        break;
8532     case 64:
8533        kortestql(src1, src2);
8534        break;
8535     default:
8536       fatal("Unexpected mask length %d", masklen);
8537       break;
8538   }
8539 }
8540 
8541 
8542 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
8543   switch(masklen)  {
8544     case 8:
8545        ktestbl(src1, src2);
8546        break;
8547     case 16:
8548        ktestwl(src1, src2);
8549        break;
8550     case 32:
8551        ktestdl(src1, src2);
8552        break;
8553     case 64:
8554        ktestql(src1, src2);
8555        break;
8556     default:
8557       fatal("Unexpected mask length %d", masklen);
8558       break;
8559   }
8560 }
8561 
8562 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
8563   switch(type) {
8564     case T_INT:
8565       evprold(dst, mask, src, shift, merge, vlen_enc); break;
8566     case T_LONG:
8567       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
8568     default:
8569       fatal("Unexpected type argument %s", type2name(type)); break;
8570       break;
8571   }
8572 }
8573 
8574 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
8575   switch(type) {
8576     case T_INT:
8577       evprord(dst, mask, src, shift, merge, vlen_enc); break;
8578     case T_LONG:
8579       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
8580     default:
8581       fatal("Unexpected type argument %s", type2name(type)); break;
8582   }
8583 }
8584 
8585 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
8586   switch(type) {
8587     case T_INT:
8588       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
8589     case T_LONG:
8590       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
8591     default:
8592       fatal("Unexpected type argument %s", type2name(type)); break;
8593   }
8594 }
8595 
8596 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
8597   switch(type) {
8598     case T_INT:
8599       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
8600     case T_LONG:
8601       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
8602     default:
8603       fatal("Unexpected type argument %s", type2name(type)); break;
8604   }
8605 }
8606 #if COMPILER2_OR_JVMCI
8607 
8608 
8609 // Set memory operation for length "less than" 64 bytes.
8610 void MacroAssembler::fill64_masked_avx(uint shift, Register dst, int disp,
8611                                        XMMRegister xmm, KRegister mask, Register length,
8612                                        Register temp, bool use64byteVector) {
8613   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8614   assert(shift != 0, "shift value should be 1 (short),2(int) or 3(long)");
8615   BasicType type[] = { T_BYTE, T_SHORT,  T_INT,   T_LONG};
8616   if (!use64byteVector) {
8617     fill32_avx(dst, disp, xmm);
8618     subptr(length, 32 >> shift);
8619     fill32_masked_avx(shift, dst, disp + 32, xmm, mask, length, temp);
8620   } else {
8621     assert(MaxVectorSize == 64, "vector length != 64");
8622     movl(temp, 1);
8623     shlxl(temp, temp, length);
8624     subptr(temp, 1);
8625     kmovwl(mask, temp);
8626     evmovdqu(type[shift], mask, Address(dst, disp), xmm, Assembler::AVX_512bit);
8627   }
8628 }
8629 
8630 
8631 void MacroAssembler::fill32_masked_avx(uint shift, Register dst, int disp,
8632                                        XMMRegister xmm, KRegister mask, Register length,
8633                                        Register temp) {
8634   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8635   assert(shift != 0, "shift value should be 1 (short), 2(int) or 3(long)");
8636   BasicType type[] = { T_BYTE, T_SHORT,  T_INT,   T_LONG};
8637   movl(temp, 1);
8638   shlxl(temp, temp, length);
8639   subptr(temp, 1);
8640   kmovwl(mask, temp);
8641   evmovdqu(type[shift], mask, Address(dst, disp), xmm, Assembler::AVX_256bit);
8642 }
8643 
8644 
8645 void MacroAssembler::fill32_avx(Register dst, int disp, XMMRegister xmm) {
8646   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8647   vmovdqu(Address(dst, disp), xmm);
8648 }
8649 
8650 void MacroAssembler::fill64_avx(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
8651   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8652   BasicType type[] = {T_BYTE,  T_SHORT,  T_INT,   T_LONG};
8653   if (!use64byteVector) {
8654     fill32_avx(dst, disp, xmm);
8655     fill32_avx(dst, disp + 32, xmm);
8656   } else {
8657     evmovdquq(Address(dst, disp), xmm, Assembler::AVX_512bit);
8658   }
8659 }
8660 
8661 #endif //COMPILER2_OR_JVMCI
8662 
8663 
8664 #ifdef _LP64
8665 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
8666   Label done;
8667   cvttss2sil(dst, src);
8668   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
8669   cmpl(dst, 0x80000000); // float_sign_flip
8670   jccb(Assembler::notEqual, done);
8671   subptr(rsp, 8);
8672   movflt(Address(rsp, 0), src);
8673   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
8674   pop(dst);
8675   bind(done);
8676 }
8677 
8678 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
8679   Label done;
8680   cvttsd2sil(dst, src);
8681   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
8682   cmpl(dst, 0x80000000); // float_sign_flip
8683   jccb(Assembler::notEqual, done);
8684   subptr(rsp, 8);
8685   movdbl(Address(rsp, 0), src);
8686   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
8687   pop(dst);
8688   bind(done);
8689 }
8690 
8691 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
8692   Label done;
8693   cvttss2siq(dst, src);
8694   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
8695   jccb(Assembler::notEqual, done);
8696   subptr(rsp, 8);
8697   movflt(Address(rsp, 0), src);
8698   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
8699   pop(dst);
8700   bind(done);
8701 }
8702 
8703 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
8704   Label done;
8705   cvttsd2siq(dst, src);
8706   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
8707   jccb(Assembler::notEqual, done);
8708   subptr(rsp, 8);
8709   movdbl(Address(rsp, 0), src);
8710   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
8711   pop(dst);
8712   bind(done);
8713 }
8714 
8715 void MacroAssembler::cache_wb(Address line)
8716 {
8717   // 64 bit cpus always support clflush
8718   assert(VM_Version::supports_clflush(), "clflush should be available");
8719   bool optimized = VM_Version::supports_clflushopt();
8720   bool no_evict = VM_Version::supports_clwb();
8721 
8722   // prefer clwb (writeback without evict) otherwise
8723   // prefer clflushopt (potentially parallel writeback with evict)
8724   // otherwise fallback on clflush (serial writeback with evict)
8725 
8726   if (optimized) {
8727     if (no_evict) {
8728       clwb(line);
8729     } else {
8730       clflushopt(line);
8731     }
8732   } else {
8733     // no need for fence when using CLFLUSH
8734     clflush(line);
8735   }
8736 }
8737 
8738 void MacroAssembler::cache_wbsync(bool is_pre)
8739 {
8740   assert(VM_Version::supports_clflush(), "clflush should be available");
8741   bool optimized = VM_Version::supports_clflushopt();
8742   bool no_evict = VM_Version::supports_clwb();
8743 
8744   // pick the correct implementation
8745 
8746   if (!is_pre && (optimized || no_evict)) {
8747     // need an sfence for post flush when using clflushopt or clwb
8748     // otherwise no no need for any synchroniaztion
8749 
8750     sfence();
8751   }
8752 }
8753 
8754 #endif // _LP64
8755 
8756 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
8757   switch (cond) {
8758     // Note some conditions are synonyms for others
8759     case Assembler::zero:         return Assembler::notZero;
8760     case Assembler::notZero:      return Assembler::zero;
8761     case Assembler::less:         return Assembler::greaterEqual;
8762     case Assembler::lessEqual:    return Assembler::greater;
8763     case Assembler::greater:      return Assembler::lessEqual;
8764     case Assembler::greaterEqual: return Assembler::less;
8765     case Assembler::below:        return Assembler::aboveEqual;
8766     case Assembler::belowEqual:   return Assembler::above;
8767     case Assembler::above:        return Assembler::belowEqual;
8768     case Assembler::aboveEqual:   return Assembler::below;
8769     case Assembler::overflow:     return Assembler::noOverflow;
8770     case Assembler::noOverflow:   return Assembler::overflow;
8771     case Assembler::negative:     return Assembler::positive;
8772     case Assembler::positive:     return Assembler::negative;
8773     case Assembler::parity:       return Assembler::noParity;
8774     case Assembler::noParity:     return Assembler::parity;
8775   }
8776   ShouldNotReachHere(); return Assembler::overflow;
8777 }
8778 
8779 SkipIfEqual::SkipIfEqual(
8780     MacroAssembler* masm, const bool* flag_addr, bool value) {
8781   _masm = masm;
8782   _masm->cmp8(ExternalAddress((address)flag_addr), value);
8783   _masm->jcc(Assembler::equal, _label);
8784 }
8785 
8786 SkipIfEqual::~SkipIfEqual() {
8787   _masm->bind(_label);
8788 }
8789 
8790 // 32-bit Windows has its own fast-path implementation
8791 // of get_thread
8792 #if !defined(WIN32) || defined(_LP64)
8793 
8794 // This is simply a call to Thread::current()
8795 void MacroAssembler::get_thread(Register thread) {
8796   if (thread != rax) {
8797     push(rax);
8798   }
8799   LP64_ONLY(push(rdi);)
8800   LP64_ONLY(push(rsi);)
8801   push(rdx);
8802   push(rcx);
8803 #ifdef _LP64
8804   push(r8);
8805   push(r9);
8806   push(r10);
8807   push(r11);
8808 #endif
8809 
8810   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8811 
8812 #ifdef _LP64
8813   pop(r11);
8814   pop(r10);
8815   pop(r9);
8816   pop(r8);
8817 #endif
8818   pop(rcx);
8819   pop(rdx);
8820   LP64_ONLY(pop(rsi);)
8821   LP64_ONLY(pop(rdi);)
8822   if (thread != rax) {
8823     mov(thread, rax);
8824     pop(rax);
8825   }
8826 }
8827 
8828 #endif // !WIN32 || _LP64