1 /*
   2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/assembler.hpp"
  26 #include "asm/assembler.inline.hpp"
  27 #include "code/aotCodeCache.hpp"
  28 #include "code/compiledIC.hpp"
  29 #include "compiler/compiler_globals.hpp"
  30 #include "compiler/disassembler.hpp"
  31 #include "crc32c.h"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/barrierSetAssembler.hpp"
  34 #include "gc/shared/collectedHeap.inline.hpp"
  35 #include "gc/shared/tlab_globals.hpp"
  36 #include "interpreter/bytecodeHistogram.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interpreterRuntime.hpp"
  39 #include "jvm.h"
  40 #include "memory/resourceArea.hpp"
  41 #include "memory/universe.hpp"
  42 #include "oops/accessDecorators.hpp"
  43 #include "oops/compressedKlass.inline.hpp"
  44 #include "oops/compressedOops.inline.hpp"
  45 #include "oops/klass.inline.hpp"
  46 #include "prims/methodHandles.hpp"
  47 #include "runtime/continuation.hpp"
  48 #include "runtime/interfaceSupport.inline.hpp"
  49 #include "runtime/javaThread.hpp"
  50 #include "runtime/jniHandles.hpp"
  51 #include "runtime/objectMonitor.hpp"
  52 #include "runtime/os.hpp"
  53 #include "runtime/safepoint.hpp"
  54 #include "runtime/safepointMechanism.hpp"
  55 #include "runtime/sharedRuntime.hpp"
  56 #include "runtime/stubRoutines.hpp"
  57 #include "utilities/checkedCast.hpp"
  58 #include "utilities/macros.hpp"
  59 
  60 #ifdef PRODUCT
  61 #define BLOCK_COMMENT(str) /* nothing */
  62 #define STOP(error) stop(error)
  63 #else
  64 #define BLOCK_COMMENT(str) block_comment(str)
  65 #define STOP(error) block_comment(error); stop(error)
  66 #endif
  67 
  68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  69 
  70 #ifdef ASSERT
  71 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  72 #endif
  73 
  74 static const Assembler::Condition reverse[] = {
  75     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  76     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  77     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  78     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  79     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  80     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  81     Assembler::above          /* belowEqual    = 0x6 */ ,
  82     Assembler::belowEqual     /* above         = 0x7 */ ,
  83     Assembler::positive       /* negative      = 0x8 */ ,
  84     Assembler::negative       /* positive      = 0x9 */ ,
  85     Assembler::noParity       /* parity        = 0xa */ ,
  86     Assembler::parity         /* noParity      = 0xb */ ,
  87     Assembler::greaterEqual   /* less          = 0xc */ ,
  88     Assembler::less           /* greaterEqual  = 0xd */ ,
  89     Assembler::greater        /* lessEqual     = 0xe */ ,
  90     Assembler::lessEqual      /* greater       = 0xf, */
  91 
  92 };
  93 
  94 
  95 // Implementation of MacroAssembler
  96 
  97 Address MacroAssembler::as_Address(AddressLiteral adr) {
  98   // amd64 always does this as a pc-rel
  99   // we can be absolute or disp based on the instruction type
 100   // jmp/call are displacements others are absolute
 101   assert(!adr.is_lval(), "must be rval");
 102   assert(reachable(adr), "must be");
 103   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
 104 
 105 }
 106 
 107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
 108   AddressLiteral base = adr.base();
 109   lea(rscratch, base);
 110   Address index = adr.index();
 111   assert(index._disp == 0, "must not have disp"); // maybe it can?
 112   Address array(rscratch, index._index, index._scale, index._disp);
 113   return array;
 114 }
 115 
 116 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 117   Label L, E;
 118 
 119 #ifdef _WIN64
 120   // Windows always allocates space for it's register args
 121   assert(num_args <= 4, "only register arguments supported");
 122   subq(rsp,  frame::arg_reg_save_area_bytes);
 123 #endif
 124 
 125   // Align stack if necessary
 126   testl(rsp, 15);
 127   jcc(Assembler::zero, L);
 128 
 129   subq(rsp, 8);
 130   call(RuntimeAddress(entry_point));
 131   addq(rsp, 8);
 132   jmp(E);
 133 
 134   bind(L);
 135   call(RuntimeAddress(entry_point));
 136 
 137   bind(E);
 138 
 139 #ifdef _WIN64
 140   // restore stack pointer
 141   addq(rsp, frame::arg_reg_save_area_bytes);
 142 #endif
 143 }
 144 
 145 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
 146   assert(!src2.is_lval(), "should use cmpptr");
 147   assert(rscratch != noreg || always_reachable(src2), "missing");
 148 
 149   if (reachable(src2)) {
 150     cmpq(src1, as_Address(src2));
 151   } else {
 152     lea(rscratch, src2);
 153     Assembler::cmpq(src1, Address(rscratch, 0));
 154   }
 155 }
 156 
 157 int MacroAssembler::corrected_idivq(Register reg) {
 158   // Full implementation of Java ldiv and lrem; checks for special
 159   // case as described in JVM spec., p.243 & p.271.  The function
 160   // returns the (pc) offset of the idivl instruction - may be needed
 161   // for implicit exceptions.
 162   //
 163   //         normal case                           special case
 164   //
 165   // input : rax: dividend                         min_long
 166   //         reg: divisor   (may not be eax/edx)   -1
 167   //
 168   // output: rax: quotient  (= rax idiv reg)       min_long
 169   //         rdx: remainder (= rax irem reg)       0
 170   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 171   static const int64_t min_long = 0x8000000000000000;
 172   Label normal_case, special_case;
 173 
 174   // check for special case
 175   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
 176   jcc(Assembler::notEqual, normal_case);
 177   xorl(rdx, rdx); // prepare rdx for possible special case (where
 178                   // remainder = 0)
 179   cmpq(reg, -1);
 180   jcc(Assembler::equal, special_case);
 181 
 182   // handle normal case
 183   bind(normal_case);
 184   cdqq();
 185   int idivq_offset = offset();
 186   idivq(reg);
 187 
 188   // normal and special case exit
 189   bind(special_case);
 190 
 191   return idivq_offset;
 192 }
 193 
 194 void MacroAssembler::decrementq(Register reg, int value) {
 195   if (value == min_jint) { subq(reg, value); return; }
 196   if (value <  0) { incrementq(reg, -value); return; }
 197   if (value == 0) {                        ; return; }
 198   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 199   /* else */      { subq(reg, value)       ; return; }
 200 }
 201 
 202 void MacroAssembler::decrementq(Address dst, int value) {
 203   if (value == min_jint) { subq(dst, value); return; }
 204   if (value <  0) { incrementq(dst, -value); return; }
 205   if (value == 0) {                        ; return; }
 206   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 207   /* else */      { subq(dst, value)       ; return; }
 208 }
 209 
 210 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
 211   assert(rscratch != noreg || always_reachable(dst), "missing");
 212 
 213   if (reachable(dst)) {
 214     incrementq(as_Address(dst));
 215   } else {
 216     lea(rscratch, dst);
 217     incrementq(Address(rscratch, 0));
 218   }
 219 }
 220 
 221 void MacroAssembler::incrementq(Register reg, int value) {
 222   if (value == min_jint) { addq(reg, value); return; }
 223   if (value <  0) { decrementq(reg, -value); return; }
 224   if (value == 0) {                        ; return; }
 225   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 226   /* else */      { addq(reg, value)       ; return; }
 227 }
 228 
 229 void MacroAssembler::incrementq(Address dst, int value) {
 230   if (value == min_jint) { addq(dst, value); return; }
 231   if (value <  0) { decrementq(dst, -value); return; }
 232   if (value == 0) {                        ; return; }
 233   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 234   /* else */      { addq(dst, value)       ; return; }
 235 }
 236 
 237 // 32bit can do a case table jump in one instruction but we no longer allow the base
 238 // to be installed in the Address class
 239 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
 240   lea(rscratch, entry.base());
 241   Address dispatch = entry.index();
 242   assert(dispatch._base == noreg, "must be");
 243   dispatch._base = rscratch;
 244   jmp(dispatch);
 245 }
 246 
 247 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 248   ShouldNotReachHere(); // 64bit doesn't use two regs
 249   cmpq(x_lo, y_lo);
 250 }
 251 
 252 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 253   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 254 }
 255 
 256 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
 257   lea(rscratch, adr);
 258   movptr(dst, rscratch);
 259 }
 260 
 261 void MacroAssembler::leave() {
 262   // %%% is this really better? Why not on 32bit too?
 263   emit_int8((unsigned char)0xC9); // LEAVE
 264 }
 265 
 266 void MacroAssembler::lneg(Register hi, Register lo) {
 267   ShouldNotReachHere(); // 64bit doesn't use two regs
 268   negq(lo);
 269 }
 270 
 271 void MacroAssembler::movoop(Register dst, jobject obj) {
 272   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 273 }
 274 
 275 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
 276   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 277   movq(dst, rscratch);
 278 }
 279 
 280 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 281   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 282 }
 283 
 284 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
 285   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 286   movq(dst, rscratch);
 287 }
 288 
 289 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 290   if (src.is_lval()) {
 291     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 292   } else {
 293     if (reachable(src)) {
 294       movq(dst, as_Address(src));
 295     } else {
 296       lea(dst, src);
 297       movq(dst, Address(dst, 0));
 298     }
 299   }
 300 }
 301 
 302 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
 303   movq(as_Address(dst, rscratch), src);
 304 }
 305 
 306 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 307   movq(dst, as_Address(src, dst /*rscratch*/));
 308 }
 309 
 310 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 311 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
 312   if (is_simm32(src)) {
 313     movptr(dst, checked_cast<int32_t>(src));
 314   } else {
 315     mov64(rscratch, src);
 316     movq(dst, rscratch);
 317   }
 318 }
 319 
 320 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
 321   movoop(rscratch, obj);
 322   push(rscratch);
 323 }
 324 
 325 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
 326   mov_metadata(rscratch, obj);
 327   push(rscratch);
 328 }
 329 
 330 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
 331   lea(rscratch, src);
 332   if (src.is_lval()) {
 333     push(rscratch);
 334   } else {
 335     pushq(Address(rscratch, 0));
 336   }
 337 }
 338 
 339 static void pass_arg0(MacroAssembler* masm, Register arg) {
 340   if (c_rarg0 != arg ) {
 341     masm->mov(c_rarg0, arg);
 342   }
 343 }
 344 
 345 static void pass_arg1(MacroAssembler* masm, Register arg) {
 346   if (c_rarg1 != arg ) {
 347     masm->mov(c_rarg1, arg);
 348   }
 349 }
 350 
 351 static void pass_arg2(MacroAssembler* masm, Register arg) {
 352   if (c_rarg2 != arg ) {
 353     masm->mov(c_rarg2, arg);
 354   }
 355 }
 356 
 357 static void pass_arg3(MacroAssembler* masm, Register arg) {
 358   if (c_rarg3 != arg ) {
 359     masm->mov(c_rarg3, arg);
 360   }
 361 }
 362 
 363 void MacroAssembler::stop(const char* msg) {
 364   if (ShowMessageBoxOnError) {
 365     address rip = pc();
 366     pusha(); // get regs on stack
 367     lea(c_rarg1, InternalAddress(rip));
 368     movq(c_rarg2, rsp); // pass pointer to regs array
 369   }
 370   // Skip AOT caching C strings in scratch buffer.
 371   const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
 372   lea(c_rarg0, ExternalAddress((address) str));
 373   andq(rsp, -16); // align stack as required by ABI
 374   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 375   hlt();
 376 }
 377 
 378 void MacroAssembler::warn(const char* msg) {
 379   push(rbp);
 380   movq(rbp, rsp);
 381   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 382   push_CPU_state();   // keeps alignment at 16 bytes
 383 
 384 #ifdef _WIN64
 385   // Windows always allocates space for its register args
 386   subq(rsp,  frame::arg_reg_save_area_bytes);
 387 #endif
 388   lea(c_rarg0, ExternalAddress((address) msg));
 389   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 390 
 391 #ifdef _WIN64
 392   // restore stack pointer
 393   addq(rsp, frame::arg_reg_save_area_bytes);
 394 #endif
 395   pop_CPU_state();
 396   mov(rsp, rbp);
 397   pop(rbp);
 398 }
 399 
 400 void MacroAssembler::print_state() {
 401   address rip = pc();
 402   pusha();            // get regs on stack
 403   push(rbp);
 404   movq(rbp, rsp);
 405   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 406   push_CPU_state();   // keeps alignment at 16 bytes
 407 
 408   lea(c_rarg0, InternalAddress(rip));
 409   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 410   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 411 
 412   pop_CPU_state();
 413   mov(rsp, rbp);
 414   pop(rbp);
 415   popa();
 416 }
 417 
 418 #ifndef PRODUCT
 419 extern "C" void findpc(intptr_t x);
 420 #endif
 421 
 422 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 423   // In order to get locks to work, we need to fake a in_VM state
 424   if (ShowMessageBoxOnError) {
 425     JavaThread* thread = JavaThread::current();
 426     JavaThreadState saved_state = thread->thread_state();
 427     thread->set_thread_state(_thread_in_vm);
 428 #ifndef PRODUCT
 429     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 430       ttyLocker ttyl;
 431       BytecodeCounter::print();
 432     }
 433 #endif
 434     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 435     // XXX correct this offset for amd64
 436     // This is the value of eip which points to where verify_oop will return.
 437     if (os::message_box(msg, "Execution stopped, print registers?")) {
 438       print_state64(pc, regs);
 439       BREAKPOINT;
 440     }
 441   }
 442   fatal("DEBUG MESSAGE: %s", msg);
 443 }
 444 
 445 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 446   ttyLocker ttyl;
 447   DebuggingContext debugging{};
 448   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 449 #ifndef PRODUCT
 450   tty->cr();
 451   findpc(pc);
 452   tty->cr();
 453 #endif
 454 #define PRINT_REG(rax, value) \
 455   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 456   PRINT_REG(rax, regs[15]);
 457   PRINT_REG(rbx, regs[12]);
 458   PRINT_REG(rcx, regs[14]);
 459   PRINT_REG(rdx, regs[13]);
 460   PRINT_REG(rdi, regs[8]);
 461   PRINT_REG(rsi, regs[9]);
 462   PRINT_REG(rbp, regs[10]);
 463   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
 464   PRINT_REG(rsp, (intptr_t)(&regs[16]));
 465   PRINT_REG(r8 , regs[7]);
 466   PRINT_REG(r9 , regs[6]);
 467   PRINT_REG(r10, regs[5]);
 468   PRINT_REG(r11, regs[4]);
 469   PRINT_REG(r12, regs[3]);
 470   PRINT_REG(r13, regs[2]);
 471   PRINT_REG(r14, regs[1]);
 472   PRINT_REG(r15, regs[0]);
 473 #undef PRINT_REG
 474   // Print some words near the top of the stack.
 475   int64_t* rsp = &regs[16];
 476   int64_t* dump_sp = rsp;
 477   for (int col1 = 0; col1 < 8; col1++) {
 478     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 479     os::print_location(tty, *dump_sp++);
 480   }
 481   for (int row = 0; row < 25; row++) {
 482     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 483     for (int col = 0; col < 4; col++) {
 484       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 485     }
 486     tty->cr();
 487   }
 488   // Print some instructions around pc:
 489   Disassembler::decode((address)pc-64, (address)pc);
 490   tty->print_cr("--------");
 491   Disassembler::decode((address)pc, (address)pc+32);
 492 }
 493 
 494 // The java_calling_convention describes stack locations as ideal slots on
 495 // a frame with no abi restrictions. Since we must observe abi restrictions
 496 // (like the placement of the register window) the slots must be biased by
 497 // the following value.
 498 static int reg2offset_in(VMReg r) {
 499   // Account for saved rbp and return address
 500   // This should really be in_preserve_stack_slots
 501   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
 502 }
 503 
 504 static int reg2offset_out(VMReg r) {
 505   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 506 }
 507 
 508 // A long move
 509 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 510 
 511   // The calling conventions assures us that each VMregpair is either
 512   // all really one physical register or adjacent stack slots.
 513 
 514   if (src.is_single_phys_reg() ) {
 515     if (dst.is_single_phys_reg()) {
 516       if (dst.first() != src.first()) {
 517         mov(dst.first()->as_Register(), src.first()->as_Register());
 518       }
 519     } else {
 520       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
 521              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
 522       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 523     }
 524   } else if (dst.is_single_phys_reg()) {
 525     assert(src.is_single_reg(),  "not a stack pair");
 526     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 527   } else {
 528     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 529     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 530     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 531   }
 532 }
 533 
 534 // A double move
 535 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 536 
 537   // The calling conventions assures us that each VMregpair is either
 538   // all really one physical register or adjacent stack slots.
 539 
 540   if (src.is_single_phys_reg() ) {
 541     if (dst.is_single_phys_reg()) {
 542       // In theory these overlap but the ordering is such that this is likely a nop
 543       if ( src.first() != dst.first()) {
 544         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
 545       }
 546     } else {
 547       assert(dst.is_single_reg(), "not a stack pair");
 548       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
 549     }
 550   } else if (dst.is_single_phys_reg()) {
 551     assert(src.is_single_reg(),  "not a stack pair");
 552     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 553   } else {
 554     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 555     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 556     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 557   }
 558 }
 559 
 560 
 561 // A float arg may have to do float reg int reg conversion
 562 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 563   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
 564 
 565   // The calling conventions assures us that each VMregpair is either
 566   // all really one physical register or adjacent stack slots.
 567 
 568   if (src.first()->is_stack()) {
 569     if (dst.first()->is_stack()) {
 570       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 571       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 572     } else {
 573       // stack to reg
 574       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
 575       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 576     }
 577   } else if (dst.first()->is_stack()) {
 578     // reg to stack
 579     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
 580     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
 581   } else {
 582     // reg to reg
 583     // In theory these overlap but the ordering is such that this is likely a nop
 584     if ( src.first() != dst.first()) {
 585       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
 586     }
 587   }
 588 }
 589 
 590 // On 64 bit we will store integer like items to the stack as
 591 // 64 bits items (x86_32/64 abi) even though java would only store
 592 // 32bits for a parameter. On 32bit it will simply be 32 bits
 593 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
 594 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 595   if (src.first()->is_stack()) {
 596     if (dst.first()->is_stack()) {
 597       // stack to stack
 598       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 599       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 600     } else {
 601       // stack to reg
 602       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 603     }
 604   } else if (dst.first()->is_stack()) {
 605     // reg to stack
 606     // Do we really have to sign extend???
 607     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 608     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 609   } else {
 610     // Do we really have to sign extend???
 611     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 612     if (dst.first() != src.first()) {
 613       movq(dst.first()->as_Register(), src.first()->as_Register());
 614     }
 615   }
 616 }
 617 
 618 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 619   if (src.first()->is_stack()) {
 620     if (dst.first()->is_stack()) {
 621       // stack to stack
 622       movq(rax, Address(rbp, reg2offset_in(src.first())));
 623       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 624     } else {
 625       // stack to reg
 626       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 627     }
 628   } else if (dst.first()->is_stack()) {
 629     // reg to stack
 630     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 631   } else {
 632     if (dst.first() != src.first()) {
 633       movq(dst.first()->as_Register(), src.first()->as_Register());
 634     }
 635   }
 636 }
 637 
 638 // An oop arg. Must pass a handle not the oop itself
 639 void MacroAssembler::object_move(OopMap* map,
 640                         int oop_handle_offset,
 641                         int framesize_in_slots,
 642                         VMRegPair src,
 643                         VMRegPair dst,
 644                         bool is_receiver,
 645                         int* receiver_offset) {
 646 
 647   // must pass a handle. First figure out the location we use as a handle
 648 
 649   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 650 
 651   // See if oop is null if it is we need no handle
 652 
 653   if (src.first()->is_stack()) {
 654 
 655     // Oop is already on the stack as an argument
 656     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 657     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 658     if (is_receiver) {
 659       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 660     }
 661 
 662     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 663     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 664     // conditionally move a null
 665     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 666   } else {
 667 
 668     // Oop is in a register we must store it to the space we reserve
 669     // on the stack for oop_handles and pass a handle if oop is non-null
 670 
 671     const Register rOop = src.first()->as_Register();
 672     int oop_slot;
 673     if (rOop == j_rarg0)
 674       oop_slot = 0;
 675     else if (rOop == j_rarg1)
 676       oop_slot = 1;
 677     else if (rOop == j_rarg2)
 678       oop_slot = 2;
 679     else if (rOop == j_rarg3)
 680       oop_slot = 3;
 681     else if (rOop == j_rarg4)
 682       oop_slot = 4;
 683     else {
 684       assert(rOop == j_rarg5, "wrong register");
 685       oop_slot = 5;
 686     }
 687 
 688     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 689     int offset = oop_slot*VMRegImpl::stack_slot_size;
 690 
 691     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 692     // Store oop in handle area, may be null
 693     movptr(Address(rsp, offset), rOop);
 694     if (is_receiver) {
 695       *receiver_offset = offset;
 696     }
 697 
 698     cmpptr(rOop, NULL_WORD);
 699     lea(rHandle, Address(rsp, offset));
 700     // conditionally move a null from the handle area where it was just stored
 701     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 702   }
 703 
 704   // If arg is on the stack then place it otherwise it is already in correct reg.
 705   if (dst.first()->is_stack()) {
 706     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 707   }
 708 }
 709 
 710 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 711   addq(dst, imm32);
 712 }
 713 
 714 void MacroAssembler::addptr(Register dst, Register src) {
 715   addq(dst, src);
 716 }
 717 
 718 void MacroAssembler::addptr(Address dst, Register src) {
 719   addq(dst, src);
 720 }
 721 
 722 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 723   assert(rscratch != noreg || always_reachable(src), "missing");
 724 
 725   if (reachable(src)) {
 726     Assembler::addsd(dst, as_Address(src));
 727   } else {
 728     lea(rscratch, src);
 729     Assembler::addsd(dst, Address(rscratch, 0));
 730   }
 731 }
 732 
 733 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 734   assert(rscratch != noreg || always_reachable(src), "missing");
 735 
 736   if (reachable(src)) {
 737     addss(dst, as_Address(src));
 738   } else {
 739     lea(rscratch, src);
 740     addss(dst, Address(rscratch, 0));
 741   }
 742 }
 743 
 744 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 745   assert(rscratch != noreg || always_reachable(src), "missing");
 746 
 747   if (reachable(src)) {
 748     Assembler::addpd(dst, as_Address(src));
 749   } else {
 750     lea(rscratch, src);
 751     Assembler::addpd(dst, Address(rscratch, 0));
 752   }
 753 }
 754 
 755 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 756 // Stub code is generated once and never copied.
 757 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 758 void MacroAssembler::align64() {
 759   align(64, (uint)(uintptr_t)pc());
 760 }
 761 
 762 void MacroAssembler::align32() {
 763   align(32, (uint)(uintptr_t)pc());
 764 }
 765 
 766 void MacroAssembler::align(uint modulus) {
 767   // 8273459: Ensure alignment is possible with current segment alignment
 768   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 769   align(modulus, offset());
 770 }
 771 
 772 void MacroAssembler::align(uint modulus, uint target) {
 773   if (target % modulus != 0) {
 774     nop(modulus - (target % modulus));
 775   }
 776 }
 777 
 778 void MacroAssembler::push_f(XMMRegister r) {
 779   subptr(rsp, wordSize);
 780   movflt(Address(rsp, 0), r);
 781 }
 782 
 783 void MacroAssembler::pop_f(XMMRegister r) {
 784   movflt(r, Address(rsp, 0));
 785   addptr(rsp, wordSize);
 786 }
 787 
 788 void MacroAssembler::push_d(XMMRegister r) {
 789   subptr(rsp, 2 * wordSize);
 790   movdbl(Address(rsp, 0), r);
 791 }
 792 
 793 void MacroAssembler::pop_d(XMMRegister r) {
 794   movdbl(r, Address(rsp, 0));
 795   addptr(rsp, 2 * Interpreter::stackElementSize);
 796 }
 797 
 798 void MacroAssembler::push_ppx(Register src) {
 799   if (VM_Version::supports_apx_f()) {
 800     pushp(src);
 801   } else {
 802     Assembler::push(src);
 803   }
 804 }
 805 
 806 void MacroAssembler::pop_ppx(Register dst) {
 807   if (VM_Version::supports_apx_f()) {
 808     popp(dst);
 809   } else {
 810     Assembler::pop(dst);
 811   }
 812 }
 813 
 814 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 815   // Used in sign-masking with aligned address.
 816   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 817   assert(rscratch != noreg || always_reachable(src), "missing");
 818 
 819   if (UseAVX > 2 &&
 820       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 821       (dst->encoding() >= 16)) {
 822     vpand(dst, dst, src, AVX_512bit, rscratch);
 823   } else if (reachable(src)) {
 824     Assembler::andpd(dst, as_Address(src));
 825   } else {
 826     lea(rscratch, src);
 827     Assembler::andpd(dst, Address(rscratch, 0));
 828   }
 829 }
 830 
 831 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 832   // Used in sign-masking with aligned address.
 833   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 834   assert(rscratch != noreg || always_reachable(src), "missing");
 835 
 836   if (reachable(src)) {
 837     Assembler::andps(dst, as_Address(src));
 838   } else {
 839     lea(rscratch, src);
 840     Assembler::andps(dst, Address(rscratch, 0));
 841   }
 842 }
 843 
 844 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 845   andq(dst, imm32);
 846 }
 847 
 848 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 849   assert(rscratch != noreg || always_reachable(src), "missing");
 850 
 851   if (reachable(src)) {
 852     andq(dst, as_Address(src));
 853   } else {
 854     lea(rscratch, src);
 855     andq(dst, Address(rscratch, 0));
 856   }
 857 }
 858 
 859 void MacroAssembler::atomic_incl(Address counter_addr) {
 860   lock();
 861   incrementl(counter_addr);
 862 }
 863 
 864 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 865   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 866 
 867   if (reachable(counter_addr)) {
 868     atomic_incl(as_Address(counter_addr));
 869   } else {
 870     lea(rscratch, counter_addr);
 871     atomic_incl(Address(rscratch, 0));
 872   }
 873 }
 874 
 875 void MacroAssembler::atomic_incq(Address counter_addr) {
 876   lock();
 877   incrementq(counter_addr);
 878 }
 879 
 880 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 881   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 882 
 883   if (reachable(counter_addr)) {
 884     atomic_incq(as_Address(counter_addr));
 885   } else {
 886     lea(rscratch, counter_addr);
 887     atomic_incq(Address(rscratch, 0));
 888   }
 889 }
 890 
 891 // Writes to stack successive pages until offset reached to check for
 892 // stack overflow + shadow pages.  This clobbers tmp.
 893 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 894   movptr(tmp, rsp);
 895   // Bang stack for total size given plus shadow page size.
 896   // Bang one page at a time because large size can bang beyond yellow and
 897   // red zones.
 898   Label loop;
 899   bind(loop);
 900   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 901   subptr(tmp, (int)os::vm_page_size());
 902   subl(size, (int)os::vm_page_size());
 903   jcc(Assembler::greater, loop);
 904 
 905   // Bang down shadow pages too.
 906   // At this point, (tmp-0) is the last address touched, so don't
 907   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 908   // was post-decremented.)  Skip this address by starting at i=1, and
 909   // touch a few more pages below.  N.B.  It is important to touch all
 910   // the way down including all pages in the shadow zone.
 911   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 912     // this could be any sized move but this is can be a debugging crumb
 913     // so the bigger the better.
 914     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 915   }
 916 }
 917 
 918 void MacroAssembler::reserved_stack_check() {
 919   // testing if reserved zone needs to be enabled
 920   Label no_reserved_zone_enabling;
 921 
 922   cmpptr(rsp, Address(r15_thread, JavaThread::reserved_stack_activation_offset()));
 923   jcc(Assembler::below, no_reserved_zone_enabling);
 924 
 925   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), r15_thread);
 926   jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
 927   should_not_reach_here();
 928 
 929   bind(no_reserved_zone_enabling);
 930 }
 931 
 932 void MacroAssembler::c2bool(Register x) {
 933   // implements x == 0 ? 0 : 1
 934   // note: must only look at least-significant byte of x
 935   //       since C-style booleans are stored in one byte
 936   //       only! (was bug)
 937   andl(x, 0xFF);
 938   setb(Assembler::notZero, x);
 939 }
 940 
 941 // Wouldn't need if AddressLiteral version had new name
 942 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 943   Assembler::call(L, rtype);
 944 }
 945 
 946 void MacroAssembler::call(Register entry) {
 947   Assembler::call(entry);
 948 }
 949 
 950 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 951   assert(rscratch != noreg || always_reachable(entry), "missing");
 952 
 953   if (reachable(entry)) {
 954     Assembler::call_literal(entry.target(), entry.rspec());
 955   } else {
 956     lea(rscratch, entry);
 957     Assembler::call(rscratch);
 958   }
 959 }
 960 
 961 void MacroAssembler::ic_call(address entry, jint method_index) {
 962   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 963   // Needs full 64-bit immediate for later patching.
 964   mov64(rax, (int64_t)Universe::non_oop_word());
 965   call(AddressLiteral(entry, rh));
 966 }
 967 
 968 int MacroAssembler::ic_check_size() {
 969   return UseCompactObjectHeaders ? 17 : 14;
 970 }
 971 
 972 int MacroAssembler::ic_check(int end_alignment) {
 973   Register receiver = j_rarg0;
 974   Register data = rax;
 975   Register temp = rscratch1;
 976 
 977   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 978   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 979   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 980   // before the inline cache check here, and not after
 981   align(end_alignment, offset() + ic_check_size());
 982 
 983   int uep_offset = offset();
 984 
 985   if (UseCompactObjectHeaders) {
 986     load_narrow_klass_compact(temp, receiver);
 987     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 988   } else if (UseCompressedClassPointers) {
 989     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 990     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 991   } else {
 992     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 993     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 994   }
 995 
 996   // if inline cache check fails, then jump to runtime routine
 997   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 998   assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
 999 
1000   return uep_offset;
1001 }
1002 
1003 void MacroAssembler::emit_static_call_stub() {
1004   // Static stub relocation also tags the Method* in the code-stream.
1005   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
1006   // This is recognized as unresolved by relocs/nativeinst/ic code.
1007   jump(RuntimeAddress(pc()));
1008 }
1009 
1010 // Implementation of call_VM versions
1011 
1012 void MacroAssembler::call_VM(Register oop_result,
1013                              address entry_point,
1014                              bool check_exceptions) {
1015   Label C, E;
1016   call(C, relocInfo::none);
1017   jmp(E);
1018 
1019   bind(C);
1020   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
1021   ret(0);
1022 
1023   bind(E);
1024 }
1025 
1026 void MacroAssembler::call_VM(Register oop_result,
1027                              address entry_point,
1028                              Register arg_1,
1029                              bool check_exceptions) {
1030   Label C, E;
1031   call(C, relocInfo::none);
1032   jmp(E);
1033 
1034   bind(C);
1035   pass_arg1(this, arg_1);
1036   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
1037   ret(0);
1038 
1039   bind(E);
1040 }
1041 
1042 void MacroAssembler::call_VM(Register oop_result,
1043                              address entry_point,
1044                              Register arg_1,
1045                              Register arg_2,
1046                              bool check_exceptions) {
1047   Label C, E;
1048   call(C, relocInfo::none);
1049   jmp(E);
1050 
1051   bind(C);
1052 
1053   assert_different_registers(arg_1, c_rarg2);
1054 
1055   pass_arg2(this, arg_2);
1056   pass_arg1(this, arg_1);
1057   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
1058   ret(0);
1059 
1060   bind(E);
1061 }
1062 
1063 void MacroAssembler::call_VM(Register oop_result,
1064                              address entry_point,
1065                              Register arg_1,
1066                              Register arg_2,
1067                              Register arg_3,
1068                              bool check_exceptions) {
1069   Label C, E;
1070   call(C, relocInfo::none);
1071   jmp(E);
1072 
1073   bind(C);
1074 
1075   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1076   assert_different_registers(arg_2, c_rarg3);
1077   pass_arg3(this, arg_3);
1078   pass_arg2(this, arg_2);
1079   pass_arg1(this, arg_1);
1080   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
1081   ret(0);
1082 
1083   bind(E);
1084 }
1085 
1086 void MacroAssembler::call_VM(Register oop_result,
1087                              Register last_java_sp,
1088                              address entry_point,
1089                              int number_of_arguments,
1090                              bool check_exceptions) {
1091   call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1092 }
1093 
1094 void MacroAssembler::call_VM(Register oop_result,
1095                              Register last_java_sp,
1096                              address entry_point,
1097                              Register arg_1,
1098                              bool check_exceptions) {
1099   pass_arg1(this, arg_1);
1100   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1101 }
1102 
1103 void MacroAssembler::call_VM(Register oop_result,
1104                              Register last_java_sp,
1105                              address entry_point,
1106                              Register arg_1,
1107                              Register arg_2,
1108                              bool check_exceptions) {
1109 
1110   assert_different_registers(arg_1, c_rarg2);
1111   pass_arg2(this, arg_2);
1112   pass_arg1(this, arg_1);
1113   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1114 }
1115 
1116 void MacroAssembler::call_VM(Register oop_result,
1117                              Register last_java_sp,
1118                              address entry_point,
1119                              Register arg_1,
1120                              Register arg_2,
1121                              Register arg_3,
1122                              bool check_exceptions) {
1123   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1124   assert_different_registers(arg_2, c_rarg3);
1125   pass_arg3(this, arg_3);
1126   pass_arg2(this, arg_2);
1127   pass_arg1(this, arg_1);
1128   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1129 }
1130 
1131 void MacroAssembler::super_call_VM(Register oop_result,
1132                                    Register last_java_sp,
1133                                    address entry_point,
1134                                    int number_of_arguments,
1135                                    bool check_exceptions) {
1136   MacroAssembler::call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1137 }
1138 
1139 void MacroAssembler::super_call_VM(Register oop_result,
1140                                    Register last_java_sp,
1141                                    address entry_point,
1142                                    Register arg_1,
1143                                    bool check_exceptions) {
1144   pass_arg1(this, arg_1);
1145   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1146 }
1147 
1148 void MacroAssembler::super_call_VM(Register oop_result,
1149                                    Register last_java_sp,
1150                                    address entry_point,
1151                                    Register arg_1,
1152                                    Register arg_2,
1153                                    bool check_exceptions) {
1154 
1155   assert_different_registers(arg_1, c_rarg2);
1156   pass_arg2(this, arg_2);
1157   pass_arg1(this, arg_1);
1158   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1159 }
1160 
1161 void MacroAssembler::super_call_VM(Register oop_result,
1162                                    Register last_java_sp,
1163                                    address entry_point,
1164                                    Register arg_1,
1165                                    Register arg_2,
1166                                    Register arg_3,
1167                                    bool check_exceptions) {
1168   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1169   assert_different_registers(arg_2, c_rarg3);
1170   pass_arg3(this, arg_3);
1171   pass_arg2(this, arg_2);
1172   pass_arg1(this, arg_1);
1173   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1174 }
1175 
1176 void MacroAssembler::call_VM_base(Register oop_result,
1177                                   Register last_java_sp,
1178                                   address  entry_point,
1179                                   int      number_of_arguments,
1180                                   bool     check_exceptions) {
1181   Register java_thread = r15_thread;
1182 
1183   // determine last_java_sp register
1184   if (!last_java_sp->is_valid()) {
1185     last_java_sp = rsp;
1186   }
1187   // debugging support
1188   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
1189 #ifdef ASSERT
1190   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
1191   // r12 is the heapbase.
1192   if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
1193 #endif // ASSERT
1194 
1195   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
1196   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
1197 
1198   // push java thread (becomes first argument of C function)
1199 
1200   mov(c_rarg0, r15_thread);
1201 
1202   // set last Java frame before call
1203   assert(last_java_sp != rbp, "can't use ebp/rbp");
1204 
1205   // Only interpreter should have to set fp
1206   set_last_Java_frame(last_java_sp, rbp, nullptr, rscratch1);
1207 
1208   // do the call, remove parameters
1209   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1210 
1211 #ifdef ASSERT
1212   // Check that thread register is not clobbered.
1213   guarantee(java_thread != rax, "change this code");
1214   push(rax);
1215   { Label L;
1216     get_thread_slow(rax);
1217     cmpptr(java_thread, rax);
1218     jcc(Assembler::equal, L);
1219     STOP("MacroAssembler::call_VM_base: java_thread not callee saved?");
1220     bind(L);
1221   }
1222   pop(rax);
1223 #endif
1224 
1225   // reset last Java frame
1226   // Only interpreter should have to clear fp
1227   reset_last_Java_frame(true);
1228 
1229    // C++ interp handles this in the interpreter
1230   check_and_handle_popframe();
1231   check_and_handle_earlyret();
1232 
1233   if (check_exceptions) {
1234     // check for pending exceptions (java_thread is set upon return)
1235     cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
1236     // This used to conditionally jump to forward_exception however it is
1237     // possible if we relocate that the branch will not reach. So we must jump
1238     // around so we can always reach
1239 
1240     Label ok;
1241     jcc(Assembler::equal, ok);
1242     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1243     bind(ok);
1244   }
1245 
1246   // get oop result if there is one and reset the value in the thread
1247   if (oop_result->is_valid()) {
1248     get_vm_result_oop(oop_result);
1249   }
1250 }
1251 
1252 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
1253   // Calculate the value for last_Java_sp somewhat subtle.
1254   // call_VM does an intermediate call which places a return address on
1255   // the stack just under the stack pointer as the user finished with it.
1256   // This allows use to retrieve last_Java_pc from last_Java_sp[-1].
1257 
1258   // We've pushed one address, correct last_Java_sp
1259   lea(rax, Address(rsp, wordSize));
1260 
1261   call_VM_base(oop_result, rax, entry_point, number_of_arguments, check_exceptions);
1262 }
1263 
1264 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
1265 void MacroAssembler::call_VM_leaf0(address entry_point) {
1266   MacroAssembler::call_VM_leaf_base(entry_point, 0);
1267 }
1268 
1269 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1270   call_VM_leaf_base(entry_point, number_of_arguments);
1271 }
1272 
1273 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1274   pass_arg0(this, arg_0);
1275   call_VM_leaf(entry_point, 1);
1276 }
1277 
1278 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1279 
1280   assert_different_registers(arg_0, c_rarg1);
1281   pass_arg1(this, arg_1);
1282   pass_arg0(this, arg_0);
1283   call_VM_leaf(entry_point, 2);
1284 }
1285 
1286 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1287   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1288   assert_different_registers(arg_1, c_rarg2);
1289   pass_arg2(this, arg_2);
1290   pass_arg1(this, arg_1);
1291   pass_arg0(this, arg_0);
1292   call_VM_leaf(entry_point, 3);
1293 }
1294 
1295 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1296   assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3);
1297   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1298   assert_different_registers(arg_2, c_rarg3);
1299   pass_arg3(this, arg_3);
1300   pass_arg2(this, arg_2);
1301   pass_arg1(this, arg_1);
1302   pass_arg0(this, arg_0);
1303   call_VM_leaf(entry_point, 3);
1304 }
1305 
1306 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1307   pass_arg0(this, arg_0);
1308   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1309 }
1310 
1311 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1312   assert_different_registers(arg_0, c_rarg1);
1313   pass_arg1(this, arg_1);
1314   pass_arg0(this, arg_0);
1315   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1316 }
1317 
1318 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1319   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1320   assert_different_registers(arg_1, c_rarg2);
1321   pass_arg2(this, arg_2);
1322   pass_arg1(this, arg_1);
1323   pass_arg0(this, arg_0);
1324   MacroAssembler::call_VM_leaf_base(entry_point, 3);
1325 }
1326 
1327 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1328   assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3);
1329   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1330   assert_different_registers(arg_2, c_rarg3);
1331   pass_arg3(this, arg_3);
1332   pass_arg2(this, arg_2);
1333   pass_arg1(this, arg_1);
1334   pass_arg0(this, arg_0);
1335   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1336 }
1337 
1338 void MacroAssembler::get_vm_result_oop(Register oop_result) {
1339   movptr(oop_result, Address(r15_thread, JavaThread::vm_result_oop_offset()));
1340   movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
1341   verify_oop_msg(oop_result, "broken oop in call_VM_base");
1342 }
1343 
1344 void MacroAssembler::get_vm_result_metadata(Register metadata_result) {
1345   movptr(metadata_result, Address(r15_thread, JavaThread::vm_result_metadata_offset()));
1346   movptr(Address(r15_thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
1347 }
1348 
1349 void MacroAssembler::check_and_handle_earlyret() {
1350 }
1351 
1352 void MacroAssembler::check_and_handle_popframe() {
1353 }
1354 
1355 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
1356   assert(rscratch != noreg || always_reachable(src1), "missing");
1357 
1358   if (reachable(src1)) {
1359     cmpl(as_Address(src1), imm);
1360   } else {
1361     lea(rscratch, src1);
1362     cmpl(Address(rscratch, 0), imm);
1363   }
1364 }
1365 
1366 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
1367   assert(!src2.is_lval(), "use cmpptr");
1368   assert(rscratch != noreg || always_reachable(src2), "missing");
1369 
1370   if (reachable(src2)) {
1371     cmpl(src1, as_Address(src2));
1372   } else {
1373     lea(rscratch, src2);
1374     cmpl(src1, Address(rscratch, 0));
1375   }
1376 }
1377 
1378 void MacroAssembler::cmp32(Register src1, int32_t imm) {
1379   Assembler::cmpl(src1, imm);
1380 }
1381 
1382 void MacroAssembler::cmp32(Register src1, Address src2) {
1383   Assembler::cmpl(src1, src2);
1384 }
1385 
1386 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1387   ucomisd(opr1, opr2);
1388 
1389   Label L;
1390   if (unordered_is_less) {
1391     movl(dst, -1);
1392     jcc(Assembler::parity, L);
1393     jcc(Assembler::below , L);
1394     movl(dst, 0);
1395     jcc(Assembler::equal , L);
1396     increment(dst);
1397   } else { // unordered is greater
1398     movl(dst, 1);
1399     jcc(Assembler::parity, L);
1400     jcc(Assembler::above , L);
1401     movl(dst, 0);
1402     jcc(Assembler::equal , L);
1403     decrementl(dst);
1404   }
1405   bind(L);
1406 }
1407 
1408 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1409   ucomiss(opr1, opr2);
1410 
1411   Label L;
1412   if (unordered_is_less) {
1413     movl(dst, -1);
1414     jcc(Assembler::parity, L);
1415     jcc(Assembler::below , L);
1416     movl(dst, 0);
1417     jcc(Assembler::equal , L);
1418     increment(dst);
1419   } else { // unordered is greater
1420     movl(dst, 1);
1421     jcc(Assembler::parity, L);
1422     jcc(Assembler::above , L);
1423     movl(dst, 0);
1424     jcc(Assembler::equal , L);
1425     decrementl(dst);
1426   }
1427   bind(L);
1428 }
1429 
1430 
1431 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
1432   assert(rscratch != noreg || always_reachable(src1), "missing");
1433 
1434   if (reachable(src1)) {
1435     cmpb(as_Address(src1), imm);
1436   } else {
1437     lea(rscratch, src1);
1438     cmpb(Address(rscratch, 0), imm);
1439   }
1440 }
1441 
1442 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
1443   assert(rscratch != noreg || always_reachable(src2), "missing");
1444 
1445   if (src2.is_lval()) {
1446     movptr(rscratch, src2);
1447     Assembler::cmpq(src1, rscratch);
1448   } else if (reachable(src2)) {
1449     cmpq(src1, as_Address(src2));
1450   } else {
1451     lea(rscratch, src2);
1452     Assembler::cmpq(src1, Address(rscratch, 0));
1453   }
1454 }
1455 
1456 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
1457   assert(src2.is_lval(), "not a mem-mem compare");
1458   // moves src2's literal address
1459   movptr(rscratch, src2);
1460   Assembler::cmpq(src1, rscratch);
1461 }
1462 
1463 void MacroAssembler::cmpoop(Register src1, Register src2) {
1464   cmpptr(src1, src2);
1465 }
1466 
1467 void MacroAssembler::cmpoop(Register src1, Address src2) {
1468   cmpptr(src1, src2);
1469 }
1470 
1471 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
1472   movoop(rscratch, src2);
1473   cmpptr(src1, rscratch);
1474 }
1475 
1476 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
1477   assert(rscratch != noreg || always_reachable(adr), "missing");
1478 
1479   if (reachable(adr)) {
1480     lock();
1481     cmpxchgptr(reg, as_Address(adr));
1482   } else {
1483     lea(rscratch, adr);
1484     lock();
1485     cmpxchgptr(reg, Address(rscratch, 0));
1486   }
1487 }
1488 
1489 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
1490   cmpxchgq(reg, adr);
1491 }
1492 
1493 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1494   assert(rscratch != noreg || always_reachable(src), "missing");
1495 
1496   if (reachable(src)) {
1497     Assembler::comisd(dst, as_Address(src));
1498   } else {
1499     lea(rscratch, src);
1500     Assembler::comisd(dst, Address(rscratch, 0));
1501   }
1502 }
1503 
1504 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
1505   assert(rscratch != noreg || always_reachable(src), "missing");
1506 
1507   if (reachable(src)) {
1508     Assembler::comiss(dst, as_Address(src));
1509   } else {
1510     lea(rscratch, src);
1511     Assembler::comiss(dst, Address(rscratch, 0));
1512   }
1513 }
1514 
1515 
1516 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
1517   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
1518 
1519   Condition negated_cond = negate_condition(cond);
1520   Label L;
1521   jcc(negated_cond, L);
1522   pushf(); // Preserve flags
1523   atomic_incl(counter_addr, rscratch);
1524   popf();
1525   bind(L);
1526 }
1527 
1528 int MacroAssembler::corrected_idivl(Register reg) {
1529   // Full implementation of Java idiv and irem; checks for
1530   // special case as described in JVM spec., p.243 & p.271.
1531   // The function returns the (pc) offset of the idivl
1532   // instruction - may be needed for implicit exceptions.
1533   //
1534   //         normal case                           special case
1535   //
1536   // input : rax,: dividend                         min_int
1537   //         reg: divisor   (may not be rax,/rdx)   -1
1538   //
1539   // output: rax,: quotient  (= rax, idiv reg)       min_int
1540   //         rdx: remainder (= rax, irem reg)       0
1541   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
1542   const int min_int = 0x80000000;
1543   Label normal_case, special_case;
1544 
1545   // check for special case
1546   cmpl(rax, min_int);
1547   jcc(Assembler::notEqual, normal_case);
1548   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
1549   cmpl(reg, -1);
1550   jcc(Assembler::equal, special_case);
1551 
1552   // handle normal case
1553   bind(normal_case);
1554   cdql();
1555   int idivl_offset = offset();
1556   idivl(reg);
1557 
1558   // normal and special case exit
1559   bind(special_case);
1560 
1561   return idivl_offset;
1562 }
1563 
1564 
1565 
1566 void MacroAssembler::decrementl(Register reg, int value) {
1567   if (value == min_jint) {subl(reg, value) ; return; }
1568   if (value <  0) { incrementl(reg, -value); return; }
1569   if (value == 0) {                        ; return; }
1570   if (value == 1 && UseIncDec) { decl(reg) ; return; }
1571   /* else */      { subl(reg, value)       ; return; }
1572 }
1573 
1574 void MacroAssembler::decrementl(Address dst, int value) {
1575   if (value == min_jint) {subl(dst, value) ; return; }
1576   if (value <  0) { incrementl(dst, -value); return; }
1577   if (value == 0) {                        ; return; }
1578   if (value == 1 && UseIncDec) { decl(dst) ; return; }
1579   /* else */      { subl(dst, value)       ; return; }
1580 }
1581 
1582 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
1583   assert(shift_value > 0, "illegal shift value");
1584   Label _is_positive;
1585   testl (reg, reg);
1586   jcc (Assembler::positive, _is_positive);
1587   int offset = (1 << shift_value) - 1 ;
1588 
1589   if (offset == 1) {
1590     incrementl(reg);
1591   } else {
1592     addl(reg, offset);
1593   }
1594 
1595   bind (_is_positive);
1596   sarl(reg, shift_value);
1597 }
1598 
1599 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1600   assert(rscratch != noreg || always_reachable(src), "missing");
1601 
1602   if (reachable(src)) {
1603     Assembler::divsd(dst, as_Address(src));
1604   } else {
1605     lea(rscratch, src);
1606     Assembler::divsd(dst, Address(rscratch, 0));
1607   }
1608 }
1609 
1610 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
1611   assert(rscratch != noreg || always_reachable(src), "missing");
1612 
1613   if (reachable(src)) {
1614     Assembler::divss(dst, as_Address(src));
1615   } else {
1616     lea(rscratch, src);
1617     Assembler::divss(dst, Address(rscratch, 0));
1618   }
1619 }
1620 
1621 void MacroAssembler::enter() {
1622   push(rbp);
1623   mov(rbp, rsp);
1624 }
1625 
1626 void MacroAssembler::post_call_nop() {
1627   if (!Continuations::enabled()) {
1628     return;
1629   }
1630   InstructionMark im(this);
1631   relocate(post_call_nop_Relocation::spec());
1632   InlineSkippedInstructionsCounter skipCounter(this);
1633   emit_int8((uint8_t)0x0f);
1634   emit_int8((uint8_t)0x1f);
1635   emit_int8((uint8_t)0x84);
1636   emit_int8((uint8_t)0x00);
1637   emit_int32(0x00);
1638 }
1639 
1640 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1641   assert(rscratch != noreg || always_reachable(src), "missing");
1642   if (reachable(src)) {
1643     Assembler::mulpd(dst, as_Address(src));
1644   } else {
1645     lea(rscratch, src);
1646     Assembler::mulpd(dst, Address(rscratch, 0));
1647   }
1648 }
1649 
1650 // dst = c = a * b + c
1651 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
1652   Assembler::vfmadd231sd(c, a, b);
1653   if (dst != c) {
1654     movdbl(dst, c);
1655   }
1656 }
1657 
1658 // dst = c = a * b + c
1659 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
1660   Assembler::vfmadd231ss(c, a, b);
1661   if (dst != c) {
1662     movflt(dst, c);
1663   }
1664 }
1665 
1666 // dst = c = a * b + c
1667 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
1668   Assembler::vfmadd231pd(c, a, b, vector_len);
1669   if (dst != c) {
1670     vmovdqu(dst, c);
1671   }
1672 }
1673 
1674 // dst = c = a * b + c
1675 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
1676   Assembler::vfmadd231ps(c, a, b, vector_len);
1677   if (dst != c) {
1678     vmovdqu(dst, c);
1679   }
1680 }
1681 
1682 // dst = c = a * b + c
1683 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
1684   Assembler::vfmadd231pd(c, a, b, vector_len);
1685   if (dst != c) {
1686     vmovdqu(dst, c);
1687   }
1688 }
1689 
1690 // dst = c = a * b + c
1691 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
1692   Assembler::vfmadd231ps(c, a, b, vector_len);
1693   if (dst != c) {
1694     vmovdqu(dst, c);
1695   }
1696 }
1697 
1698 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
1699   assert(rscratch != noreg || always_reachable(dst), "missing");
1700 
1701   if (reachable(dst)) {
1702     incrementl(as_Address(dst));
1703   } else {
1704     lea(rscratch, dst);
1705     incrementl(Address(rscratch, 0));
1706   }
1707 }
1708 
1709 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
1710   incrementl(as_Address(dst, rscratch));
1711 }
1712 
1713 void MacroAssembler::incrementl(Register reg, int value) {
1714   if (value == min_jint) {addl(reg, value) ; return; }
1715   if (value <  0) { decrementl(reg, -value); return; }
1716   if (value == 0) {                        ; return; }
1717   if (value == 1 && UseIncDec) { incl(reg) ; return; }
1718   /* else */      { addl(reg, value)       ; return; }
1719 }
1720 
1721 void MacroAssembler::incrementl(Address dst, int value) {
1722   if (value == min_jint) {addl(dst, value) ; return; }
1723   if (value <  0) { decrementl(dst, -value); return; }
1724   if (value == 0) {                        ; return; }
1725   if (value == 1 && UseIncDec) { incl(dst) ; return; }
1726   /* else */      { addl(dst, value)       ; return; }
1727 }
1728 
1729 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
1730   assert(rscratch != noreg || always_reachable(dst), "missing");
1731   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump");
1732   if (reachable(dst)) {
1733     jmp_literal(dst.target(), dst.rspec());
1734   } else {
1735     lea(rscratch, dst);
1736     jmp(rscratch);
1737   }
1738 }
1739 
1740 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
1741   assert(rscratch != noreg || always_reachable(dst), "missing");
1742   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc");
1743   if (reachable(dst)) {
1744     InstructionMark im(this);
1745     relocate(dst.reloc());
1746     const int short_size = 2;
1747     const int long_size = 6;
1748     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
1749     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
1750       // 0111 tttn #8-bit disp
1751       emit_int8(0x70 | cc);
1752       emit_int8((offs - short_size) & 0xFF);
1753     } else {
1754       // 0000 1111 1000 tttn #32-bit disp
1755       emit_int8(0x0F);
1756       emit_int8((unsigned char)(0x80 | cc));
1757       emit_int32(offs - long_size);
1758     }
1759   } else {
1760 #ifdef ASSERT
1761     warning("reversing conditional branch");
1762 #endif /* ASSERT */
1763     Label skip;
1764     jccb(reverse[cc], skip);
1765     lea(rscratch, dst);
1766     Assembler::jmp(rscratch);
1767     bind(skip);
1768   }
1769 }
1770 
1771 void MacroAssembler::cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch) {
1772   ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
1773   assert(rscratch != noreg || always_reachable(mxcsr_std), "missing");
1774 
1775   stmxcsr(mxcsr_save);
1776   movl(tmp, mxcsr_save);
1777   if (EnableX86ECoreOpts) {
1778     // The mxcsr_std has status bits set for performance on ECore
1779     orl(tmp, 0x003f);
1780   } else {
1781     // Mask out status bits (only check control and mask bits)
1782     andl(tmp, 0xFFC0);
1783   }
1784   cmp32(tmp, mxcsr_std, rscratch);
1785 }
1786 
1787 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
1788   assert(rscratch != noreg || always_reachable(src), "missing");
1789 
1790   if (reachable(src)) {
1791     Assembler::ldmxcsr(as_Address(src));
1792   } else {
1793     lea(rscratch, src);
1794     Assembler::ldmxcsr(Address(rscratch, 0));
1795   }
1796 }
1797 
1798 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1799   int off = offset();
1800   movsbl(dst, src); // movsxb
1801   return off;
1802 }
1803 
1804 // Note: load_signed_short used to be called load_signed_word.
1805 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
1806 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
1807 // The term "word" in HotSpot means a 32- or 64-bit machine word.
1808 int MacroAssembler::load_signed_short(Register dst, Address src) {
1809   // This is dubious to me since it seems safe to do a signed 16 => 64 bit
1810   // version but this is what 64bit has always done. This seems to imply
1811   // that users are only using 32bits worth.
1812   int off = offset();
1813   movswl(dst, src); // movsxw
1814   return off;
1815 }
1816 
1817 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1818   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
1819   // and "3.9 Partial Register Penalties", p. 22).
1820   int off = offset();
1821   movzbl(dst, src); // movzxb
1822   return off;
1823 }
1824 
1825 // Note: load_unsigned_short used to be called load_unsigned_word.
1826 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1827   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
1828   // and "3.9 Partial Register Penalties", p. 22).
1829   int off = offset();
1830   movzwl(dst, src); // movzxw
1831   return off;
1832 }
1833 
1834 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
1835   switch (size_in_bytes) {
1836   case  8:  movq(dst, src); break;
1837   case  4:  movl(dst, src); break;
1838   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1839   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1840   default:  ShouldNotReachHere();
1841   }
1842 }
1843 
1844 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
1845   switch (size_in_bytes) {
1846   case  8:  movq(dst, src); break;
1847   case  4:  movl(dst, src); break;
1848   case  2:  movw(dst, src); break;
1849   case  1:  movb(dst, src); break;
1850   default:  ShouldNotReachHere();
1851   }
1852 }
1853 
1854 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
1855   assert(rscratch != noreg || always_reachable(dst), "missing");
1856 
1857   if (reachable(dst)) {
1858     movl(as_Address(dst), src);
1859   } else {
1860     lea(rscratch, dst);
1861     movl(Address(rscratch, 0), src);
1862   }
1863 }
1864 
1865 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
1866   if (reachable(src)) {
1867     movl(dst, as_Address(src));
1868   } else {
1869     lea(dst, src);
1870     movl(dst, Address(dst, 0));
1871   }
1872 }
1873 
1874 // C++ bool manipulation
1875 
1876 void MacroAssembler::movbool(Register dst, Address src) {
1877   if(sizeof(bool) == 1)
1878     movb(dst, src);
1879   else if(sizeof(bool) == 2)
1880     movw(dst, src);
1881   else if(sizeof(bool) == 4)
1882     movl(dst, src);
1883   else
1884     // unsupported
1885     ShouldNotReachHere();
1886 }
1887 
1888 void MacroAssembler::movbool(Address dst, bool boolconst) {
1889   if(sizeof(bool) == 1)
1890     movb(dst, (int) boolconst);
1891   else if(sizeof(bool) == 2)
1892     movw(dst, (int) boolconst);
1893   else if(sizeof(bool) == 4)
1894     movl(dst, (int) boolconst);
1895   else
1896     // unsupported
1897     ShouldNotReachHere();
1898 }
1899 
1900 void MacroAssembler::movbool(Address dst, Register src) {
1901   if(sizeof(bool) == 1)
1902     movb(dst, src);
1903   else if(sizeof(bool) == 2)
1904     movw(dst, src);
1905   else if(sizeof(bool) == 4)
1906     movl(dst, src);
1907   else
1908     // unsupported
1909     ShouldNotReachHere();
1910 }
1911 
1912 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
1913   assert(rscratch != noreg || always_reachable(src), "missing");
1914 
1915   if (reachable(src)) {
1916     movdl(dst, as_Address(src));
1917   } else {
1918     lea(rscratch, src);
1919     movdl(dst, Address(rscratch, 0));
1920   }
1921 }
1922 
1923 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
1924   assert(rscratch != noreg || always_reachable(src), "missing");
1925 
1926   if (reachable(src)) {
1927     movq(dst, as_Address(src));
1928   } else {
1929     lea(rscratch, src);
1930     movq(dst, Address(rscratch, 0));
1931   }
1932 }
1933 
1934 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
1935   assert(rscratch != noreg || always_reachable(src), "missing");
1936 
1937   if (reachable(src)) {
1938     if (UseXmmLoadAndClearUpper) {
1939       movsd (dst, as_Address(src));
1940     } else {
1941       movlpd(dst, as_Address(src));
1942     }
1943   } else {
1944     lea(rscratch, src);
1945     if (UseXmmLoadAndClearUpper) {
1946       movsd (dst, Address(rscratch, 0));
1947     } else {
1948       movlpd(dst, Address(rscratch, 0));
1949     }
1950   }
1951 }
1952 
1953 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
1954   assert(rscratch != noreg || always_reachable(src), "missing");
1955 
1956   if (reachable(src)) {
1957     movss(dst, as_Address(src));
1958   } else {
1959     lea(rscratch, src);
1960     movss(dst, Address(rscratch, 0));
1961   }
1962 }
1963 
1964 void MacroAssembler::movptr(Register dst, Register src) {
1965   movq(dst, src);
1966 }
1967 
1968 void MacroAssembler::movptr(Register dst, Address src) {
1969   movq(dst, src);
1970 }
1971 
1972 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
1973 void MacroAssembler::movptr(Register dst, intptr_t src) {
1974   if (is_uimm32(src)) {
1975     movl(dst, checked_cast<uint32_t>(src));
1976   } else if (is_simm32(src)) {
1977     movq(dst, checked_cast<int32_t>(src));
1978   } else {
1979     mov64(dst, src);
1980   }
1981 }
1982 
1983 void MacroAssembler::movptr(Address dst, Register src) {
1984   movq(dst, src);
1985 }
1986 
1987 void MacroAssembler::movptr(Address dst, int32_t src) {
1988   movslq(dst, src);
1989 }
1990 
1991 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
1992   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
1993   Assembler::movdqu(dst, src);
1994 }
1995 
1996 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
1997   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
1998   Assembler::movdqu(dst, src);
1999 }
2000 
2001 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
2002   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2003   Assembler::movdqu(dst, src);
2004 }
2005 
2006 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
2007   assert(rscratch != noreg || always_reachable(src), "missing");
2008 
2009   if (reachable(src)) {
2010     movdqu(dst, as_Address(src));
2011   } else {
2012     lea(rscratch, src);
2013     movdqu(dst, Address(rscratch, 0));
2014   }
2015 }
2016 
2017 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
2018   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2019   Assembler::vmovdqu(dst, src);
2020 }
2021 
2022 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
2023   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2024   Assembler::vmovdqu(dst, src);
2025 }
2026 
2027 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2028   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2029   Assembler::vmovdqu(dst, src);
2030 }
2031 
2032 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
2033   assert(rscratch != noreg || always_reachable(src), "missing");
2034 
2035   if (reachable(src)) {
2036     vmovdqu(dst, as_Address(src));
2037   }
2038   else {
2039     lea(rscratch, src);
2040     vmovdqu(dst, Address(rscratch, 0));
2041   }
2042 }
2043 
2044 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2045   assert(rscratch != noreg || always_reachable(src), "missing");
2046 
2047   if (vector_len == AVX_512bit) {
2048     evmovdquq(dst, src, AVX_512bit, rscratch);
2049   } else if (vector_len == AVX_256bit) {
2050     vmovdqu(dst, src, rscratch);
2051   } else {
2052     movdqu(dst, src, rscratch);
2053   }
2054 }
2055 
2056 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src, int vector_len) {
2057   if (vector_len == AVX_512bit) {
2058     evmovdquq(dst, src, AVX_512bit);
2059   } else if (vector_len == AVX_256bit) {
2060     vmovdqu(dst, src);
2061   } else {
2062     movdqu(dst, src);
2063   }
2064 }
2065 
2066 void MacroAssembler::vmovdqu(Address dst, XMMRegister src, int vector_len) {
2067   if (vector_len == AVX_512bit) {
2068     evmovdquq(dst, src, AVX_512bit);
2069   } else if (vector_len == AVX_256bit) {
2070     vmovdqu(dst, src);
2071   } else {
2072     movdqu(dst, src);
2073   }
2074 }
2075 
2076 void MacroAssembler::vmovdqu(XMMRegister dst, Address src, int vector_len) {
2077   if (vector_len == AVX_512bit) {
2078     evmovdquq(dst, src, AVX_512bit);
2079   } else if (vector_len == AVX_256bit) {
2080     vmovdqu(dst, src);
2081   } else {
2082     movdqu(dst, src);
2083   }
2084 }
2085 
2086 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
2087   assert(rscratch != noreg || always_reachable(src), "missing");
2088 
2089   if (reachable(src)) {
2090     vmovdqa(dst, as_Address(src));
2091   }
2092   else {
2093     lea(rscratch, src);
2094     vmovdqa(dst, Address(rscratch, 0));
2095   }
2096 }
2097 
2098 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2099   assert(rscratch != noreg || always_reachable(src), "missing");
2100 
2101   if (vector_len == AVX_512bit) {
2102     evmovdqaq(dst, src, AVX_512bit, rscratch);
2103   } else if (vector_len == AVX_256bit) {
2104     vmovdqa(dst, src, rscratch);
2105   } else {
2106     movdqa(dst, src, rscratch);
2107   }
2108 }
2109 
2110 void MacroAssembler::kmov(KRegister dst, Address src) {
2111   if (VM_Version::supports_avx512bw()) {
2112     kmovql(dst, src);
2113   } else {
2114     assert(VM_Version::supports_evex(), "");
2115     kmovwl(dst, src);
2116   }
2117 }
2118 
2119 void MacroAssembler::kmov(Address dst, KRegister src) {
2120   if (VM_Version::supports_avx512bw()) {
2121     kmovql(dst, src);
2122   } else {
2123     assert(VM_Version::supports_evex(), "");
2124     kmovwl(dst, src);
2125   }
2126 }
2127 
2128 void MacroAssembler::kmov(KRegister dst, KRegister src) {
2129   if (VM_Version::supports_avx512bw()) {
2130     kmovql(dst, src);
2131   } else {
2132     assert(VM_Version::supports_evex(), "");
2133     kmovwl(dst, src);
2134   }
2135 }
2136 
2137 void MacroAssembler::kmov(Register dst, KRegister src) {
2138   if (VM_Version::supports_avx512bw()) {
2139     kmovql(dst, src);
2140   } else {
2141     assert(VM_Version::supports_evex(), "");
2142     kmovwl(dst, src);
2143   }
2144 }
2145 
2146 void MacroAssembler::kmov(KRegister dst, Register src) {
2147   if (VM_Version::supports_avx512bw()) {
2148     kmovql(dst, src);
2149   } else {
2150     assert(VM_Version::supports_evex(), "");
2151     kmovwl(dst, src);
2152   }
2153 }
2154 
2155 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
2156   assert(rscratch != noreg || always_reachable(src), "missing");
2157 
2158   if (reachable(src)) {
2159     kmovql(dst, as_Address(src));
2160   } else {
2161     lea(rscratch, src);
2162     kmovql(dst, Address(rscratch, 0));
2163   }
2164 }
2165 
2166 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
2167   assert(rscratch != noreg || always_reachable(src), "missing");
2168 
2169   if (reachable(src)) {
2170     kmovwl(dst, as_Address(src));
2171   } else {
2172     lea(rscratch, src);
2173     kmovwl(dst, Address(rscratch, 0));
2174   }
2175 }
2176 
2177 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2178                                int vector_len, Register rscratch) {
2179   assert(rscratch != noreg || always_reachable(src), "missing");
2180 
2181   if (reachable(src)) {
2182     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
2183   } else {
2184     lea(rscratch, src);
2185     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
2186   }
2187 }
2188 
2189 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2190                                int vector_len, Register rscratch) {
2191   assert(rscratch != noreg || always_reachable(src), "missing");
2192 
2193   if (reachable(src)) {
2194     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
2195   } else {
2196     lea(rscratch, src);
2197     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
2198   }
2199 }
2200 
2201 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2202   assert(rscratch != noreg || always_reachable(src), "missing");
2203 
2204   if (reachable(src)) {
2205     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
2206   } else {
2207     lea(rscratch, src);
2208     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
2209   }
2210 }
2211 
2212 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2213   assert(rscratch != noreg || always_reachable(src), "missing");
2214 
2215   if (reachable(src)) {
2216     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
2217   } else {
2218     lea(rscratch, src);
2219     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
2220   }
2221 }
2222 
2223 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2224   assert(rscratch != noreg || always_reachable(src), "missing");
2225 
2226   if (reachable(src)) {
2227     Assembler::evmovdquq(dst, as_Address(src), vector_len);
2228   } else {
2229     lea(rscratch, src);
2230     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
2231   }
2232 }
2233 
2234 void MacroAssembler::evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2235   assert(rscratch != noreg || always_reachable(src), "missing");
2236 
2237   if (reachable(src)) {
2238     Assembler::evmovdqaq(dst, mask, as_Address(src), merge, vector_len);
2239   } else {
2240     lea(rscratch, src);
2241     Assembler::evmovdqaq(dst, mask, Address(rscratch, 0), merge, vector_len);
2242   }
2243 }
2244 
2245 void MacroAssembler::evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2246   assert(rscratch != noreg || always_reachable(src), "missing");
2247 
2248   if (reachable(src)) {
2249     Assembler::evmovdqaq(dst, as_Address(src), vector_len);
2250   } else {
2251     lea(rscratch, src);
2252     Assembler::evmovdqaq(dst, Address(rscratch, 0), vector_len);
2253   }
2254 }
2255 
2256 void MacroAssembler::movapd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2257   assert(rscratch != noreg || always_reachable(src), "missing");
2258 
2259   if (reachable(src)) {
2260     Assembler::movapd(dst, as_Address(src));
2261   } else {
2262     lea(rscratch, src);
2263     Assembler::movapd(dst, Address(rscratch, 0));
2264   }
2265 }
2266 
2267 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
2268   assert(rscratch != noreg || always_reachable(src), "missing");
2269 
2270   if (reachable(src)) {
2271     Assembler::movdqa(dst, as_Address(src));
2272   } else {
2273     lea(rscratch, src);
2274     Assembler::movdqa(dst, Address(rscratch, 0));
2275   }
2276 }
2277 
2278 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2279   assert(rscratch != noreg || always_reachable(src), "missing");
2280 
2281   if (reachable(src)) {
2282     Assembler::movsd(dst, as_Address(src));
2283   } else {
2284     lea(rscratch, src);
2285     Assembler::movsd(dst, Address(rscratch, 0));
2286   }
2287 }
2288 
2289 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2290   assert(rscratch != noreg || always_reachable(src), "missing");
2291 
2292   if (reachable(src)) {
2293     Assembler::movss(dst, as_Address(src));
2294   } else {
2295     lea(rscratch, src);
2296     Assembler::movss(dst, Address(rscratch, 0));
2297   }
2298 }
2299 
2300 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
2301   assert(rscratch != noreg || always_reachable(src), "missing");
2302 
2303   if (reachable(src)) {
2304     Assembler::movddup(dst, as_Address(src));
2305   } else {
2306     lea(rscratch, src);
2307     Assembler::movddup(dst, Address(rscratch, 0));
2308   }
2309 }
2310 
2311 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2312   assert(rscratch != noreg || always_reachable(src), "missing");
2313 
2314   if (reachable(src)) {
2315     Assembler::vmovddup(dst, as_Address(src), vector_len);
2316   } else {
2317     lea(rscratch, src);
2318     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
2319   }
2320 }
2321 
2322 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2323   assert(rscratch != noreg || always_reachable(src), "missing");
2324 
2325   if (reachable(src)) {
2326     Assembler::mulsd(dst, as_Address(src));
2327   } else {
2328     lea(rscratch, src);
2329     Assembler::mulsd(dst, Address(rscratch, 0));
2330   }
2331 }
2332 
2333 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2334   assert(rscratch != noreg || always_reachable(src), "missing");
2335 
2336   if (reachable(src)) {
2337     Assembler::mulss(dst, as_Address(src));
2338   } else {
2339     lea(rscratch, src);
2340     Assembler::mulss(dst, Address(rscratch, 0));
2341   }
2342 }
2343 
2344 void MacroAssembler::null_check(Register reg, int offset) {
2345   if (needs_explicit_null_check(offset)) {
2346     // provoke OS null exception if reg is null by
2347     // accessing M[reg] w/o changing any (non-CC) registers
2348     // NOTE: cmpl is plenty here to provoke a segv
2349     cmpptr(rax, Address(reg, 0));
2350     // Note: should probably use testl(rax, Address(reg, 0));
2351     //       may be shorter code (however, this version of
2352     //       testl needs to be implemented first)
2353   } else {
2354     // nothing to do, (later) access of M[reg + offset]
2355     // will provoke OS null exception if reg is null
2356   }
2357 }
2358 
2359 void MacroAssembler::os_breakpoint() {
2360   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
2361   // (e.g., MSVC can't call ps() otherwise)
2362   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
2363 }
2364 
2365 void MacroAssembler::unimplemented(const char* what) {
2366   const char* buf = nullptr;
2367   {
2368     ResourceMark rm;
2369     stringStream ss;
2370     ss.print("unimplemented: %s", what);
2371     buf = code_string(ss.as_string());
2372   }
2373   stop(buf);
2374 }
2375 
2376 #define XSTATE_BV 0x200
2377 
2378 void MacroAssembler::pop_CPU_state() {
2379   pop_FPU_state();
2380   pop_IU_state();
2381 }
2382 
2383 void MacroAssembler::pop_FPU_state() {
2384   fxrstor(Address(rsp, 0));
2385   addptr(rsp, FPUStateSizeInWords * wordSize);
2386 }
2387 
2388 void MacroAssembler::pop_IU_state() {
2389   popa();
2390   addq(rsp, 8);
2391   popf();
2392 }
2393 
2394 // Save Integer and Float state
2395 // Warning: Stack must be 16 byte aligned (64bit)
2396 void MacroAssembler::push_CPU_state() {
2397   push_IU_state();
2398   push_FPU_state();
2399 }
2400 
2401 void MacroAssembler::push_FPU_state() {
2402   subptr(rsp, FPUStateSizeInWords * wordSize);
2403   fxsave(Address(rsp, 0));
2404 }
2405 
2406 void MacroAssembler::push_IU_state() {
2407   // Push flags first because pusha kills them
2408   pushf();
2409   // Make sure rsp stays 16-byte aligned
2410   subq(rsp, 8);
2411   pusha();
2412 }
2413 
2414 void MacroAssembler::push_cont_fastpath() {
2415   if (!Continuations::enabled()) return;
2416 
2417   Label L_done;
2418   cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset()));
2419   jccb(Assembler::belowEqual, L_done);
2420   movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rsp);
2421   bind(L_done);
2422 }
2423 
2424 void MacroAssembler::pop_cont_fastpath() {
2425   if (!Continuations::enabled()) return;
2426 
2427   Label L_done;
2428   cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset()));
2429   jccb(Assembler::below, L_done);
2430   movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
2431   bind(L_done);
2432 }
2433 
2434 void MacroAssembler::inc_held_monitor_count() {
2435   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
2436 }
2437 
2438 void MacroAssembler::dec_held_monitor_count() {
2439   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
2440 }
2441 
2442 #ifdef ASSERT
2443 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
2444   Label no_cont;
2445   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
2446   testl(cont, cont);
2447   jcc(Assembler::zero, no_cont);
2448   stop(name);
2449   bind(no_cont);
2450 }
2451 #endif
2452 
2453 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { // determine java_thread register
2454   // we must set sp to zero to clear frame
2455   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
2456   // must clear fp, so that compiled frames are not confused; it is
2457   // possible that we need it only for debugging
2458   if (clear_fp) {
2459     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
2460   }
2461   // Always clear the pc because it could have been set by make_walkable()
2462   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
2463   vzeroupper();
2464 }
2465 
2466 void MacroAssembler::round_to(Register reg, int modulus) {
2467   addptr(reg, modulus - 1);
2468   andptr(reg, -modulus);
2469 }
2470 
2471 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod) {
2472   if (at_return) {
2473     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
2474     // we may safely use rsp instead to perform the stack watermark check.
2475     cmpptr(in_nmethod ? rsp : rbp, Address(r15_thread, JavaThread::polling_word_offset()));
2476     jcc(Assembler::above, slow_path);
2477     return;
2478   }
2479   testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2480   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
2481 }
2482 
2483 // Calls to C land
2484 //
2485 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
2486 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
2487 // has to be reset to 0. This is required to allow proper stack traversal.
2488 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
2489                                          Register last_java_fp,
2490                                          address  last_java_pc,
2491                                          Register rscratch) {
2492   vzeroupper();
2493   // determine last_java_sp register
2494   if (!last_java_sp->is_valid()) {
2495     last_java_sp = rsp;
2496   }
2497   // last_java_fp is optional
2498   if (last_java_fp->is_valid()) {
2499     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
2500   }
2501   // last_java_pc is optional
2502   if (last_java_pc != nullptr) {
2503     Address java_pc(r15_thread,
2504                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
2505     lea(java_pc, InternalAddress(last_java_pc), rscratch);
2506   }
2507   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
2508 }
2509 
2510 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
2511                                          Register last_java_fp,
2512                                          Label &L,
2513                                          Register scratch) {
2514   lea(scratch, L);
2515   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch);
2516   set_last_Java_frame(last_java_sp, last_java_fp, nullptr, scratch);
2517 }
2518 
2519 void MacroAssembler::shlptr(Register dst, int imm8) {
2520   shlq(dst, imm8);
2521 }
2522 
2523 void MacroAssembler::shrptr(Register dst, int imm8) {
2524   shrq(dst, imm8);
2525 }
2526 
2527 void MacroAssembler::sign_extend_byte(Register reg) {
2528   movsbl(reg, reg); // movsxb
2529 }
2530 
2531 void MacroAssembler::sign_extend_short(Register reg) {
2532   movswl(reg, reg); // movsxw
2533 }
2534 
2535 void MacroAssembler::testl(Address dst, int32_t imm32) {
2536   if (imm32 >= 0 && is8bit(imm32)) {
2537     testb(dst, imm32);
2538   } else {
2539     Assembler::testl(dst, imm32);
2540   }
2541 }
2542 
2543 void MacroAssembler::testl(Register dst, int32_t imm32) {
2544   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
2545     testb(dst, imm32);
2546   } else {
2547     Assembler::testl(dst, imm32);
2548   }
2549 }
2550 
2551 void MacroAssembler::testl(Register dst, AddressLiteral src) {
2552   assert(always_reachable(src), "Address should be reachable");
2553   testl(dst, as_Address(src));
2554 }
2555 
2556 void MacroAssembler::testq(Address dst, int32_t imm32) {
2557   if (imm32 >= 0) {
2558     testl(dst, imm32);
2559   } else {
2560     Assembler::testq(dst, imm32);
2561   }
2562 }
2563 
2564 void MacroAssembler::testq(Register dst, int32_t imm32) {
2565   if (imm32 >= 0) {
2566     testl(dst, imm32);
2567   } else {
2568     Assembler::testq(dst, imm32);
2569   }
2570 }
2571 
2572 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
2573   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2574   Assembler::pcmpeqb(dst, src);
2575 }
2576 
2577 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
2578   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2579   Assembler::pcmpeqw(dst, src);
2580 }
2581 
2582 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2583   assert((dst->encoding() < 16),"XMM register should be 0-15");
2584   Assembler::pcmpestri(dst, src, imm8);
2585 }
2586 
2587 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2588   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2589   Assembler::pcmpestri(dst, src, imm8);
2590 }
2591 
2592 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2593   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2594   Assembler::pmovzxbw(dst, src);
2595 }
2596 
2597 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
2598   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2599   Assembler::pmovzxbw(dst, src);
2600 }
2601 
2602 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
2603   assert((src->encoding() < 16),"XMM register should be 0-15");
2604   Assembler::pmovmskb(dst, src);
2605 }
2606 
2607 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
2608   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2609   Assembler::ptest(dst, src);
2610 }
2611 
2612 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2613   assert(rscratch != noreg || always_reachable(src), "missing");
2614 
2615   if (reachable(src)) {
2616     Assembler::sqrtss(dst, as_Address(src));
2617   } else {
2618     lea(rscratch, src);
2619     Assembler::sqrtss(dst, Address(rscratch, 0));
2620   }
2621 }
2622 
2623 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2624   assert(rscratch != noreg || always_reachable(src), "missing");
2625 
2626   if (reachable(src)) {
2627     Assembler::subsd(dst, as_Address(src));
2628   } else {
2629     lea(rscratch, src);
2630     Assembler::subsd(dst, Address(rscratch, 0));
2631   }
2632 }
2633 
2634 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
2635   assert(rscratch != noreg || always_reachable(src), "missing");
2636 
2637   if (reachable(src)) {
2638     Assembler::roundsd(dst, as_Address(src), rmode);
2639   } else {
2640     lea(rscratch, src);
2641     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
2642   }
2643 }
2644 
2645 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2646   assert(rscratch != noreg || always_reachable(src), "missing");
2647 
2648   if (reachable(src)) {
2649     Assembler::subss(dst, as_Address(src));
2650   } else {
2651     lea(rscratch, src);
2652     Assembler::subss(dst, Address(rscratch, 0));
2653   }
2654 }
2655 
2656 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2657   assert(rscratch != noreg || always_reachable(src), "missing");
2658 
2659   if (reachable(src)) {
2660     Assembler::ucomisd(dst, as_Address(src));
2661   } else {
2662     lea(rscratch, src);
2663     Assembler::ucomisd(dst, Address(rscratch, 0));
2664   }
2665 }
2666 
2667 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2668   assert(rscratch != noreg || always_reachable(src), "missing");
2669 
2670   if (reachable(src)) {
2671     Assembler::ucomiss(dst, as_Address(src));
2672   } else {
2673     lea(rscratch, src);
2674     Assembler::ucomiss(dst, Address(rscratch, 0));
2675   }
2676 }
2677 
2678 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2679   assert(rscratch != noreg || always_reachable(src), "missing");
2680 
2681   // Used in sign-bit flipping with aligned address.
2682   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
2683 
2684   if (UseAVX > 2 &&
2685       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2686       (dst->encoding() >= 16)) {
2687     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
2688   } else if (reachable(src)) {
2689     Assembler::xorpd(dst, as_Address(src));
2690   } else {
2691     lea(rscratch, src);
2692     Assembler::xorpd(dst, Address(rscratch, 0));
2693   }
2694 }
2695 
2696 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
2697   if (UseAVX > 2 &&
2698       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2699       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
2700     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
2701   } else {
2702     Assembler::xorpd(dst, src);
2703   }
2704 }
2705 
2706 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
2707   if (UseAVX > 2 &&
2708       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2709       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
2710     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
2711   } else {
2712     Assembler::xorps(dst, src);
2713   }
2714 }
2715 
2716 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
2717   assert(rscratch != noreg || always_reachable(src), "missing");
2718 
2719   // Used in sign-bit flipping with aligned address.
2720   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
2721 
2722   if (UseAVX > 2 &&
2723       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2724       (dst->encoding() >= 16)) {
2725     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
2726   } else if (reachable(src)) {
2727     Assembler::xorps(dst, as_Address(src));
2728   } else {
2729     lea(rscratch, src);
2730     Assembler::xorps(dst, Address(rscratch, 0));
2731   }
2732 }
2733 
2734 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
2735   assert(rscratch != noreg || always_reachable(src), "missing");
2736 
2737   // Used in sign-bit flipping with aligned address.
2738   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
2739   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
2740   if (reachable(src)) {
2741     Assembler::pshufb(dst, as_Address(src));
2742   } else {
2743     lea(rscratch, src);
2744     Assembler::pshufb(dst, Address(rscratch, 0));
2745   }
2746 }
2747 
2748 // AVX 3-operands instructions
2749 
2750 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
2751   assert(rscratch != noreg || always_reachable(src), "missing");
2752 
2753   if (reachable(src)) {
2754     vaddsd(dst, nds, as_Address(src));
2755   } else {
2756     lea(rscratch, src);
2757     vaddsd(dst, nds, Address(rscratch, 0));
2758   }
2759 }
2760 
2761 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
2762   assert(rscratch != noreg || always_reachable(src), "missing");
2763 
2764   if (reachable(src)) {
2765     vaddss(dst, nds, as_Address(src));
2766   } else {
2767     lea(rscratch, src);
2768     vaddss(dst, nds, Address(rscratch, 0));
2769   }
2770 }
2771 
2772 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2773   assert(UseAVX > 0, "requires some form of AVX");
2774   assert(rscratch != noreg || always_reachable(src), "missing");
2775 
2776   if (reachable(src)) {
2777     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
2778   } else {
2779     lea(rscratch, src);
2780     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
2781   }
2782 }
2783 
2784 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2785   assert(UseAVX > 0, "requires some form of AVX");
2786   assert(rscratch != noreg || always_reachable(src), "missing");
2787 
2788   if (reachable(src)) {
2789     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
2790   } else {
2791     lea(rscratch, src);
2792     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
2793   }
2794 }
2795 
2796 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
2797   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
2798   assert(rscratch != noreg || always_reachable(negate_field), "missing");
2799 
2800   vandps(dst, nds, negate_field, vector_len, rscratch);
2801 }
2802 
2803 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
2804   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
2805   assert(rscratch != noreg || always_reachable(negate_field), "missing");
2806 
2807   vandpd(dst, nds, negate_field, vector_len, rscratch);
2808 }
2809 
2810 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2811   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2812   Assembler::vpaddb(dst, nds, src, vector_len);
2813 }
2814 
2815 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2816   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2817   Assembler::vpaddb(dst, nds, src, vector_len);
2818 }
2819 
2820 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2821   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2822   Assembler::vpaddw(dst, nds, src, vector_len);
2823 }
2824 
2825 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2826   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2827   Assembler::vpaddw(dst, nds, src, vector_len);
2828 }
2829 
2830 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2831   assert(rscratch != noreg || always_reachable(src), "missing");
2832 
2833   if (reachable(src)) {
2834     Assembler::vpand(dst, nds, as_Address(src), vector_len);
2835   } else {
2836     lea(rscratch, src);
2837     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
2838   }
2839 }
2840 
2841 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2842   assert(rscratch != noreg || always_reachable(src), "missing");
2843 
2844   if (reachable(src)) {
2845     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
2846   } else {
2847     lea(rscratch, src);
2848     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
2849   }
2850 }
2851 
2852 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2853   assert(rscratch != noreg || always_reachable(src), "missing");
2854 
2855   if (reachable(src)) {
2856     Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
2857   } else {
2858     lea(rscratch, src);
2859     Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
2860   }
2861 }
2862 
2863 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2864   assert(rscratch != noreg || always_reachable(src), "missing");
2865 
2866   if (reachable(src)) {
2867     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
2868   } else {
2869     lea(rscratch, src);
2870     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
2871   }
2872 }
2873 
2874 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2875   assert(rscratch != noreg || always_reachable(src), "missing");
2876 
2877   if (reachable(src)) {
2878     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
2879   } else {
2880     lea(rscratch, src);
2881     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
2882   }
2883 }
2884 
2885 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2886   assert(rscratch != noreg || always_reachable(src), "missing");
2887 
2888   if (reachable(src)) {
2889     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
2890   } else {
2891     lea(rscratch, src);
2892     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
2893   }
2894 }
2895 
2896 // Vector float blend
2897 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
2898 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
2899   // WARN: Allow dst == (src1|src2), mask == scratch
2900   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
2901   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
2902   bool dst_available = dst != mask && (dst != src1 || dst != src2);
2903   if (blend_emulation && scratch_available && dst_available) {
2904     if (compute_mask) {
2905       vpsrad(scratch, mask, 32, vector_len);
2906       mask = scratch;
2907     }
2908     if (dst == src1) {
2909       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
2910       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
2911     } else {
2912       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
2913       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
2914     }
2915     vpor(dst, dst, scratch, vector_len);
2916   } else {
2917     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
2918   }
2919 }
2920 
2921 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
2922 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
2923   // WARN: Allow dst == (src1|src2), mask == scratch
2924   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
2925   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
2926   bool dst_available = dst != mask && (dst != src1 || dst != src2);
2927   if (blend_emulation && scratch_available && dst_available) {
2928     if (compute_mask) {
2929       vpxor(scratch, scratch, scratch, vector_len);
2930       vpcmpgtq(scratch, scratch, mask, vector_len);
2931       mask = scratch;
2932     }
2933     if (dst == src1) {
2934       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
2935       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
2936     } else {
2937       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
2938       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
2939     }
2940     vpor(dst, dst, scratch, vector_len);
2941   } else {
2942     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
2943   }
2944 }
2945 
2946 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2947   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2948   Assembler::vpcmpeqb(dst, nds, src, vector_len);
2949 }
2950 
2951 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
2952   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2953   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
2954 }
2955 
2956 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2957   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2958   Assembler::vpcmpeqw(dst, nds, src, vector_len);
2959 }
2960 
2961 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2962   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2963   Assembler::vpcmpeqw(dst, nds, src, vector_len);
2964 }
2965 
2966 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2967   assert(rscratch != noreg || always_reachable(src), "missing");
2968 
2969   if (reachable(src)) {
2970     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
2971   } else {
2972     lea(rscratch, src);
2973     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
2974   }
2975 }
2976 
2977 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
2978                              int comparison, bool is_signed, int vector_len, Register rscratch) {
2979   assert(rscratch != noreg || always_reachable(src), "missing");
2980 
2981   if (reachable(src)) {
2982     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
2983   } else {
2984     lea(rscratch, src);
2985     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
2986   }
2987 }
2988 
2989 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
2990                              int comparison, bool is_signed, int vector_len, Register rscratch) {
2991   assert(rscratch != noreg || always_reachable(src), "missing");
2992 
2993   if (reachable(src)) {
2994     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
2995   } else {
2996     lea(rscratch, src);
2997     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
2998   }
2999 }
3000 
3001 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3002                              int comparison, bool is_signed, int vector_len, Register rscratch) {
3003   assert(rscratch != noreg || always_reachable(src), "missing");
3004 
3005   if (reachable(src)) {
3006     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3007   } else {
3008     lea(rscratch, src);
3009     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
3010   }
3011 }
3012 
3013 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3014                              int comparison, bool is_signed, int vector_len, Register rscratch) {
3015   assert(rscratch != noreg || always_reachable(src), "missing");
3016 
3017   if (reachable(src)) {
3018     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3019   } else {
3020     lea(rscratch, src);
3021     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
3022   }
3023 }
3024 
3025 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
3026   if (width == Assembler::Q) {
3027     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
3028   } else {
3029     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
3030   }
3031 }
3032 
3033 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
3034   int eq_cond_enc = 0x29;
3035   int gt_cond_enc = 0x37;
3036   if (width != Assembler::Q) {
3037     eq_cond_enc = 0x74 + width;
3038     gt_cond_enc = 0x64 + width;
3039   }
3040   switch (cond) {
3041   case eq:
3042     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3043     break;
3044   case neq:
3045     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3046     vallones(xtmp, vector_len);
3047     vpxor(dst, xtmp, dst, vector_len);
3048     break;
3049   case le:
3050     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3051     vallones(xtmp, vector_len);
3052     vpxor(dst, xtmp, dst, vector_len);
3053     break;
3054   case nlt:
3055     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3056     vallones(xtmp, vector_len);
3057     vpxor(dst, xtmp, dst, vector_len);
3058     break;
3059   case lt:
3060     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3061     break;
3062   case nle:
3063     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3064     break;
3065   default:
3066     assert(false, "Should not reach here");
3067   }
3068 }
3069 
3070 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3071   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3072   Assembler::vpmovzxbw(dst, src, vector_len);
3073 }
3074 
3075 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
3076   assert((src->encoding() < 16),"XMM register should be 0-15");
3077   Assembler::vpmovmskb(dst, src, vector_len);
3078 }
3079 
3080 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3081   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3082   Assembler::vpmullw(dst, nds, src, vector_len);
3083 }
3084 
3085 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3086   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3087   Assembler::vpmullw(dst, nds, src, vector_len);
3088 }
3089 
3090 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3091   assert((UseAVX > 0), "AVX support is needed");
3092   assert(rscratch != noreg || always_reachable(src), "missing");
3093 
3094   if (reachable(src)) {
3095     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
3096   } else {
3097     lea(rscratch, src);
3098     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
3099   }
3100 }
3101 
3102 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3103   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3104   Assembler::vpsubb(dst, nds, src, vector_len);
3105 }
3106 
3107 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3108   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3109   Assembler::vpsubb(dst, nds, src, vector_len);
3110 }
3111 
3112 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3113   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3114   Assembler::vpsubw(dst, nds, src, vector_len);
3115 }
3116 
3117 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3118   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3119   Assembler::vpsubw(dst, nds, src, vector_len);
3120 }
3121 
3122 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3123   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3124   Assembler::vpsraw(dst, nds, shift, vector_len);
3125 }
3126 
3127 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3128   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3129   Assembler::vpsraw(dst, nds, shift, vector_len);
3130 }
3131 
3132 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3133   assert(UseAVX > 2,"");
3134   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3135      vector_len = 2;
3136   }
3137   Assembler::evpsraq(dst, nds, shift, vector_len);
3138 }
3139 
3140 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3141   assert(UseAVX > 2,"");
3142   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3143      vector_len = 2;
3144   }
3145   Assembler::evpsraq(dst, nds, shift, vector_len);
3146 }
3147 
3148 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3149   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3150   Assembler::vpsrlw(dst, nds, shift, vector_len);
3151 }
3152 
3153 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3154   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3155   Assembler::vpsrlw(dst, nds, shift, vector_len);
3156 }
3157 
3158 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3159   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3160   Assembler::vpsllw(dst, nds, shift, vector_len);
3161 }
3162 
3163 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3164   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3165   Assembler::vpsllw(dst, nds, shift, vector_len);
3166 }
3167 
3168 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
3169   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
3170   Assembler::vptest(dst, src);
3171 }
3172 
3173 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3174   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3175   Assembler::punpcklbw(dst, src);
3176 }
3177 
3178 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
3179   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3180   Assembler::pshufd(dst, src, mode);
3181 }
3182 
3183 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3184   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3185   Assembler::pshuflw(dst, src, mode);
3186 }
3187 
3188 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3189   assert(rscratch != noreg || always_reachable(src), "missing");
3190 
3191   if (reachable(src)) {
3192     vandpd(dst, nds, as_Address(src), vector_len);
3193   } else {
3194     lea(rscratch, src);
3195     vandpd(dst, nds, Address(rscratch, 0), vector_len);
3196   }
3197 }
3198 
3199 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3200   assert(rscratch != noreg || always_reachable(src), "missing");
3201 
3202   if (reachable(src)) {
3203     vandps(dst, nds, as_Address(src), vector_len);
3204   } else {
3205     lea(rscratch, src);
3206     vandps(dst, nds, Address(rscratch, 0), vector_len);
3207   }
3208 }
3209 
3210 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
3211                             bool merge, int vector_len, Register rscratch) {
3212   assert(rscratch != noreg || always_reachable(src), "missing");
3213 
3214   if (reachable(src)) {
3215     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
3216   } else {
3217     lea(rscratch, src);
3218     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
3219   }
3220 }
3221 
3222 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3223   assert(rscratch != noreg || always_reachable(src), "missing");
3224 
3225   if (reachable(src)) {
3226     vdivsd(dst, nds, as_Address(src));
3227   } else {
3228     lea(rscratch, src);
3229     vdivsd(dst, nds, Address(rscratch, 0));
3230   }
3231 }
3232 
3233 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3234   assert(rscratch != noreg || always_reachable(src), "missing");
3235 
3236   if (reachable(src)) {
3237     vdivss(dst, nds, as_Address(src));
3238   } else {
3239     lea(rscratch, src);
3240     vdivss(dst, nds, Address(rscratch, 0));
3241   }
3242 }
3243 
3244 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3245   assert(rscratch != noreg || always_reachable(src), "missing");
3246 
3247   if (reachable(src)) {
3248     vmulsd(dst, nds, as_Address(src));
3249   } else {
3250     lea(rscratch, src);
3251     vmulsd(dst, nds, Address(rscratch, 0));
3252   }
3253 }
3254 
3255 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3256   assert(rscratch != noreg || always_reachable(src), "missing");
3257 
3258   if (reachable(src)) {
3259     vmulss(dst, nds, as_Address(src));
3260   } else {
3261     lea(rscratch, src);
3262     vmulss(dst, nds, Address(rscratch, 0));
3263   }
3264 }
3265 
3266 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3267   assert(rscratch != noreg || always_reachable(src), "missing");
3268 
3269   if (reachable(src)) {
3270     vsubsd(dst, nds, as_Address(src));
3271   } else {
3272     lea(rscratch, src);
3273     vsubsd(dst, nds, Address(rscratch, 0));
3274   }
3275 }
3276 
3277 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3278   assert(rscratch != noreg || always_reachable(src), "missing");
3279 
3280   if (reachable(src)) {
3281     vsubss(dst, nds, as_Address(src));
3282   } else {
3283     lea(rscratch, src);
3284     vsubss(dst, nds, Address(rscratch, 0));
3285   }
3286 }
3287 
3288 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3289   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3290   assert(rscratch != noreg || always_reachable(src), "missing");
3291 
3292   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
3293 }
3294 
3295 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3296   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3297   assert(rscratch != noreg || always_reachable(src), "missing");
3298 
3299   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
3300 }
3301 
3302 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3303   assert(rscratch != noreg || always_reachable(src), "missing");
3304 
3305   if (reachable(src)) {
3306     vxorpd(dst, nds, as_Address(src), vector_len);
3307   } else {
3308     lea(rscratch, src);
3309     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
3310   }
3311 }
3312 
3313 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3314   assert(rscratch != noreg || always_reachable(src), "missing");
3315 
3316   if (reachable(src)) {
3317     vxorps(dst, nds, as_Address(src), vector_len);
3318   } else {
3319     lea(rscratch, src);
3320     vxorps(dst, nds, Address(rscratch, 0), vector_len);
3321   }
3322 }
3323 
3324 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3325   assert(rscratch != noreg || always_reachable(src), "missing");
3326 
3327   if (UseAVX > 1 || (vector_len < 1)) {
3328     if (reachable(src)) {
3329       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
3330     } else {
3331       lea(rscratch, src);
3332       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
3333     }
3334   } else {
3335     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
3336   }
3337 }
3338 
3339 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3340   assert(rscratch != noreg || always_reachable(src), "missing");
3341 
3342   if (reachable(src)) {
3343     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
3344   } else {
3345     lea(rscratch, src);
3346     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
3347   }
3348 }
3349 
3350 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
3351   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
3352   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
3353   // The inverted mask is sign-extended
3354   andptr(possibly_non_local, inverted_mask);
3355 }
3356 
3357 void MacroAssembler::resolve_jobject(Register value,
3358                                      Register tmp) {
3359   Register thread = r15_thread;
3360   assert_different_registers(value, thread, tmp);
3361   Label done, tagged, weak_tagged;
3362   testptr(value, value);
3363   jcc(Assembler::zero, done);           // Use null as-is.
3364   testptr(value, JNIHandles::tag_mask); // Test for tag.
3365   jcc(Assembler::notZero, tagged);
3366 
3367   // Resolve local handle
3368   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp);
3369   verify_oop(value);
3370   jmp(done);
3371 
3372   bind(tagged);
3373   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
3374   jcc(Assembler::notZero, weak_tagged);
3375 
3376   // Resolve global handle
3377   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp);
3378   verify_oop(value);
3379   jmp(done);
3380 
3381   bind(weak_tagged);
3382   // Resolve jweak.
3383   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3384                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp);
3385   verify_oop(value);
3386 
3387   bind(done);
3388 }
3389 
3390 void MacroAssembler::resolve_global_jobject(Register value,
3391                                             Register tmp) {
3392   Register thread = r15_thread;
3393   assert_different_registers(value, thread, tmp);
3394   Label done;
3395 
3396   testptr(value, value);
3397   jcc(Assembler::zero, done);           // Use null as-is.
3398 
3399 #ifdef ASSERT
3400   {
3401     Label valid_global_tag;
3402     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
3403     jcc(Assembler::notZero, valid_global_tag);
3404     stop("non global jobject using resolve_global_jobject");
3405     bind(valid_global_tag);
3406   }
3407 #endif
3408 
3409   // Resolve global handle
3410   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp);
3411   verify_oop(value);
3412 
3413   bind(done);
3414 }
3415 
3416 void MacroAssembler::subptr(Register dst, int32_t imm32) {
3417   subq(dst, imm32);
3418 }
3419 
3420 // Force generation of a 4 byte immediate value even if it fits into 8bit
3421 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
3422   subq_imm32(dst, imm32);
3423 }
3424 
3425 void MacroAssembler::subptr(Register dst, Register src) {
3426   subq(dst, src);
3427 }
3428 
3429 // C++ bool manipulation
3430 void MacroAssembler::testbool(Register dst) {
3431   if(sizeof(bool) == 1)
3432     testb(dst, 0xff);
3433   else if(sizeof(bool) == 2) {
3434     // testw implementation needed for two byte bools
3435     ShouldNotReachHere();
3436   } else if(sizeof(bool) == 4)
3437     testl(dst, dst);
3438   else
3439     // unsupported
3440     ShouldNotReachHere();
3441 }
3442 
3443 void MacroAssembler::testptr(Register dst, Register src) {
3444   testq(dst, src);
3445 }
3446 
3447 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3448 void MacroAssembler::tlab_allocate(Register obj,
3449                                    Register var_size_in_bytes,
3450                                    int con_size_in_bytes,
3451                                    Register t1,
3452                                    Register t2,
3453                                    Label& slow_case) {
3454   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3455   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3456 }
3457 
3458 RegSet MacroAssembler::call_clobbered_gp_registers() {
3459   RegSet regs;
3460   regs += RegSet::of(rax, rcx, rdx);
3461 #ifndef _WINDOWS
3462   regs += RegSet::of(rsi, rdi);
3463 #endif
3464   regs += RegSet::range(r8, r11);
3465   if (UseAPX) {
3466     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
3467   }
3468   return regs;
3469 }
3470 
3471 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
3472   int num_xmm_registers = XMMRegister::available_xmm_registers();
3473 #if defined(_WINDOWS)
3474   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
3475   if (num_xmm_registers > 16) {
3476      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
3477   }
3478   return result;
3479 #else
3480   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
3481 #endif
3482 }
3483 
3484 // C1 only ever uses the first double/float of the XMM register.
3485 static int xmm_save_size() { return sizeof(double); }
3486 
3487 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
3488   masm->movdbl(Address(rsp, offset), reg);
3489 }
3490 
3491 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
3492   masm->movdbl(reg, Address(rsp, offset));
3493 }
3494 
3495 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
3496                                   bool save_fpu, int& gp_area_size, int& xmm_area_size) {
3497 
3498   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
3499                          StackAlignmentInBytes);
3500   xmm_area_size = save_fpu ? xmm_registers.size() * xmm_save_size() : 0;
3501 
3502   return gp_area_size + xmm_area_size;
3503 }
3504 
3505 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
3506   block_comment("push_call_clobbered_registers start");
3507   // Regular registers
3508   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
3509 
3510   int gp_area_size;
3511   int xmm_area_size;
3512   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
3513                                                gp_area_size, xmm_area_size);
3514   subptr(rsp, total_save_size);
3515 
3516   push_set(gp_registers_to_push, 0);
3517 
3518   if (save_fpu) {
3519     push_set(call_clobbered_xmm_registers(), gp_area_size);
3520   }
3521 
3522   block_comment("push_call_clobbered_registers end");
3523 }
3524 
3525 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
3526   block_comment("pop_call_clobbered_registers start");
3527 
3528   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
3529 
3530   int gp_area_size;
3531   int xmm_area_size;
3532   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
3533                                                gp_area_size, xmm_area_size);
3534 
3535   if (restore_fpu) {
3536     pop_set(call_clobbered_xmm_registers(), gp_area_size);
3537   }
3538 
3539   pop_set(gp_registers_to_pop, 0);
3540 
3541   addptr(rsp, total_save_size);
3542 
3543   vzeroupper();
3544 
3545   block_comment("pop_call_clobbered_registers end");
3546 }
3547 
3548 void MacroAssembler::push_set(XMMRegSet set, int offset) {
3549   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
3550   int spill_offset = offset;
3551 
3552   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
3553     save_xmm_register(this, spill_offset, *it);
3554     spill_offset += xmm_save_size();
3555   }
3556 }
3557 
3558 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
3559   int restore_size = set.size() * xmm_save_size();
3560   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
3561 
3562   int restore_offset = offset + restore_size - xmm_save_size();
3563 
3564   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
3565     restore_xmm_register(this, restore_offset, *it);
3566     restore_offset -= xmm_save_size();
3567   }
3568 }
3569 
3570 void MacroAssembler::push_set(RegSet set, int offset) {
3571   int spill_offset;
3572   if (offset == -1) {
3573     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3574     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
3575     subptr(rsp, aligned_size);
3576     spill_offset = 0;
3577   } else {
3578     spill_offset = offset;
3579   }
3580 
3581   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
3582     movptr(Address(rsp, spill_offset), *it);
3583     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3584   }
3585 }
3586 
3587 void MacroAssembler::pop_set(RegSet set, int offset) {
3588 
3589   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3590   int restore_size = set.size() * gp_reg_size;
3591   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
3592 
3593   int restore_offset;
3594   if (offset == -1) {
3595     restore_offset = restore_size - gp_reg_size;
3596   } else {
3597     restore_offset = offset + restore_size - gp_reg_size;
3598   }
3599   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
3600     movptr(*it, Address(rsp, restore_offset));
3601     restore_offset -= gp_reg_size;
3602   }
3603 
3604   if (offset == -1) {
3605     addptr(rsp, aligned_size);
3606   }
3607 }
3608 
3609 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3610 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3611   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3612   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3613   Label done;
3614 
3615   testptr(length_in_bytes, length_in_bytes);
3616   jcc(Assembler::zero, done);
3617 
3618   // initialize topmost word, divide index by 2, check if odd and test if zero
3619   // note: for the remaining code to work, index must be a multiple of BytesPerWord
3620 #ifdef ASSERT
3621   {
3622     Label L;
3623     testptr(length_in_bytes, BytesPerWord - 1);
3624     jcc(Assembler::zero, L);
3625     stop("length must be a multiple of BytesPerWord");
3626     bind(L);
3627   }
3628 #endif
3629   Register index = length_in_bytes;
3630   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
3631   if (UseIncDec) {
3632     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
3633   } else {
3634     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
3635     shrptr(index, 1);
3636   }
3637 
3638   // initialize remaining object fields: index is a multiple of 2 now
3639   {
3640     Label loop;
3641     bind(loop);
3642     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
3643     decrement(index);
3644     jcc(Assembler::notZero, loop);
3645   }
3646 
3647   bind(done);
3648 }
3649 
3650 // Look up the method for a megamorphic invokeinterface call.
3651 // The target method is determined by <intf_klass, itable_index>.
3652 // The receiver klass is in recv_klass.
3653 // On success, the result will be in method_result, and execution falls through.
3654 // On failure, execution transfers to the given label.
3655 void MacroAssembler::lookup_interface_method(Register recv_klass,
3656                                              Register intf_klass,
3657                                              RegisterOrConstant itable_index,
3658                                              Register method_result,
3659                                              Register scan_temp,
3660                                              Label& L_no_such_interface,
3661                                              bool return_method) {
3662   assert_different_registers(recv_klass, intf_klass, scan_temp);
3663   assert_different_registers(method_result, intf_klass, scan_temp);
3664   assert(recv_klass != method_result || !return_method,
3665          "recv_klass can be destroyed when method isn't needed");
3666 
3667   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
3668          "caller must use same register for non-constant itable index as for method");
3669 
3670   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
3671   int vtable_base = in_bytes(Klass::vtable_start_offset());
3672   int itentry_off = in_bytes(itableMethodEntry::method_offset());
3673   int scan_step   = itableOffsetEntry::size() * wordSize;
3674   int vte_size    = vtableEntry::size_in_bytes();
3675   Address::ScaleFactor times_vte_scale = Address::times_ptr;
3676   assert(vte_size == wordSize, "else adjust times_vte_scale");
3677 
3678   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
3679 
3680   // Could store the aligned, prescaled offset in the klass.
3681   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
3682 
3683   if (return_method) {
3684     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
3685     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
3686     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
3687   }
3688 
3689   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
3690   //   if (scan->interface() == intf) {
3691   //     result = (klass + scan->offset() + itable_index);
3692   //   }
3693   // }
3694   Label search, found_method;
3695 
3696   for (int peel = 1; peel >= 0; peel--) {
3697     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
3698     cmpptr(intf_klass, method_result);
3699 
3700     if (peel) {
3701       jccb(Assembler::equal, found_method);
3702     } else {
3703       jccb(Assembler::notEqual, search);
3704       // (invert the test to fall through to found_method...)
3705     }
3706 
3707     if (!peel)  break;
3708 
3709     bind(search);
3710 
3711     // Check that the previous entry is non-null.  A null entry means that
3712     // the receiver class doesn't implement the interface, and wasn't the
3713     // same as when the caller was compiled.
3714     testptr(method_result, method_result);
3715     jcc(Assembler::zero, L_no_such_interface);
3716     addptr(scan_temp, scan_step);
3717   }
3718 
3719   bind(found_method);
3720 
3721   if (return_method) {
3722     // Got a hit.
3723     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
3724     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
3725   }
3726 }
3727 
3728 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
3729 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
3730 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
3731 // The target method is determined by <holder_klass, itable_index>.
3732 // The receiver klass is in recv_klass.
3733 // On success, the result will be in method_result, and execution falls through.
3734 // On failure, execution transfers to the given label.
3735 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
3736                                                   Register holder_klass,
3737                                                   Register resolved_klass,
3738                                                   Register method_result,
3739                                                   Register scan_temp,
3740                                                   Register temp_reg2,
3741                                                   Register receiver,
3742                                                   int itable_index,
3743                                                   Label& L_no_such_interface) {
3744   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
3745   Register temp_itbl_klass = method_result;
3746   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
3747 
3748   int vtable_base = in_bytes(Klass::vtable_start_offset());
3749   int itentry_off = in_bytes(itableMethodEntry::method_offset());
3750   int scan_step = itableOffsetEntry::size() * wordSize;
3751   int vte_size = vtableEntry::size_in_bytes();
3752   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
3753   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
3754   Address::ScaleFactor times_vte_scale = Address::times_ptr;
3755   assert(vte_size == wordSize, "adjust times_vte_scale");
3756 
3757   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
3758 
3759   // temp_itbl_klass = recv_klass.itable[0]
3760   // scan_temp = &recv_klass.itable[0] + step
3761   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
3762   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
3763   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
3764   xorptr(temp_reg, temp_reg);
3765 
3766   // Initial checks:
3767   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
3768   //   - if (itable[0] == 0), no such interface
3769   //   - if (itable[0] == holder_klass), shortcut to "holder found"
3770   cmpptr(holder_klass, resolved_klass);
3771   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
3772   testptr(temp_itbl_klass, temp_itbl_klass);
3773   jccb(Assembler::zero, L_no_such_interface);
3774   cmpptr(holder_klass, temp_itbl_klass);
3775   jccb(Assembler::equal, L_holder_found);
3776 
3777   // Loop: Look for holder_klass record in itable
3778   //   do {
3779   //     tmp = itable[index];
3780   //     index += step;
3781   //     if (tmp == holder_klass) {
3782   //       goto L_holder_found; // Found!
3783   //     }
3784   //   } while (tmp != 0);
3785   //   goto L_no_such_interface // Not found.
3786   Label L_scan_holder;
3787   bind(L_scan_holder);
3788     movptr(temp_itbl_klass, Address(scan_temp, 0));
3789     addptr(scan_temp, scan_step);
3790     cmpptr(holder_klass, temp_itbl_klass);
3791     jccb(Assembler::equal, L_holder_found);
3792     testptr(temp_itbl_klass, temp_itbl_klass);
3793     jccb(Assembler::notZero, L_scan_holder);
3794 
3795   jmpb(L_no_such_interface);
3796 
3797   // Loop: Look for resolved_class record in itable
3798   //   do {
3799   //     tmp = itable[index];
3800   //     index += step;
3801   //     if (tmp == holder_klass) {
3802   //        // Also check if we have met a holder klass
3803   //        holder_tmp = itable[index-step-ioffset];
3804   //     }
3805   //     if (tmp == resolved_klass) {
3806   //        goto L_resolved_found;  // Found!
3807   //     }
3808   //   } while (tmp != 0);
3809   //   goto L_no_such_interface // Not found.
3810   //
3811   Label L_loop_scan_resolved;
3812   bind(L_loop_scan_resolved);
3813     movptr(temp_itbl_klass, Address(scan_temp, 0));
3814     addptr(scan_temp, scan_step);
3815     bind(L_loop_scan_resolved_entry);
3816     cmpptr(holder_klass, temp_itbl_klass);
3817     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
3818     cmpptr(resolved_klass, temp_itbl_klass);
3819     jccb(Assembler::equal, L_resolved_found);
3820     testptr(temp_itbl_klass, temp_itbl_klass);
3821     jccb(Assembler::notZero, L_loop_scan_resolved);
3822 
3823   jmpb(L_no_such_interface);
3824 
3825   Label L_ready;
3826 
3827   // See if we already have a holder klass. If not, go and scan for it.
3828   bind(L_resolved_found);
3829   testptr(temp_reg, temp_reg);
3830   jccb(Assembler::zero, L_scan_holder);
3831   jmpb(L_ready);
3832 
3833   bind(L_holder_found);
3834   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
3835 
3836   // Finally, temp_reg contains holder_klass vtable offset
3837   bind(L_ready);
3838   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
3839   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
3840     load_klass(scan_temp, receiver, noreg);
3841     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
3842   } else {
3843     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
3844   }
3845 }
3846 
3847 
3848 // virtual method calling
3849 void MacroAssembler::lookup_virtual_method(Register recv_klass,
3850                                            RegisterOrConstant vtable_index,
3851                                            Register method_result) {
3852   const ByteSize base = Klass::vtable_start_offset();
3853   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
3854   Address vtable_entry_addr(recv_klass,
3855                             vtable_index, Address::times_ptr,
3856                             base + vtableEntry::method_offset());
3857   movptr(method_result, vtable_entry_addr);
3858 }
3859 
3860 
3861 void MacroAssembler::check_klass_subtype(Register sub_klass,
3862                            Register super_klass,
3863                            Register temp_reg,
3864                            Label& L_success) {
3865   Label L_failure;
3866   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
3867   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
3868   bind(L_failure);
3869 }
3870 
3871 
3872 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3873                                                    Register super_klass,
3874                                                    Register temp_reg,
3875                                                    Label* L_success,
3876                                                    Label* L_failure,
3877                                                    Label* L_slow_path,
3878                                         RegisterOrConstant super_check_offset) {
3879   assert_different_registers(sub_klass, super_klass, temp_reg);
3880   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
3881   if (super_check_offset.is_register()) {
3882     assert_different_registers(sub_klass, super_klass,
3883                                super_check_offset.as_register());
3884   } else if (must_load_sco) {
3885     assert(temp_reg != noreg, "supply either a temp or a register offset");
3886   }
3887 
3888   Label L_fallthrough;
3889   int label_nulls = 0;
3890   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
3891   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
3892   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
3893   assert(label_nulls <= 1, "at most one null in the batch");
3894 
3895   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3896   int sco_offset = in_bytes(Klass::super_check_offset_offset());
3897   Address super_check_offset_addr(super_klass, sco_offset);
3898 
3899   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
3900   // range of a jccb.  If this routine grows larger, reconsider at
3901   // least some of these.
3902 #define local_jcc(assembler_cond, label)                                \
3903   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
3904   else                             jcc( assembler_cond, label) /*omit semi*/
3905 
3906   // Hacked jmp, which may only be used just before L_fallthrough.
3907 #define final_jmp(label)                                                \
3908   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
3909   else                            jmp(label)                /*omit semi*/
3910 
3911   // If the pointers are equal, we are done (e.g., String[] elements).
3912   // This self-check enables sharing of secondary supertype arrays among
3913   // non-primary types such as array-of-interface.  Otherwise, each such
3914   // type would need its own customized SSA.
3915   // We move this check to the front of the fast path because many
3916   // type checks are in fact trivially successful in this manner,
3917   // so we get a nicely predicted branch right at the start of the check.
3918   cmpptr(sub_klass, super_klass);
3919   local_jcc(Assembler::equal, *L_success);
3920 
3921   // Check the supertype display:
3922   if (must_load_sco) {
3923     // Positive movl does right thing on LP64.
3924     movl(temp_reg, super_check_offset_addr);
3925     super_check_offset = RegisterOrConstant(temp_reg);
3926   }
3927   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
3928   cmpptr(super_klass, super_check_addr); // load displayed supertype
3929 
3930   // This check has worked decisively for primary supers.
3931   // Secondary supers are sought in the super_cache ('super_cache_addr').
3932   // (Secondary supers are interfaces and very deeply nested subtypes.)
3933   // This works in the same check above because of a tricky aliasing
3934   // between the super_cache and the primary super display elements.
3935   // (The 'super_check_addr' can address either, as the case requires.)
3936   // Note that the cache is updated below if it does not help us find
3937   // what we need immediately.
3938   // So if it was a primary super, we can just fail immediately.
3939   // Otherwise, it's the slow path for us (no success at this point).
3940 
3941   if (super_check_offset.is_register()) {
3942     local_jcc(Assembler::equal, *L_success);
3943     cmpl(super_check_offset.as_register(), sc_offset);
3944     if (L_failure == &L_fallthrough) {
3945       local_jcc(Assembler::equal, *L_slow_path);
3946     } else {
3947       local_jcc(Assembler::notEqual, *L_failure);
3948       final_jmp(*L_slow_path);
3949     }
3950   } else if (super_check_offset.as_constant() == sc_offset) {
3951     // Need a slow path; fast failure is impossible.
3952     if (L_slow_path == &L_fallthrough) {
3953       local_jcc(Assembler::equal, *L_success);
3954     } else {
3955       local_jcc(Assembler::notEqual, *L_slow_path);
3956       final_jmp(*L_success);
3957     }
3958   } else {
3959     // No slow path; it's a fast decision.
3960     if (L_failure == &L_fallthrough) {
3961       local_jcc(Assembler::equal, *L_success);
3962     } else {
3963       local_jcc(Assembler::notEqual, *L_failure);
3964       final_jmp(*L_success);
3965     }
3966   }
3967 
3968   bind(L_fallthrough);
3969 
3970 #undef local_jcc
3971 #undef final_jmp
3972 }
3973 
3974 
3975 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
3976                                                           Register super_klass,
3977                                                           Register temp_reg,
3978                                                           Register temp2_reg,
3979                                                           Label* L_success,
3980                                                           Label* L_failure,
3981                                                           bool set_cond_codes) {
3982   assert_different_registers(sub_klass, super_klass, temp_reg);
3983   if (temp2_reg != noreg)
3984     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
3985 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
3986 
3987   Label L_fallthrough;
3988   int label_nulls = 0;
3989   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
3990   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
3991   assert(label_nulls <= 1, "at most one null in the batch");
3992 
3993   // a couple of useful fields in sub_klass:
3994   int ss_offset = in_bytes(Klass::secondary_supers_offset());
3995   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3996   Address secondary_supers_addr(sub_klass, ss_offset);
3997   Address super_cache_addr(     sub_klass, sc_offset);
3998 
3999   // Do a linear scan of the secondary super-klass chain.
4000   // This code is rarely used, so simplicity is a virtue here.
4001   // The repne_scan instruction uses fixed registers, which we must spill.
4002   // Don't worry too much about pre-existing connections with the input regs.
4003 
4004   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
4005   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
4006 
4007   // Get super_klass value into rax (even if it was in rdi or rcx).
4008   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
4009   if (super_klass != rax) {
4010     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
4011     mov(rax, super_klass);
4012   }
4013   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
4014   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
4015 
4016 #ifndef PRODUCT
4017   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
4018   ExternalAddress pst_counter_addr((address) pst_counter);
4019   lea(rcx, pst_counter_addr);
4020   incrementl(Address(rcx, 0));
4021 #endif //PRODUCT
4022 
4023   // We will consult the secondary-super array.
4024   movptr(rdi, secondary_supers_addr);
4025   // Load the array length.  (Positive movl does right thing on LP64.)
4026   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
4027   // Skip to start of data.
4028   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
4029 
4030   // Scan RCX words at [RDI] for an occurrence of RAX.
4031   // Set NZ/Z based on last compare.
4032   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
4033   // not change flags (only scas instruction which is repeated sets flags).
4034   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
4035 
4036     testptr(rax,rax); // Set Z = 0
4037     repne_scan();
4038 
4039   // Unspill the temp. registers:
4040   if (pushed_rdi)  pop(rdi);
4041   if (pushed_rcx)  pop(rcx);
4042   if (pushed_rax)  pop(rax);
4043 
4044   if (set_cond_codes) {
4045     // Special hack for the AD files:  rdi is guaranteed non-zero.
4046     assert(!pushed_rdi, "rdi must be left non-null");
4047     // Also, the condition codes are properly set Z/NZ on succeed/failure.
4048   }
4049 
4050   if (L_failure == &L_fallthrough)
4051         jccb(Assembler::notEqual, *L_failure);
4052   else  jcc(Assembler::notEqual, *L_failure);
4053 
4054   // Success.  Cache the super we found and proceed in triumph.
4055   movptr(super_cache_addr, super_klass);
4056 
4057   if (L_success != &L_fallthrough) {
4058     jmp(*L_success);
4059   }
4060 
4061 #undef IS_A_TEMP
4062 
4063   bind(L_fallthrough);
4064 }
4065 
4066 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
4067                                                    Register super_klass,
4068                                                    Register temp_reg,
4069                                                    Register temp2_reg,
4070                                                    Label* L_success,
4071                                                    Label* L_failure,
4072                                                    bool set_cond_codes) {
4073   assert(set_cond_codes == false, "must be false on 64-bit x86");
4074   check_klass_subtype_slow_path
4075     (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg,
4076      L_success, L_failure);
4077 }
4078 
4079 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
4080                                                    Register super_klass,
4081                                                    Register temp_reg,
4082                                                    Register temp2_reg,
4083                                                    Register temp3_reg,
4084                                                    Register temp4_reg,
4085                                                    Label* L_success,
4086                                                    Label* L_failure) {
4087   if (UseSecondarySupersTable) {
4088     check_klass_subtype_slow_path_table
4089       (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg,
4090        L_success, L_failure);
4091   } else {
4092     check_klass_subtype_slow_path_linear
4093       (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false);
4094   }
4095 }
4096 
4097 Register MacroAssembler::allocate_if_noreg(Register r,
4098                                   RegSetIterator<Register> &available_regs,
4099                                   RegSet &regs_to_push) {
4100   if (!r->is_valid()) {
4101     r = *available_regs++;
4102     regs_to_push += r;
4103   }
4104   return r;
4105 }
4106 
4107 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
4108                                                          Register super_klass,
4109                                                          Register temp_reg,
4110                                                          Register temp2_reg,
4111                                                          Register temp3_reg,
4112                                                          Register result_reg,
4113                                                          Label* L_success,
4114                                                          Label* L_failure) {
4115   // NB! Callers may assume that, when temp2_reg is a valid register,
4116   // this code sets it to a nonzero value.
4117   bool temp2_reg_was_valid = temp2_reg->is_valid();
4118 
4119   RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg);
4120 
4121   Label L_fallthrough;
4122   int label_nulls = 0;
4123   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4124   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4125   assert(label_nulls <= 1, "at most one null in the batch");
4126 
4127   BLOCK_COMMENT("check_klass_subtype_slow_path_table");
4128 
4129   RegSetIterator<Register> available_regs
4130     = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin();
4131 
4132   RegSet pushed_regs;
4133 
4134   temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
4135   temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
4136   temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);
4137   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
4138   Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs);
4139 
4140   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg);
4141 
4142   {
4143 
4144     int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
4145     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
4146     subptr(rsp, aligned_size);
4147     push_set(pushed_regs, 0);
4148 
4149     lookup_secondary_supers_table_var(sub_klass,
4150                                       super_klass,
4151                                       temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
4152     cmpq(result_reg, 0);
4153 
4154     // Unspill the temp. registers:
4155     pop_set(pushed_regs, 0);
4156     // Increment SP but do not clobber flags.
4157     lea(rsp, Address(rsp, aligned_size));
4158   }
4159 
4160   if (temp2_reg_was_valid) {
4161     movq(temp2_reg, 1);
4162   }
4163 
4164   jcc(Assembler::notEqual, *L_failure);
4165 
4166   if (L_success != &L_fallthrough) {
4167     jmp(*L_success);
4168   }
4169 
4170   bind(L_fallthrough);
4171 }
4172 
4173 // population_count variant for running without the POPCNT
4174 // instruction, which was introduced with SSE4.2 in 2008.
4175 void MacroAssembler::population_count(Register dst, Register src,
4176                                       Register scratch1, Register scratch2) {
4177   assert_different_registers(src, scratch1, scratch2);
4178   if (UsePopCountInstruction) {
4179     Assembler::popcntq(dst, src);
4180   } else {
4181     assert_different_registers(src, scratch1, scratch2);
4182     assert_different_registers(dst, scratch1, scratch2);
4183     Label loop, done;
4184 
4185     mov(scratch1, src);
4186     // dst = 0;
4187     // while(scratch1 != 0) {
4188     //   dst++;
4189     //   scratch1 &= (scratch1 - 1);
4190     // }
4191     xorl(dst, dst);
4192     testq(scratch1, scratch1);
4193     jccb(Assembler::equal, done);
4194     {
4195       bind(loop);
4196       incq(dst);
4197       movq(scratch2, scratch1);
4198       decq(scratch2);
4199       andq(scratch1, scratch2);
4200       jccb(Assembler::notEqual, loop);
4201     }
4202     bind(done);
4203   }
4204 #ifdef ASSERT
4205   mov64(scratch1, 0xCafeBabeDeadBeef);
4206   movq(scratch2, scratch1);
4207 #endif
4208 }
4209 
4210 // Ensure that the inline code and the stub are using the same registers.
4211 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
4212 do {                                                                 \
4213   assert(r_super_klass  == rax, "mismatch");                         \
4214   assert(r_array_base   == rbx, "mismatch");                         \
4215   assert(r_array_length == rcx, "mismatch");                         \
4216   assert(r_array_index  == rdx, "mismatch");                         \
4217   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
4218   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
4219   assert(result         == rdi || result      == noreg, "mismatch"); \
4220 } while(0)
4221 
4222 // Versions of salq and rorq that don't need count to be in rcx
4223 
4224 void MacroAssembler::salq(Register dest, Register count) {
4225   if (count == rcx) {
4226     Assembler::salq(dest);
4227   } else {
4228     assert_different_registers(rcx, dest);
4229     xchgq(rcx, count);
4230     Assembler::salq(dest);
4231     xchgq(rcx, count);
4232   }
4233 }
4234 
4235 void MacroAssembler::rorq(Register dest, Register count) {
4236   if (count == rcx) {
4237     Assembler::rorq(dest);
4238   } else {
4239     assert_different_registers(rcx, dest);
4240     xchgq(rcx, count);
4241     Assembler::rorq(dest);
4242     xchgq(rcx, count);
4243   }
4244 }
4245 
4246 // Return true: we succeeded in generating this code
4247 //
4248 // At runtime, return 0 in result if r_super_klass is a superclass of
4249 // r_sub_klass, otherwise return nonzero. Use this if you know the
4250 // super_klass_slot of the class you're looking for. This is always
4251 // the case for instanceof and checkcast.
4252 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
4253                                                          Register r_super_klass,
4254                                                          Register temp1,
4255                                                          Register temp2,
4256                                                          Register temp3,
4257                                                          Register temp4,
4258                                                          Register result,
4259                                                          u1 super_klass_slot) {
4260   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
4261 
4262   Label L_fallthrough, L_success, L_failure;
4263 
4264   BLOCK_COMMENT("lookup_secondary_supers_table {");
4265 
4266   const Register
4267     r_array_index  = temp1,
4268     r_array_length = temp2,
4269     r_array_base   = temp3,
4270     r_bitmap       = temp4;
4271 
4272   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
4273 
4274   xorq(result, result); // = 0
4275 
4276   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
4277   movq(r_array_index, r_bitmap);
4278 
4279   // First check the bitmap to see if super_klass might be present. If
4280   // the bit is zero, we are certain that super_klass is not one of
4281   // the secondary supers.
4282   u1 bit = super_klass_slot;
4283   {
4284     // NB: If the count in a x86 shift instruction is 0, the flags are
4285     // not affected, so we do a testq instead.
4286     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
4287     if (shift_count != 0) {
4288       salq(r_array_index, shift_count);
4289     } else {
4290       testq(r_array_index, r_array_index);
4291     }
4292   }
4293   // We test the MSB of r_array_index, i.e. its sign bit
4294   jcc(Assembler::positive, L_failure);
4295 
4296   // Get the first array index that can contain super_klass into r_array_index.
4297   if (bit != 0) {
4298     population_count(r_array_index, r_array_index, temp2, temp3);
4299   } else {
4300     movl(r_array_index, 1);
4301   }
4302   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
4303 
4304   // We will consult the secondary-super array.
4305   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4306 
4307   // We're asserting that the first word in an Array<Klass*> is the
4308   // length, and the second word is the first word of the data. If
4309   // that ever changes, r_array_base will have to be adjusted here.
4310   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
4311   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
4312 
4313   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4314   jccb(Assembler::equal, L_success);
4315 
4316   // Is there another entry to check? Consult the bitmap.
4317   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
4318   jccb(Assembler::carryClear, L_failure);
4319 
4320   // Linear probe. Rotate the bitmap so that the next bit to test is
4321   // in Bit 1.
4322   if (bit != 0) {
4323     rorq(r_bitmap, bit);
4324   }
4325 
4326   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
4327   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
4328   // Kills: r_array_length.
4329   // Returns: result.
4330   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
4331   // Result (0/1) is in rdi
4332   jmpb(L_fallthrough);
4333 
4334   bind(L_failure);
4335   incq(result); // 0 => 1
4336 
4337   bind(L_success);
4338   // result = 0;
4339 
4340   bind(L_fallthrough);
4341   BLOCK_COMMENT("} lookup_secondary_supers_table");
4342 
4343   if (VerifySecondarySupers) {
4344     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
4345                                   temp1, temp2, temp3);
4346   }
4347 }
4348 
4349 // At runtime, return 0 in result if r_super_klass is a superclass of
4350 // r_sub_klass, otherwise return nonzero. Use this version of
4351 // lookup_secondary_supers_table() if you don't know ahead of time
4352 // which superclass will be searched for. Used by interpreter and
4353 // runtime stubs. It is larger and has somewhat greater latency than
4354 // the version above, which takes a constant super_klass_slot.
4355 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
4356                                                        Register r_super_klass,
4357                                                        Register temp1,
4358                                                        Register temp2,
4359                                                        Register temp3,
4360                                                        Register temp4,
4361                                                        Register result) {
4362   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
4363   assert_different_registers(r_sub_klass, r_super_klass, rcx);
4364   RegSet temps = RegSet::of(temp1, temp2, temp3, temp4);
4365 
4366   Label L_fallthrough, L_success, L_failure;
4367 
4368   BLOCK_COMMENT("lookup_secondary_supers_table {");
4369 
4370   RegSetIterator<Register> available_regs = (temps - rcx).begin();
4371 
4372   // FIXME. Once we are sure that all paths reaching this point really
4373   // do pass rcx as one of our temps we can get rid of the following
4374   // workaround.
4375   assert(temps.contains(rcx), "fix this code");
4376 
4377   // We prefer to have our shift count in rcx. If rcx is one of our
4378   // temps, use it for slot. If not, pick any of our temps.
4379   Register slot;
4380   if (!temps.contains(rcx)) {
4381     slot = *available_regs++;
4382   } else {
4383     slot = rcx;
4384   }
4385 
4386   const Register r_array_index = *available_regs++;
4387   const Register r_bitmap      = *available_regs++;
4388 
4389   // The logic above guarantees this property, but we state it here.
4390   assert_different_registers(r_array_index, r_bitmap, rcx);
4391 
4392   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
4393   movq(r_array_index, r_bitmap);
4394 
4395   // First check the bitmap to see if super_klass might be present. If
4396   // the bit is zero, we are certain that super_klass is not one of
4397   // the secondary supers.
4398   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
4399   xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
4400   salq(r_array_index, slot);
4401 
4402   testq(r_array_index, r_array_index);
4403   // We test the MSB of r_array_index, i.e. its sign bit
4404   jcc(Assembler::positive, L_failure);
4405 
4406   const Register r_array_base = *available_regs++;
4407 
4408   // Get the first array index that can contain super_klass into r_array_index.
4409   // Note: Clobbers r_array_base and slot.
4410   population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot);
4411 
4412   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
4413 
4414   // We will consult the secondary-super array.
4415   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4416 
4417   // We're asserting that the first word in an Array<Klass*> is the
4418   // length, and the second word is the first word of the data. If
4419   // that ever changes, r_array_base will have to be adjusted here.
4420   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
4421   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
4422 
4423   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4424   jccb(Assembler::equal, L_success);
4425 
4426   // Restore slot to its true value
4427   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
4428 
4429   // Linear probe. Rotate the bitmap so that the next bit to test is
4430   // in Bit 1.
4431   rorq(r_bitmap, slot);
4432 
4433   // Is there another entry to check? Consult the bitmap.
4434   btq(r_bitmap, 1);
4435   jccb(Assembler::carryClear, L_failure);
4436 
4437   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
4438   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
4439   // Kills: r_array_length.
4440   // Returns: result.
4441   lookup_secondary_supers_table_slow_path(r_super_klass,
4442                                           r_array_base,
4443                                           r_array_index,
4444                                           r_bitmap,
4445                                           /*temp1*/result,
4446                                           /*temp2*/slot,
4447                                           &L_success,
4448                                           nullptr);
4449 
4450   bind(L_failure);
4451   movq(result, 1);
4452   jmpb(L_fallthrough);
4453 
4454   bind(L_success);
4455   xorq(result, result); // = 0
4456 
4457   bind(L_fallthrough);
4458   BLOCK_COMMENT("} lookup_secondary_supers_table");
4459 
4460   if (VerifySecondarySupers) {
4461     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
4462                                   temp1, temp2, temp3);
4463   }
4464 }
4465 
4466 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
4467                                  Label* L_success, Label* L_failure) {
4468   Label L_loop, L_fallthrough;
4469   {
4470     int label_nulls = 0;
4471     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
4472     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
4473     assert(label_nulls <= 1, "at most one null in the batch");
4474   }
4475   bind(L_loop);
4476   cmpq(value, Address(addr, count, Address::times_8));
4477   jcc(Assembler::equal, *L_success);
4478   addl(count, 1);
4479   cmpl(count, limit);
4480   jcc(Assembler::less, L_loop);
4481 
4482   if (&L_fallthrough != L_failure) {
4483     jmp(*L_failure);
4484   }
4485   bind(L_fallthrough);
4486 }
4487 
4488 // Called by code generated by check_klass_subtype_slow_path
4489 // above. This is called when there is a collision in the hashed
4490 // lookup in the secondary supers array.
4491 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
4492                                                              Register r_array_base,
4493                                                              Register r_array_index,
4494                                                              Register r_bitmap,
4495                                                              Register temp1,
4496                                                              Register temp2,
4497                                                              Label* L_success,
4498                                                              Label* L_failure) {
4499   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
4500 
4501   const Register
4502     r_array_length = temp1,
4503     r_sub_klass    = noreg,
4504     result         = noreg;
4505 
4506   Label L_fallthrough;
4507   int label_nulls = 0;
4508   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4509   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4510   assert(label_nulls <= 1, "at most one null in the batch");
4511 
4512   // Load the array length.
4513   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
4514   // And adjust the array base to point to the data.
4515   // NB! Effectively increments current slot index by 1.
4516   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
4517   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
4518 
4519   // Linear probe
4520   Label L_huge;
4521 
4522   // The bitmap is full to bursting.
4523   // Implicit invariant: BITMAP_FULL implies (length > 0)
4524   cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
4525   jcc(Assembler::greater, L_huge);
4526 
4527   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
4528   // current slot (at secondary_supers[r_array_index]) has not yet
4529   // been inspected, and r_array_index may be out of bounds if we
4530   // wrapped around the end of the array.
4531 
4532   { // This is conventional linear probing, but instead of terminating
4533     // when a null entry is found in the table, we maintain a bitmap
4534     // in which a 0 indicates missing entries.
4535     // The check above guarantees there are 0s in the bitmap, so the loop
4536     // eventually terminates.
4537 
4538     xorl(temp2, temp2); // = 0;
4539 
4540     Label L_again;
4541     bind(L_again);
4542 
4543     // Check for array wraparound.
4544     cmpl(r_array_index, r_array_length);
4545     cmovl(Assembler::greaterEqual, r_array_index, temp2);
4546 
4547     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4548     jcc(Assembler::equal, *L_success);
4549 
4550     // If the next bit in bitmap is zero, we're done.
4551     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
4552     jcc(Assembler::carryClear, *L_failure);
4553 
4554     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
4555     addl(r_array_index, 1);
4556 
4557     jmp(L_again);
4558   }
4559 
4560   { // Degenerate case: more than 64 secondary supers.
4561     // FIXME: We could do something smarter here, maybe a vectorized
4562     // comparison or a binary search, but is that worth any added
4563     // complexity?
4564     bind(L_huge);
4565     xorl(r_array_index, r_array_index); // = 0
4566     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
4567                 L_success,
4568                 (&L_fallthrough != L_failure ? L_failure : nullptr));
4569 
4570     bind(L_fallthrough);
4571   }
4572 }
4573 
4574 struct VerifyHelperArguments {
4575   Klass* _super;
4576   Klass* _sub;
4577   intptr_t _linear_result;
4578   intptr_t _table_result;
4579 };
4580 
4581 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
4582   Klass::on_secondary_supers_verification_failure(args->_super,
4583                                                   args->_sub,
4584                                                   args->_linear_result,
4585                                                   args->_table_result,
4586                                                   msg);
4587 }
4588 
4589 // Make sure that the hashed lookup and a linear scan agree.
4590 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
4591                                                    Register r_super_klass,
4592                                                    Register result,
4593                                                    Register temp1,
4594                                                    Register temp2,
4595                                                    Register temp3) {
4596   const Register
4597       r_array_index  = temp1,
4598       r_array_length = temp2,
4599       r_array_base   = temp3,
4600       r_bitmap       = noreg;
4601 
4602   BLOCK_COMMENT("verify_secondary_supers_table {");
4603 
4604   Label L_success, L_failure, L_check, L_done;
4605 
4606   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4607   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
4608   // And adjust the array base to point to the data.
4609   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
4610 
4611   testl(r_array_length, r_array_length); // array_length == 0?
4612   jcc(Assembler::zero, L_failure);
4613 
4614   movl(r_array_index, 0);
4615   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
4616   // fall through to L_failure
4617 
4618   const Register linear_result = r_array_index; // reuse temp1
4619 
4620   bind(L_failure); // not present
4621   movl(linear_result, 1);
4622   jmp(L_check);
4623 
4624   bind(L_success); // present
4625   movl(linear_result, 0);
4626 
4627   bind(L_check);
4628   cmpl(linear_result, result);
4629   jcc(Assembler::equal, L_done);
4630 
4631   { // To avoid calling convention issues, build a record on the stack
4632     // and pass the pointer to that instead.
4633     push(result);
4634     push(linear_result);
4635     push(r_sub_klass);
4636     push(r_super_klass);
4637     movptr(c_rarg1, rsp);
4638     movptr(c_rarg0, (uintptr_t) "mismatch");
4639     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
4640     should_not_reach_here();
4641   }
4642   bind(L_done);
4643 
4644   BLOCK_COMMENT("} verify_secondary_supers_table");
4645 }
4646 
4647 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
4648 
4649 void MacroAssembler::clinit_barrier(Register klass, Label* L_fast_path, Label* L_slow_path) {
4650   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
4651 
4652   Label L_fallthrough;
4653   if (L_fast_path == nullptr) {
4654     L_fast_path = &L_fallthrough;
4655   } else if (L_slow_path == nullptr) {
4656     L_slow_path = &L_fallthrough;
4657   }
4658 
4659   // Fast path check: class is fully initialized.
4660   // init_state needs acquire, but x86 is TSO, and so we are already good.
4661   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
4662   jcc(Assembler::equal, *L_fast_path);
4663 
4664   // Fast path check: current thread is initializer thread
4665   cmpptr(r15_thread, Address(klass, InstanceKlass::init_thread_offset()));
4666   if (L_slow_path == &L_fallthrough) {
4667     jcc(Assembler::equal, *L_fast_path);
4668     bind(*L_slow_path);
4669   } else if (L_fast_path == &L_fallthrough) {
4670     jcc(Assembler::notEqual, *L_slow_path);
4671     bind(*L_fast_path);
4672   } else {
4673     Unimplemented();
4674   }
4675 }
4676 
4677 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
4678   if (VM_Version::supports_cmov()) {
4679     cmovl(cc, dst, src);
4680   } else {
4681     Label L;
4682     jccb(negate_condition(cc), L);
4683     movl(dst, src);
4684     bind(L);
4685   }
4686 }
4687 
4688 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
4689   if (VM_Version::supports_cmov()) {
4690     cmovl(cc, dst, src);
4691   } else {
4692     Label L;
4693     jccb(negate_condition(cc), L);
4694     movl(dst, src);
4695     bind(L);
4696   }
4697 }
4698 
4699 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
4700   if (!VerifyOops) return;
4701 
4702   BLOCK_COMMENT("verify_oop {");
4703   push(rscratch1);
4704   push(rax);                          // save rax
4705   push(reg);                          // pass register argument
4706 
4707   // Pass register number to verify_oop_subroutine
4708   const char* b = nullptr;
4709   {
4710     ResourceMark rm;
4711     stringStream ss;
4712     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
4713     b = code_string(ss.as_string());
4714   }
4715   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
4716   pushptr(buffer.addr(), rscratch1);
4717 
4718   // call indirectly to solve generation ordering problem
4719   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4720   call(rax);
4721   // Caller pops the arguments (oop, message) and restores rax, r10
4722   BLOCK_COMMENT("} verify_oop");
4723 }
4724 
4725 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
4726   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
4727     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
4728     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
4729     vpternlogd(dst, 0xFF, dst, dst, vector_len);
4730   } else if (VM_Version::supports_avx()) {
4731     vpcmpeqd(dst, dst, dst, vector_len);
4732   } else {
4733     pcmpeqd(dst, dst);
4734   }
4735 }
4736 
4737 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
4738                                          int extra_slot_offset) {
4739   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4740   int stackElementSize = Interpreter::stackElementSize;
4741   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4742 #ifdef ASSERT
4743   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4744   assert(offset1 - offset == stackElementSize, "correct arithmetic");
4745 #endif
4746   Register             scale_reg    = noreg;
4747   Address::ScaleFactor scale_factor = Address::no_scale;
4748   if (arg_slot.is_constant()) {
4749     offset += arg_slot.as_constant() * stackElementSize;
4750   } else {
4751     scale_reg    = arg_slot.as_register();
4752     scale_factor = Address::times(stackElementSize);
4753   }
4754   offset += wordSize;           // return PC is on stack
4755   return Address(rsp, scale_reg, scale_factor, offset);
4756 }
4757 
4758 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
4759   if (!VerifyOops) return;
4760 
4761   push(rscratch1);
4762   push(rax); // save rax,
4763   // addr may contain rsp so we will have to adjust it based on the push
4764   // we just did (and on 64 bit we do two pushes)
4765   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4766   // stores rax into addr which is backwards of what was intended.
4767   if (addr.uses(rsp)) {
4768     lea(rax, addr);
4769     pushptr(Address(rax, 2 * BytesPerWord));
4770   } else {
4771     pushptr(addr);
4772   }
4773 
4774   // Pass register number to verify_oop_subroutine
4775   const char* b = nullptr;
4776   {
4777     ResourceMark rm;
4778     stringStream ss;
4779     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
4780     b = code_string(ss.as_string());
4781   }
4782   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
4783   pushptr(buffer.addr(), rscratch1);
4784 
4785   // call indirectly to solve generation ordering problem
4786   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4787   call(rax);
4788   // Caller pops the arguments (addr, message) and restores rax, r10.
4789 }
4790 
4791 void MacroAssembler::verify_tlab() {
4792 #ifdef ASSERT
4793   if (UseTLAB && VerifyOops) {
4794     Label next, ok;
4795     Register t1 = rsi;
4796 
4797     push(t1);
4798 
4799     movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
4800     cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset())));
4801     jcc(Assembler::aboveEqual, next);
4802     STOP("assert(top >= start)");
4803     should_not_reach_here();
4804 
4805     bind(next);
4806     movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
4807     cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
4808     jcc(Assembler::aboveEqual, ok);
4809     STOP("assert(top <= end)");
4810     should_not_reach_here();
4811 
4812     bind(ok);
4813     pop(t1);
4814   }
4815 #endif
4816 }
4817 
4818 class ControlWord {
4819  public:
4820   int32_t _value;
4821 
4822   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
4823   int  precision_control() const       { return  (_value >>  8) & 3      ; }
4824   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4825   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4826   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4827   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4828   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4829   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4830 
4831   void print() const {
4832     // rounding control
4833     const char* rc;
4834     switch (rounding_control()) {
4835       case 0: rc = "round near"; break;
4836       case 1: rc = "round down"; break;
4837       case 2: rc = "round up  "; break;
4838       case 3: rc = "chop      "; break;
4839       default:
4840         rc = nullptr; // silence compiler warnings
4841         fatal("Unknown rounding control: %d", rounding_control());
4842     };
4843     // precision control
4844     const char* pc;
4845     switch (precision_control()) {
4846       case 0: pc = "24 bits "; break;
4847       case 1: pc = "reserved"; break;
4848       case 2: pc = "53 bits "; break;
4849       case 3: pc = "64 bits "; break;
4850       default:
4851         pc = nullptr; // silence compiler warnings
4852         fatal("Unknown precision control: %d", precision_control());
4853     };
4854     // flags
4855     char f[9];
4856     f[0] = ' ';
4857     f[1] = ' ';
4858     f[2] = (precision   ()) ? 'P' : 'p';
4859     f[3] = (underflow   ()) ? 'U' : 'u';
4860     f[4] = (overflow    ()) ? 'O' : 'o';
4861     f[5] = (zero_divide ()) ? 'Z' : 'z';
4862     f[6] = (denormalized()) ? 'D' : 'd';
4863     f[7] = (invalid     ()) ? 'I' : 'i';
4864     f[8] = '\x0';
4865     // output
4866     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
4867   }
4868 
4869 };
4870 
4871 class StatusWord {
4872  public:
4873   int32_t _value;
4874 
4875   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
4876   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
4877   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
4878   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
4879   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
4880   int  top() const                     { return  (_value >> 11) & 7      ; }
4881   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
4882   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
4883   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4884   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4885   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4886   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4887   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4888   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4889 
4890   void print() const {
4891     // condition codes
4892     char c[5];
4893     c[0] = (C3()) ? '3' : '-';
4894     c[1] = (C2()) ? '2' : '-';
4895     c[2] = (C1()) ? '1' : '-';
4896     c[3] = (C0()) ? '0' : '-';
4897     c[4] = '\x0';
4898     // flags
4899     char f[9];
4900     f[0] = (error_status()) ? 'E' : '-';
4901     f[1] = (stack_fault ()) ? 'S' : '-';
4902     f[2] = (precision   ()) ? 'P' : '-';
4903     f[3] = (underflow   ()) ? 'U' : '-';
4904     f[4] = (overflow    ()) ? 'O' : '-';
4905     f[5] = (zero_divide ()) ? 'Z' : '-';
4906     f[6] = (denormalized()) ? 'D' : '-';
4907     f[7] = (invalid     ()) ? 'I' : '-';
4908     f[8] = '\x0';
4909     // output
4910     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
4911   }
4912 
4913 };
4914 
4915 class TagWord {
4916  public:
4917   int32_t _value;
4918 
4919   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
4920 
4921   void print() const {
4922     printf("%04x", _value & 0xFFFF);
4923   }
4924 
4925 };
4926 
4927 class FPU_Register {
4928  public:
4929   int32_t _m0;
4930   int32_t _m1;
4931   int16_t _ex;
4932 
4933   bool is_indefinite() const           {
4934     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
4935   }
4936 
4937   void print() const {
4938     char  sign = (_ex < 0) ? '-' : '+';
4939     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
4940     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
4941   };
4942 
4943 };
4944 
4945 class FPU_State {
4946  public:
4947   enum {
4948     register_size       = 10,
4949     number_of_registers =  8,
4950     register_mask       =  7
4951   };
4952 
4953   ControlWord  _control_word;
4954   StatusWord   _status_word;
4955   TagWord      _tag_word;
4956   int32_t      _error_offset;
4957   int32_t      _error_selector;
4958   int32_t      _data_offset;
4959   int32_t      _data_selector;
4960   int8_t       _register[register_size * number_of_registers];
4961 
4962   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
4963   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
4964 
4965   const char* tag_as_string(int tag) const {
4966     switch (tag) {
4967       case 0: return "valid";
4968       case 1: return "zero";
4969       case 2: return "special";
4970       case 3: return "empty";
4971     }
4972     ShouldNotReachHere();
4973     return nullptr;
4974   }
4975 
4976   void print() const {
4977     // print computation registers
4978     { int t = _status_word.top();
4979       for (int i = 0; i < number_of_registers; i++) {
4980         int j = (i - t) & register_mask;
4981         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
4982         st(j)->print();
4983         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
4984       }
4985     }
4986     printf("\n");
4987     // print control registers
4988     printf("ctrl = "); _control_word.print(); printf("\n");
4989     printf("stat = "); _status_word .print(); printf("\n");
4990     printf("tags = "); _tag_word    .print(); printf("\n");
4991   }
4992 
4993 };
4994 
4995 class Flag_Register {
4996  public:
4997   int32_t _value;
4998 
4999   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
5000   bool direction() const               { return ((_value >> 10) & 1) != 0; }
5001   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
5002   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
5003   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
5004   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
5005   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5006 
5007   void print() const {
5008     // flags
5009     char f[8];
5010     f[0] = (overflow       ()) ? 'O' : '-';
5011     f[1] = (direction      ()) ? 'D' : '-';
5012     f[2] = (sign           ()) ? 'S' : '-';
5013     f[3] = (zero           ()) ? 'Z' : '-';
5014     f[4] = (auxiliary_carry()) ? 'A' : '-';
5015     f[5] = (parity         ()) ? 'P' : '-';
5016     f[6] = (carry          ()) ? 'C' : '-';
5017     f[7] = '\x0';
5018     // output
5019     printf("%08x  flags = %s", _value, f);
5020   }
5021 
5022 };
5023 
5024 class IU_Register {
5025  public:
5026   int32_t _value;
5027 
5028   void print() const {
5029     printf("%08x  %11d", _value, _value);
5030   }
5031 
5032 };
5033 
5034 class IU_State {
5035  public:
5036   Flag_Register _eflags;
5037   IU_Register   _rdi;
5038   IU_Register   _rsi;
5039   IU_Register   _rbp;
5040   IU_Register   _rsp;
5041   IU_Register   _rbx;
5042   IU_Register   _rdx;
5043   IU_Register   _rcx;
5044   IU_Register   _rax;
5045 
5046   void print() const {
5047     // computation registers
5048     printf("rax,  = "); _rax.print(); printf("\n");
5049     printf("rbx,  = "); _rbx.print(); printf("\n");
5050     printf("rcx  = "); _rcx.print(); printf("\n");
5051     printf("rdx  = "); _rdx.print(); printf("\n");
5052     printf("rdi  = "); _rdi.print(); printf("\n");
5053     printf("rsi  = "); _rsi.print(); printf("\n");
5054     printf("rbp,  = "); _rbp.print(); printf("\n");
5055     printf("rsp  = "); _rsp.print(); printf("\n");
5056     printf("\n");
5057     // control registers
5058     printf("flgs = "); _eflags.print(); printf("\n");
5059   }
5060 };
5061 
5062 
5063 class CPU_State {
5064  public:
5065   FPU_State _fpu_state;
5066   IU_State  _iu_state;
5067 
5068   void print() const {
5069     printf("--------------------------------------------------\n");
5070     _iu_state .print();
5071     printf("\n");
5072     _fpu_state.print();
5073     printf("--------------------------------------------------\n");
5074   }
5075 
5076 };
5077 
5078 
5079 static void _print_CPU_state(CPU_State* state) {
5080   state->print();
5081 };
5082 
5083 
5084 void MacroAssembler::print_CPU_state() {
5085   push_CPU_state();
5086   push(rsp);                // pass CPU state
5087   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5088   addptr(rsp, wordSize);       // discard argument
5089   pop_CPU_state();
5090 }
5091 
5092 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
5093   // Either restore the MXCSR register after returning from the JNI Call
5094   // or verify that it wasn't changed (with -Xcheck:jni flag).
5095   if (VM_Version::supports_sse()) {
5096     if (RestoreMXCSROnJNICalls) {
5097       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
5098     } else if (CheckJNICalls) {
5099       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5100     }
5101   }
5102   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5103   vzeroupper();
5104 }
5105 
5106 // ((OopHandle)result).resolve();
5107 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
5108   assert_different_registers(result, tmp);
5109 
5110   // Only 64 bit platforms support GCs that require a tmp register
5111   // Only IN_HEAP loads require a thread_tmp register
5112   // OopHandle::resolve is an indirection like jobject.
5113   access_load_at(T_OBJECT, IN_NATIVE,
5114                  result, Address(result, 0), tmp);
5115 }
5116 
5117 // ((WeakHandle)result).resolve();
5118 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
5119   assert_different_registers(rresult, rtmp);
5120   Label resolved;
5121 
5122   // A null weak handle resolves to null.
5123   cmpptr(rresult, 0);
5124   jcc(Assembler::equal, resolved);
5125 
5126   // Only 64 bit platforms support GCs that require a tmp register
5127   // Only IN_HEAP loads require a thread_tmp register
5128   // WeakHandle::resolve is an indirection like jweak.
5129   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
5130                  rresult, Address(rresult, 0), rtmp);
5131   bind(resolved);
5132 }
5133 
5134 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
5135   // get mirror
5136   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
5137   load_method_holder(mirror, method);
5138   movptr(mirror, Address(mirror, mirror_offset));
5139   resolve_oop_handle(mirror, tmp);
5140 }
5141 
5142 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
5143   load_method_holder(rresult, rmethod);
5144   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
5145 }
5146 
5147 void MacroAssembler::load_method_holder(Register holder, Register method) {
5148   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
5149   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
5150   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
5151 }
5152 
5153 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
5154   assert(UseCompactObjectHeaders, "expect compact object headers");
5155   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
5156   shrq(dst, markWord::klass_shift);
5157 }
5158 
5159 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
5160   assert_different_registers(src, tmp);
5161   assert_different_registers(dst, tmp);
5162 
5163   if (UseCompactObjectHeaders) {
5164     load_narrow_klass_compact(dst, src);
5165     decode_klass_not_null(dst, tmp);
5166   } else if (UseCompressedClassPointers) {
5167     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5168     decode_klass_not_null(dst, tmp);
5169   } else {
5170     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5171   }
5172 }
5173 
5174 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
5175   assert(!UseCompactObjectHeaders, "not with compact headers");
5176   assert_different_registers(src, tmp);
5177   assert_different_registers(dst, tmp);
5178   if (UseCompressedClassPointers) {
5179     encode_klass_not_null(src, tmp);
5180     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5181   } else {
5182     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5183   }
5184 }
5185 
5186 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
5187   if (UseCompactObjectHeaders) {
5188     assert(tmp != noreg, "need tmp");
5189     assert_different_registers(klass, obj, tmp);
5190     load_narrow_klass_compact(tmp, obj);
5191     cmpl(klass, tmp);
5192   } else if (UseCompressedClassPointers) {
5193     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
5194   } else {
5195     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
5196   }
5197 }
5198 
5199 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
5200   if (UseCompactObjectHeaders) {
5201     assert(tmp2 != noreg, "need tmp2");
5202     assert_different_registers(obj1, obj2, tmp1, tmp2);
5203     load_narrow_klass_compact(tmp1, obj1);
5204     load_narrow_klass_compact(tmp2, obj2);
5205     cmpl(tmp1, tmp2);
5206   } else if (UseCompressedClassPointers) {
5207     movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
5208     cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
5209   } else {
5210     movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
5211     cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
5212   }
5213 }
5214 
5215 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
5216                                     Register tmp1) {
5217   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5218   decorators = AccessInternal::decorator_fixup(decorators, type);
5219   bool as_raw = (decorators & AS_RAW) != 0;
5220   if (as_raw) {
5221     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1);
5222   } else {
5223     bs->load_at(this, decorators, type, dst, src, tmp1);
5224   }
5225 }
5226 
5227 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
5228                                      Register tmp1, Register tmp2, Register tmp3) {
5229   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5230   decorators = AccessInternal::decorator_fixup(decorators, type);
5231   bool as_raw = (decorators & AS_RAW) != 0;
5232   if (as_raw) {
5233     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
5234   } else {
5235     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
5236   }
5237 }
5238 
5239 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, DecoratorSet decorators) {
5240   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1);
5241 }
5242 
5243 // Doesn't do verification, generates fixed size code
5244 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, DecoratorSet decorators) {
5245   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1);
5246 }
5247 
5248 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
5249                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
5250   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
5251 }
5252 
5253 // Used for storing nulls.
5254 void MacroAssembler::store_heap_oop_null(Address dst) {
5255   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
5256 }
5257 
5258 void MacroAssembler::store_klass_gap(Register dst, Register src) {
5259   assert(!UseCompactObjectHeaders, "Don't use with compact headers");
5260   if (UseCompressedClassPointers) {
5261     // Store to klass gap in destination
5262     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
5263   }
5264 }
5265 
5266 #ifdef ASSERT
5267 void MacroAssembler::verify_heapbase(const char* msg) {
5268   assert (UseCompressedOops, "should be compressed");
5269   assert (Universe::heap() != nullptr, "java heap should be initialized");
5270   if (CheckCompressedOops) {
5271     Label ok;
5272     ExternalAddress src2(CompressedOops::base_addr());
5273     const bool is_src2_reachable = reachable(src2);
5274     if (!is_src2_reachable) {
5275       push(rscratch1);  // cmpptr trashes rscratch1
5276     }
5277     cmpptr(r12_heapbase, src2, rscratch1);
5278     jcc(Assembler::equal, ok);
5279     STOP(msg);
5280     bind(ok);
5281     if (!is_src2_reachable) {
5282       pop(rscratch1);
5283     }
5284   }
5285 }
5286 #endif
5287 
5288 // Algorithm must match oop.inline.hpp encode_heap_oop.
5289 void MacroAssembler::encode_heap_oop(Register r) {
5290 #ifdef ASSERT
5291   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
5292 #endif
5293   verify_oop_msg(r, "broken oop in encode_heap_oop");
5294   if (CompressedOops::base() == nullptr) {
5295     if (CompressedOops::shift() != 0) {
5296       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5297       shrq(r, LogMinObjAlignmentInBytes);
5298     }
5299     return;
5300   }
5301   testq(r, r);
5302   cmovq(Assembler::equal, r, r12_heapbase);
5303   subq(r, r12_heapbase);
5304   shrq(r, LogMinObjAlignmentInBytes);
5305 }
5306 
5307 void MacroAssembler::encode_heap_oop_not_null(Register r) {
5308 #ifdef ASSERT
5309   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
5310   if (CheckCompressedOops) {
5311     Label ok;
5312     testq(r, r);
5313     jcc(Assembler::notEqual, ok);
5314     STOP("null oop passed to encode_heap_oop_not_null");
5315     bind(ok);
5316   }
5317 #endif
5318   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
5319   if (CompressedOops::base() != nullptr) {
5320     subq(r, r12_heapbase);
5321   }
5322   if (CompressedOops::shift() != 0) {
5323     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5324     shrq(r, LogMinObjAlignmentInBytes);
5325   }
5326 }
5327 
5328 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
5329 #ifdef ASSERT
5330   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
5331   if (CheckCompressedOops) {
5332     Label ok;
5333     testq(src, src);
5334     jcc(Assembler::notEqual, ok);
5335     STOP("null oop passed to encode_heap_oop_not_null2");
5336     bind(ok);
5337   }
5338 #endif
5339   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
5340   if (dst != src) {
5341     movq(dst, src);
5342   }
5343   if (CompressedOops::base() != nullptr) {
5344     subq(dst, r12_heapbase);
5345   }
5346   if (CompressedOops::shift() != 0) {
5347     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5348     shrq(dst, LogMinObjAlignmentInBytes);
5349   }
5350 }
5351 
5352 void  MacroAssembler::decode_heap_oop(Register r) {
5353 #ifdef ASSERT
5354   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
5355 #endif
5356   if (CompressedOops::base() == nullptr) {
5357     if (CompressedOops::shift() != 0) {
5358       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5359       shlq(r, LogMinObjAlignmentInBytes);
5360     }
5361   } else {
5362     Label done;
5363     shlq(r, LogMinObjAlignmentInBytes);
5364     jccb(Assembler::equal, done);
5365     addq(r, r12_heapbase);
5366     bind(done);
5367   }
5368   verify_oop_msg(r, "broken oop in decode_heap_oop");
5369 }
5370 
5371 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
5372   // Note: it will change flags
5373   assert (UseCompressedOops, "should only be used for compressed headers");
5374   assert (Universe::heap() != nullptr, "java heap should be initialized");
5375   // Cannot assert, unverified entry point counts instructions (see .ad file)
5376   // vtableStubs also counts instructions in pd_code_size_limit.
5377   // Also do not verify_oop as this is called by verify_oop.
5378   if (CompressedOops::shift() != 0) {
5379     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5380     shlq(r, LogMinObjAlignmentInBytes);
5381     if (CompressedOops::base() != nullptr) {
5382       addq(r, r12_heapbase);
5383     }
5384   } else {
5385     assert (CompressedOops::base() == nullptr, "sanity");
5386   }
5387 }
5388 
5389 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
5390   // Note: it will change flags
5391   assert (UseCompressedOops, "should only be used for compressed headers");
5392   assert (Universe::heap() != nullptr, "java heap should be initialized");
5393   // Cannot assert, unverified entry point counts instructions (see .ad file)
5394   // vtableStubs also counts instructions in pd_code_size_limit.
5395   // Also do not verify_oop as this is called by verify_oop.
5396   if (CompressedOops::shift() != 0) {
5397     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5398     if (LogMinObjAlignmentInBytes == Address::times_8) {
5399       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
5400     } else {
5401       if (dst != src) {
5402         movq(dst, src);
5403       }
5404       shlq(dst, LogMinObjAlignmentInBytes);
5405       if (CompressedOops::base() != nullptr) {
5406         addq(dst, r12_heapbase);
5407       }
5408     }
5409   } else {
5410     assert (CompressedOops::base() == nullptr, "sanity");
5411     if (dst != src) {
5412       movq(dst, src);
5413     }
5414   }
5415 }
5416 
5417 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
5418   BLOCK_COMMENT("encode_klass_not_null {");
5419   assert_different_registers(r, tmp);
5420   if (CompressedKlassPointers::base() != nullptr) {
5421     if (AOTCodeCache::is_on_for_dump()) {
5422       movptr(tmp, ExternalAddress(CompressedKlassPointers::base_addr()));
5423     } else {
5424       movptr(tmp, (intptr_t)CompressedKlassPointers::base());
5425     }
5426     subq(r, tmp);
5427   }
5428   if (CompressedKlassPointers::shift() != 0) {
5429     shrq(r, CompressedKlassPointers::shift());
5430   }
5431   BLOCK_COMMENT("} encode_klass_not_null");
5432 }
5433 
5434 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
5435   BLOCK_COMMENT("encode_and_move_klass_not_null {");
5436   assert_different_registers(src, dst);
5437   if (CompressedKlassPointers::base() != nullptr) {
5438     movptr(dst, -(intptr_t)CompressedKlassPointers::base());
5439     addq(dst, src);
5440   } else {
5441     movptr(dst, src);
5442   }
5443   if (CompressedKlassPointers::shift() != 0) {
5444     shrq(dst, CompressedKlassPointers::shift());
5445   }
5446   BLOCK_COMMENT("} encode_and_move_klass_not_null");
5447 }
5448 
5449 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
5450   BLOCK_COMMENT("decode_klass_not_null {");
5451   assert_different_registers(r, tmp);
5452   // Note: it will change flags
5453   assert(UseCompressedClassPointers, "should only be used for compressed headers");
5454   // Cannot assert, unverified entry point counts instructions (see .ad file)
5455   // vtableStubs also counts instructions in pd_code_size_limit.
5456   // Also do not verify_oop as this is called by verify_oop.
5457   if (CompressedKlassPointers::shift() != 0) {
5458     shlq(r, CompressedKlassPointers::shift());
5459   }
5460   if (CompressedKlassPointers::base() != nullptr) {
5461     if (AOTCodeCache::is_on_for_dump()) {
5462       movptr(tmp, ExternalAddress(CompressedKlassPointers::base_addr()));
5463     } else {
5464       movptr(tmp, (intptr_t)CompressedKlassPointers::base());
5465     }
5466     addq(r, tmp);
5467   }
5468   BLOCK_COMMENT("} decode_klass_not_null");
5469 }
5470 
5471 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
5472   BLOCK_COMMENT("decode_and_move_klass_not_null {");
5473   assert_different_registers(src, dst);
5474   // Note: it will change flags
5475   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5476   // Cannot assert, unverified entry point counts instructions (see .ad file)
5477   // vtableStubs also counts instructions in pd_code_size_limit.
5478   // Also do not verify_oop as this is called by verify_oop.
5479 
5480   if (CompressedKlassPointers::base() == nullptr &&
5481       CompressedKlassPointers::shift() == 0) {
5482     // The best case scenario is that there is no base or shift. Then it is already
5483     // a pointer that needs nothing but a register rename.
5484     movl(dst, src);
5485   } else {
5486     if (CompressedKlassPointers::shift() <= Address::times_8) {
5487       if (CompressedKlassPointers::base() != nullptr) {
5488         movptr(dst, (intptr_t)CompressedKlassPointers::base());
5489       } else {
5490         xorq(dst, dst);
5491       }
5492       if (CompressedKlassPointers::shift() != 0) {
5493         assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
5494         leaq(dst, Address(dst, src, Address::times_8, 0));
5495       } else {
5496         addq(dst, src);
5497       }
5498     } else {
5499       if (CompressedKlassPointers::base() != nullptr) {
5500         const intptr_t base_right_shifted =
5501             (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5502         movptr(dst, base_right_shifted);
5503       } else {
5504         xorq(dst, dst);
5505       }
5506       addq(dst, src);
5507       shlq(dst, CompressedKlassPointers::shift());
5508     }
5509   }
5510   BLOCK_COMMENT("} decode_and_move_klass_not_null");
5511 }
5512 
5513 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5514   assert (UseCompressedOops, "should only be used for compressed headers");
5515   assert (Universe::heap() != nullptr, "java heap should be initialized");
5516   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5517   int oop_index = oop_recorder()->find_index(obj);
5518   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5519   mov_narrow_oop(dst, oop_index, rspec);
5520 }
5521 
5522 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
5523   assert (UseCompressedOops, "should only be used for compressed headers");
5524   assert (Universe::heap() != nullptr, "java heap should be initialized");
5525   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5526   int oop_index = oop_recorder()->find_index(obj);
5527   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5528   mov_narrow_oop(dst, oop_index, rspec);
5529 }
5530 
5531 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
5532   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5533   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5534   int klass_index = oop_recorder()->find_index(k);
5535   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5536   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5537 }
5538 
5539 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
5540   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5541   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5542   int klass_index = oop_recorder()->find_index(k);
5543   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5544   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5545 }
5546 
5547 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
5548   assert (UseCompressedOops, "should only be used for compressed headers");
5549   assert (Universe::heap() != nullptr, "java heap should be initialized");
5550   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5551   int oop_index = oop_recorder()->find_index(obj);
5552   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5553   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5554 }
5555 
5556 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
5557   assert (UseCompressedOops, "should only be used for compressed headers");
5558   assert (Universe::heap() != nullptr, "java heap should be initialized");
5559   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5560   int oop_index = oop_recorder()->find_index(obj);
5561   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5562   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5563 }
5564 
5565 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
5566   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5567   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5568   int klass_index = oop_recorder()->find_index(k);
5569   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5570   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5571 }
5572 
5573 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
5574   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5575   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5576   int klass_index = oop_recorder()->find_index(k);
5577   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5578   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5579 }
5580 
5581 void MacroAssembler::reinit_heapbase() {
5582   if (UseCompressedOops) {
5583     if (Universe::heap() != nullptr) {
5584       if (CompressedOops::base() == nullptr) {
5585         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5586       } else {
5587         mov64(r12_heapbase, (int64_t)CompressedOops::base());
5588       }
5589     } else {
5590       movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr()));
5591     }
5592   }
5593 }
5594 
5595 #if COMPILER2_OR_JVMCI
5596 
5597 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5598 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5599   // cnt - number of qwords (8-byte words).
5600   // base - start address, qword aligned.
5601   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5602   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
5603   if (use64byteVector) {
5604     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5605   } else if (MaxVectorSize >= 32) {
5606     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5607   } else {
5608     pxor(xtmp, xtmp);
5609   }
5610   jmp(L_zero_64_bytes);
5611 
5612   BIND(L_loop);
5613   if (MaxVectorSize >= 32) {
5614     fill64(base, 0, xtmp, use64byteVector);
5615   } else {
5616     movdqu(Address(base,  0), xtmp);
5617     movdqu(Address(base, 16), xtmp);
5618     movdqu(Address(base, 32), xtmp);
5619     movdqu(Address(base, 48), xtmp);
5620   }
5621   addptr(base, 64);
5622 
5623   BIND(L_zero_64_bytes);
5624   subptr(cnt, 8);
5625   jccb(Assembler::greaterEqual, L_loop);
5626 
5627   // Copy trailing 64 bytes
5628   if (use64byteVector) {
5629     addptr(cnt, 8);
5630     jccb(Assembler::equal, L_end);
5631     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
5632     jmp(L_end);
5633   } else {
5634     addptr(cnt, 4);
5635     jccb(Assembler::less, L_tail);
5636     if (MaxVectorSize >= 32) {
5637       vmovdqu(Address(base, 0), xtmp);
5638     } else {
5639       movdqu(Address(base,  0), xtmp);
5640       movdqu(Address(base, 16), xtmp);
5641     }
5642   }
5643   addptr(base, 32);
5644   subptr(cnt, 4);
5645 
5646   BIND(L_tail);
5647   addptr(cnt, 4);
5648   jccb(Assembler::lessEqual, L_end);
5649   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
5650     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
5651   } else {
5652     decrement(cnt);
5653 
5654     BIND(L_sloop);
5655     movq(Address(base, 0), xtmp);
5656     addptr(base, 8);
5657     decrement(cnt);
5658     jccb(Assembler::greaterEqual, L_sloop);
5659   }
5660   BIND(L_end);
5661 }
5662 
5663 // Clearing constant sized memory using YMM/ZMM registers.
5664 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5665   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
5666   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
5667 
5668   int vector64_count = (cnt & (~0x7)) >> 3;
5669   cnt = cnt & 0x7;
5670   const int fill64_per_loop = 4;
5671   const int max_unrolled_fill64 = 8;
5672 
5673   // 64 byte initialization loop.
5674   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
5675   int start64 = 0;
5676   if (vector64_count > max_unrolled_fill64) {
5677     Label LOOP;
5678     Register index = rtmp;
5679 
5680     start64 = vector64_count - (vector64_count % fill64_per_loop);
5681 
5682     movl(index, 0);
5683     BIND(LOOP);
5684     for (int i = 0; i < fill64_per_loop; i++) {
5685       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
5686     }
5687     addl(index, fill64_per_loop * 64);
5688     cmpl(index, start64 * 64);
5689     jccb(Assembler::less, LOOP);
5690   }
5691   for (int i = start64; i < vector64_count; i++) {
5692     fill64(base, i * 64, xtmp, use64byteVector);
5693   }
5694 
5695   // Clear remaining 64 byte tail.
5696   int disp = vector64_count * 64;
5697   if (cnt) {
5698     switch (cnt) {
5699       case 1:
5700         movq(Address(base, disp), xtmp);
5701         break;
5702       case 2:
5703         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
5704         break;
5705       case 3:
5706         movl(rtmp, 0x7);
5707         kmovwl(mask, rtmp);
5708         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
5709         break;
5710       case 4:
5711         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5712         break;
5713       case 5:
5714         if (use64byteVector) {
5715           movl(rtmp, 0x1F);
5716           kmovwl(mask, rtmp);
5717           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5718         } else {
5719           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5720           movq(Address(base, disp + 32), xtmp);
5721         }
5722         break;
5723       case 6:
5724         if (use64byteVector) {
5725           movl(rtmp, 0x3F);
5726           kmovwl(mask, rtmp);
5727           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5728         } else {
5729           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5730           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
5731         }
5732         break;
5733       case 7:
5734         if (use64byteVector) {
5735           movl(rtmp, 0x7F);
5736           kmovwl(mask, rtmp);
5737           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5738         } else {
5739           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5740           movl(rtmp, 0x7);
5741           kmovwl(mask, rtmp);
5742           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
5743         }
5744         break;
5745       default:
5746         fatal("Unexpected length : %d\n",cnt);
5747         break;
5748     }
5749   }
5750 }
5751 
5752 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
5753                                bool is_large, KRegister mask) {
5754   // cnt      - number of qwords (8-byte words).
5755   // base     - start address, qword aligned.
5756   // is_large - if optimizers know cnt is larger than InitArrayShortSize
5757   assert(base==rdi, "base register must be edi for rep stos");
5758   assert(tmp==rax,   "tmp register must be eax for rep stos");
5759   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
5760   assert(InitArrayShortSize % BytesPerLong == 0,
5761     "InitArrayShortSize should be the multiple of BytesPerLong");
5762 
5763   Label DONE;
5764   if (!is_large || !UseXMMForObjInit) {
5765     xorptr(tmp, tmp);
5766   }
5767 
5768   if (!is_large) {
5769     Label LOOP, LONG;
5770     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
5771     jccb(Assembler::greater, LONG);
5772 
5773     decrement(cnt);
5774     jccb(Assembler::negative, DONE); // Zero length
5775 
5776     // Use individual pointer-sized stores for small counts:
5777     BIND(LOOP);
5778     movptr(Address(base, cnt, Address::times_ptr), tmp);
5779     decrement(cnt);
5780     jccb(Assembler::greaterEqual, LOOP);
5781     jmpb(DONE);
5782 
5783     BIND(LONG);
5784   }
5785 
5786   // Use longer rep-prefixed ops for non-small counts:
5787   if (UseFastStosb) {
5788     shlptr(cnt, 3); // convert to number of bytes
5789     rep_stosb();
5790   } else if (UseXMMForObjInit) {
5791     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
5792   } else {
5793     rep_stos();
5794   }
5795 
5796   BIND(DONE);
5797 }
5798 
5799 #endif //COMPILER2_OR_JVMCI
5800 
5801 
5802 void MacroAssembler::generate_fill(BasicType t, bool aligned,
5803                                    Register to, Register value, Register count,
5804                                    Register rtmp, XMMRegister xtmp) {
5805   ShortBranchVerifier sbv(this);
5806   assert_different_registers(to, value, count, rtmp);
5807   Label L_exit;
5808   Label L_fill_2_bytes, L_fill_4_bytes;
5809 
5810 #if defined(COMPILER2)
5811   if(MaxVectorSize >=32 &&
5812      VM_Version::supports_avx512vlbw() &&
5813      VM_Version::supports_bmi2()) {
5814     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
5815     return;
5816   }
5817 #endif
5818 
5819   int shift = -1;
5820   switch (t) {
5821     case T_BYTE:
5822       shift = 2;
5823       break;
5824     case T_SHORT:
5825       shift = 1;
5826       break;
5827     case T_INT:
5828       shift = 0;
5829       break;
5830     default: ShouldNotReachHere();
5831   }
5832 
5833   if (t == T_BYTE) {
5834     andl(value, 0xff);
5835     movl(rtmp, value);
5836     shll(rtmp, 8);
5837     orl(value, rtmp);
5838   }
5839   if (t == T_SHORT) {
5840     andl(value, 0xffff);
5841   }
5842   if (t == T_BYTE || t == T_SHORT) {
5843     movl(rtmp, value);
5844     shll(rtmp, 16);
5845     orl(value, rtmp);
5846   }
5847 
5848   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
5849   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
5850   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
5851     Label L_skip_align2;
5852     // align source address at 4 bytes address boundary
5853     if (t == T_BYTE) {
5854       Label L_skip_align1;
5855       // One byte misalignment happens only for byte arrays
5856       testptr(to, 1);
5857       jccb(Assembler::zero, L_skip_align1);
5858       movb(Address(to, 0), value);
5859       increment(to);
5860       decrement(count);
5861       BIND(L_skip_align1);
5862     }
5863     // Two bytes misalignment happens only for byte and short (char) arrays
5864     testptr(to, 2);
5865     jccb(Assembler::zero, L_skip_align2);
5866     movw(Address(to, 0), value);
5867     addptr(to, 2);
5868     subptr(count, 1<<(shift-1));
5869     BIND(L_skip_align2);
5870   }
5871   {
5872     Label L_fill_32_bytes;
5873     if (!UseUnalignedLoadStores) {
5874       // align to 8 bytes, we know we are 4 byte aligned to start
5875       testptr(to, 4);
5876       jccb(Assembler::zero, L_fill_32_bytes);
5877       movl(Address(to, 0), value);
5878       addptr(to, 4);
5879       subptr(count, 1<<shift);
5880     }
5881     BIND(L_fill_32_bytes);
5882     {
5883       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
5884       movdl(xtmp, value);
5885       if (UseAVX >= 2 && UseUnalignedLoadStores) {
5886         Label L_check_fill_32_bytes;
5887         if (UseAVX > 2) {
5888           // Fill 64-byte chunks
5889           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
5890 
5891           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
5892           cmpptr(count, VM_Version::avx3_threshold());
5893           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
5894 
5895           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
5896 
5897           subptr(count, 16 << shift);
5898           jccb(Assembler::less, L_check_fill_32_bytes);
5899           align(16);
5900 
5901           BIND(L_fill_64_bytes_loop_avx3);
5902           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
5903           addptr(to, 64);
5904           subptr(count, 16 << shift);
5905           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
5906           jmpb(L_check_fill_32_bytes);
5907 
5908           BIND(L_check_fill_64_bytes_avx2);
5909         }
5910         // Fill 64-byte chunks
5911         Label L_fill_64_bytes_loop;
5912         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
5913 
5914         subptr(count, 16 << shift);
5915         jcc(Assembler::less, L_check_fill_32_bytes);
5916         align(16);
5917 
5918         BIND(L_fill_64_bytes_loop);
5919         vmovdqu(Address(to, 0), xtmp);
5920         vmovdqu(Address(to, 32), xtmp);
5921         addptr(to, 64);
5922         subptr(count, 16 << shift);
5923         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
5924 
5925         BIND(L_check_fill_32_bytes);
5926         addptr(count, 8 << shift);
5927         jccb(Assembler::less, L_check_fill_8_bytes);
5928         vmovdqu(Address(to, 0), xtmp);
5929         addptr(to, 32);
5930         subptr(count, 8 << shift);
5931 
5932         BIND(L_check_fill_8_bytes);
5933         // clean upper bits of YMM registers
5934         movdl(xtmp, value);
5935         pshufd(xtmp, xtmp, 0);
5936       } else {
5937         // Fill 32-byte chunks
5938         pshufd(xtmp, xtmp, 0);
5939 
5940         subptr(count, 8 << shift);
5941         jcc(Assembler::less, L_check_fill_8_bytes);
5942         align(16);
5943 
5944         BIND(L_fill_32_bytes_loop);
5945 
5946         if (UseUnalignedLoadStores) {
5947           movdqu(Address(to, 0), xtmp);
5948           movdqu(Address(to, 16), xtmp);
5949         } else {
5950           movq(Address(to, 0), xtmp);
5951           movq(Address(to, 8), xtmp);
5952           movq(Address(to, 16), xtmp);
5953           movq(Address(to, 24), xtmp);
5954         }
5955 
5956         addptr(to, 32);
5957         subptr(count, 8 << shift);
5958         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
5959 
5960         BIND(L_check_fill_8_bytes);
5961       }
5962       addptr(count, 8 << shift);
5963       jccb(Assembler::zero, L_exit);
5964       jmpb(L_fill_8_bytes);
5965 
5966       //
5967       // length is too short, just fill qwords
5968       //
5969       BIND(L_fill_8_bytes_loop);
5970       movq(Address(to, 0), xtmp);
5971       addptr(to, 8);
5972       BIND(L_fill_8_bytes);
5973       subptr(count, 1 << (shift + 1));
5974       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
5975     }
5976   }
5977   // fill trailing 4 bytes
5978   BIND(L_fill_4_bytes);
5979   testl(count, 1<<shift);
5980   jccb(Assembler::zero, L_fill_2_bytes);
5981   movl(Address(to, 0), value);
5982   if (t == T_BYTE || t == T_SHORT) {
5983     Label L_fill_byte;
5984     addptr(to, 4);
5985     BIND(L_fill_2_bytes);
5986     // fill trailing 2 bytes
5987     testl(count, 1<<(shift-1));
5988     jccb(Assembler::zero, L_fill_byte);
5989     movw(Address(to, 0), value);
5990     if (t == T_BYTE) {
5991       addptr(to, 2);
5992       BIND(L_fill_byte);
5993       // fill trailing byte
5994       testl(count, 1);
5995       jccb(Assembler::zero, L_exit);
5996       movb(Address(to, 0), value);
5997     } else {
5998       BIND(L_fill_byte);
5999     }
6000   } else {
6001     BIND(L_fill_2_bytes);
6002   }
6003   BIND(L_exit);
6004 }
6005 
6006 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
6007   switch(type) {
6008     case T_BYTE:
6009     case T_BOOLEAN:
6010       evpbroadcastb(dst, src, vector_len);
6011       break;
6012     case T_SHORT:
6013     case T_CHAR:
6014       evpbroadcastw(dst, src, vector_len);
6015       break;
6016     case T_INT:
6017     case T_FLOAT:
6018       evpbroadcastd(dst, src, vector_len);
6019       break;
6020     case T_LONG:
6021     case T_DOUBLE:
6022       evpbroadcastq(dst, src, vector_len);
6023       break;
6024     default:
6025       fatal("Unhandled type : %s", type2name(type));
6026       break;
6027   }
6028 }
6029 
6030 // Encode given char[]/byte[] to byte[] in ISO_8859_1 or ASCII
6031 //
6032 // @IntrinsicCandidate
6033 // int sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(
6034 //         char[] sa, int sp, byte[] da, int dp, int len) {
6035 //     int i = 0;
6036 //     for (; i < len; i++) {
6037 //         char c = sa[sp++];
6038 //         if (c > '\u00FF')
6039 //             break;
6040 //         da[dp++] = (byte) c;
6041 //     }
6042 //     return i;
6043 // }
6044 //
6045 // @IntrinsicCandidate
6046 // int java.lang.StringCoding.encodeISOArray0(
6047 //         byte[] sa, int sp, byte[] da, int dp, int len) {
6048 //   int i = 0;
6049 //   for (; i < len; i++) {
6050 //     char c = StringUTF16.getChar(sa, sp++);
6051 //     if (c > '\u00FF')
6052 //       break;
6053 //     da[dp++] = (byte) c;
6054 //   }
6055 //   return i;
6056 // }
6057 //
6058 // @IntrinsicCandidate
6059 // int java.lang.StringCoding.encodeAsciiArray0(
6060 //         char[] sa, int sp, byte[] da, int dp, int len) {
6061 //   int i = 0;
6062 //   for (; i < len; i++) {
6063 //     char c = sa[sp++];
6064 //     if (c >= '\u0080')
6065 //       break;
6066 //     da[dp++] = (byte) c;
6067 //   }
6068 //   return i;
6069 // }
6070 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
6071   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
6072   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
6073   Register tmp5, Register result, bool ascii) {
6074 
6075   // rsi: src
6076   // rdi: dst
6077   // rdx: len
6078   // rcx: tmp5
6079   // rax: result
6080   ShortBranchVerifier sbv(this);
6081   assert_different_registers(src, dst, len, tmp5, result);
6082   Label L_done, L_copy_1_char, L_copy_1_char_exit;
6083 
6084   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
6085   int short_mask = ascii ? 0xff80 : 0xff00;
6086 
6087   // set result
6088   xorl(result, result);
6089   // check for zero length
6090   testl(len, len);
6091   jcc(Assembler::zero, L_done);
6092 
6093   movl(result, len);
6094 
6095   // Setup pointers
6096   lea(src, Address(src, len, Address::times_2)); // char[]
6097   lea(dst, Address(dst, len, Address::times_1)); // byte[]
6098   negptr(len);
6099 
6100   if (UseSSE42Intrinsics || UseAVX >= 2) {
6101     Label L_copy_8_chars, L_copy_8_chars_exit;
6102     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
6103 
6104     if (UseAVX >= 2) {
6105       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
6106       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
6107       movdl(tmp1Reg, tmp5);
6108       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
6109       jmp(L_chars_32_check);
6110 
6111       bind(L_copy_32_chars);
6112       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
6113       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
6114       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
6115       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
6116       jccb(Assembler::notZero, L_copy_32_chars_exit);
6117       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
6118       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
6119       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
6120 
6121       bind(L_chars_32_check);
6122       addptr(len, 32);
6123       jcc(Assembler::lessEqual, L_copy_32_chars);
6124 
6125       bind(L_copy_32_chars_exit);
6126       subptr(len, 16);
6127       jccb(Assembler::greater, L_copy_16_chars_exit);
6128 
6129     } else if (UseSSE42Intrinsics) {
6130       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
6131       movdl(tmp1Reg, tmp5);
6132       pshufd(tmp1Reg, tmp1Reg, 0);
6133       jmpb(L_chars_16_check);
6134     }
6135 
6136     bind(L_copy_16_chars);
6137     if (UseAVX >= 2) {
6138       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
6139       vptest(tmp2Reg, tmp1Reg);
6140       jcc(Assembler::notZero, L_copy_16_chars_exit);
6141       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
6142       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
6143     } else {
6144       if (UseAVX > 0) {
6145         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6146         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6147         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
6148       } else {
6149         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6150         por(tmp2Reg, tmp3Reg);
6151         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6152         por(tmp2Reg, tmp4Reg);
6153       }
6154       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
6155       jccb(Assembler::notZero, L_copy_16_chars_exit);
6156       packuswb(tmp3Reg, tmp4Reg);
6157     }
6158     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
6159 
6160     bind(L_chars_16_check);
6161     addptr(len, 16);
6162     jcc(Assembler::lessEqual, L_copy_16_chars);
6163 
6164     bind(L_copy_16_chars_exit);
6165     if (UseAVX >= 2) {
6166       // clean upper bits of YMM registers
6167       vpxor(tmp2Reg, tmp2Reg);
6168       vpxor(tmp3Reg, tmp3Reg);
6169       vpxor(tmp4Reg, tmp4Reg);
6170       movdl(tmp1Reg, tmp5);
6171       pshufd(tmp1Reg, tmp1Reg, 0);
6172     }
6173     subptr(len, 8);
6174     jccb(Assembler::greater, L_copy_8_chars_exit);
6175 
6176     bind(L_copy_8_chars);
6177     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
6178     ptest(tmp3Reg, tmp1Reg);
6179     jccb(Assembler::notZero, L_copy_8_chars_exit);
6180     packuswb(tmp3Reg, tmp1Reg);
6181     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
6182     addptr(len, 8);
6183     jccb(Assembler::lessEqual, L_copy_8_chars);
6184 
6185     bind(L_copy_8_chars_exit);
6186     subptr(len, 8);
6187     jccb(Assembler::zero, L_done);
6188   }
6189 
6190   bind(L_copy_1_char);
6191   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
6192   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
6193   jccb(Assembler::notZero, L_copy_1_char_exit);
6194   movb(Address(dst, len, Address::times_1, 0), tmp5);
6195   addptr(len, 1);
6196   jccb(Assembler::less, L_copy_1_char);
6197 
6198   bind(L_copy_1_char_exit);
6199   addptr(result, len); // len is negative count of not processed elements
6200 
6201   bind(L_done);
6202 }
6203 
6204 /**
6205  * Helper for multiply_to_len().
6206  */
6207 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
6208   addq(dest_lo, src1);
6209   adcq(dest_hi, 0);
6210   addq(dest_lo, src2);
6211   adcq(dest_hi, 0);
6212 }
6213 
6214 /**
6215  * Multiply 64 bit by 64 bit first loop.
6216  */
6217 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
6218                                            Register y, Register y_idx, Register z,
6219                                            Register carry, Register product,
6220                                            Register idx, Register kdx) {
6221   //
6222   //  jlong carry, x[], y[], z[];
6223   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
6224   //    huge_128 product = y[idx] * x[xstart] + carry;
6225   //    z[kdx] = (jlong)product;
6226   //    carry  = (jlong)(product >>> 64);
6227   //  }
6228   //  z[xstart] = carry;
6229   //
6230 
6231   Label L_first_loop, L_first_loop_exit;
6232   Label L_one_x, L_one_y, L_multiply;
6233 
6234   decrementl(xstart);
6235   jcc(Assembler::negative, L_one_x);
6236 
6237   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
6238   rorq(x_xstart, 32); // convert big-endian to little-endian
6239 
6240   bind(L_first_loop);
6241   decrementl(idx);
6242   jcc(Assembler::negative, L_first_loop_exit);
6243   decrementl(idx);
6244   jcc(Assembler::negative, L_one_y);
6245   movq(y_idx, Address(y, idx, Address::times_4,  0));
6246   rorq(y_idx, 32); // convert big-endian to little-endian
6247   bind(L_multiply);
6248   movq(product, x_xstart);
6249   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
6250   addq(product, carry);
6251   adcq(rdx, 0);
6252   subl(kdx, 2);
6253   movl(Address(z, kdx, Address::times_4,  4), product);
6254   shrq(product, 32);
6255   movl(Address(z, kdx, Address::times_4,  0), product);
6256   movq(carry, rdx);
6257   jmp(L_first_loop);
6258 
6259   bind(L_one_y);
6260   movl(y_idx, Address(y,  0));
6261   jmp(L_multiply);
6262 
6263   bind(L_one_x);
6264   movl(x_xstart, Address(x,  0));
6265   jmp(L_first_loop);
6266 
6267   bind(L_first_loop_exit);
6268 }
6269 
6270 /**
6271  * Multiply 64 bit by 64 bit and add 128 bit.
6272  */
6273 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
6274                                             Register yz_idx, Register idx,
6275                                             Register carry, Register product, int offset) {
6276   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
6277   //     z[kdx] = (jlong)product;
6278 
6279   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
6280   rorq(yz_idx, 32); // convert big-endian to little-endian
6281   movq(product, x_xstart);
6282   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
6283   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
6284   rorq(yz_idx, 32); // convert big-endian to little-endian
6285 
6286   add2_with_carry(rdx, product, carry, yz_idx);
6287 
6288   movl(Address(z, idx, Address::times_4,  offset+4), product);
6289   shrq(product, 32);
6290   movl(Address(z, idx, Address::times_4,  offset), product);
6291 
6292 }
6293 
6294 /**
6295  * Multiply 128 bit by 128 bit. Unrolled inner loop.
6296  */
6297 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
6298                                              Register yz_idx, Register idx, Register jdx,
6299                                              Register carry, Register product,
6300                                              Register carry2) {
6301   //   jlong carry, x[], y[], z[];
6302   //   int kdx = ystart+1;
6303   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
6304   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
6305   //     z[kdx+idx+1] = (jlong)product;
6306   //     jlong carry2  = (jlong)(product >>> 64);
6307   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
6308   //     z[kdx+idx] = (jlong)product;
6309   //     carry  = (jlong)(product >>> 64);
6310   //   }
6311   //   idx += 2;
6312   //   if (idx > 0) {
6313   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
6314   //     z[kdx+idx] = (jlong)product;
6315   //     carry  = (jlong)(product >>> 64);
6316   //   }
6317   //
6318 
6319   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
6320 
6321   movl(jdx, idx);
6322   andl(jdx, 0xFFFFFFFC);
6323   shrl(jdx, 2);
6324 
6325   bind(L_third_loop);
6326   subl(jdx, 1);
6327   jcc(Assembler::negative, L_third_loop_exit);
6328   subl(idx, 4);
6329 
6330   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
6331   movq(carry2, rdx);
6332 
6333   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
6334   movq(carry, rdx);
6335   jmp(L_third_loop);
6336 
6337   bind (L_third_loop_exit);
6338 
6339   andl (idx, 0x3);
6340   jcc(Assembler::zero, L_post_third_loop_done);
6341 
6342   Label L_check_1;
6343   subl(idx, 2);
6344   jcc(Assembler::negative, L_check_1);
6345 
6346   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
6347   movq(carry, rdx);
6348 
6349   bind (L_check_1);
6350   addl (idx, 0x2);
6351   andl (idx, 0x1);
6352   subl(idx, 1);
6353   jcc(Assembler::negative, L_post_third_loop_done);
6354 
6355   movl(yz_idx, Address(y, idx, Address::times_4,  0));
6356   movq(product, x_xstart);
6357   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
6358   movl(yz_idx, Address(z, idx, Address::times_4,  0));
6359 
6360   add2_with_carry(rdx, product, yz_idx, carry);
6361 
6362   movl(Address(z, idx, Address::times_4,  0), product);
6363   shrq(product, 32);
6364 
6365   shlq(rdx, 32);
6366   orq(product, rdx);
6367   movq(carry, product);
6368 
6369   bind(L_post_third_loop_done);
6370 }
6371 
6372 /**
6373  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
6374  *
6375  */
6376 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
6377                                                   Register carry, Register carry2,
6378                                                   Register idx, Register jdx,
6379                                                   Register yz_idx1, Register yz_idx2,
6380                                                   Register tmp, Register tmp3, Register tmp4) {
6381   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
6382 
6383   //   jlong carry, x[], y[], z[];
6384   //   int kdx = ystart+1;
6385   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
6386   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
6387   //     jlong carry2  = (jlong)(tmp3 >>> 64);
6388   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
6389   //     carry  = (jlong)(tmp4 >>> 64);
6390   //     z[kdx+idx+1] = (jlong)tmp3;
6391   //     z[kdx+idx] = (jlong)tmp4;
6392   //   }
6393   //   idx += 2;
6394   //   if (idx > 0) {
6395   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
6396   //     z[kdx+idx] = (jlong)yz_idx1;
6397   //     carry  = (jlong)(yz_idx1 >>> 64);
6398   //   }
6399   //
6400 
6401   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
6402 
6403   movl(jdx, idx);
6404   andl(jdx, 0xFFFFFFFC);
6405   shrl(jdx, 2);
6406 
6407   bind(L_third_loop);
6408   subl(jdx, 1);
6409   jcc(Assembler::negative, L_third_loop_exit);
6410   subl(idx, 4);
6411 
6412   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
6413   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
6414   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
6415   rorxq(yz_idx2, yz_idx2, 32);
6416 
6417   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
6418   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
6419 
6420   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
6421   rorxq(yz_idx1, yz_idx1, 32);
6422   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
6423   rorxq(yz_idx2, yz_idx2, 32);
6424 
6425   if (VM_Version::supports_adx()) {
6426     adcxq(tmp3, carry);
6427     adoxq(tmp3, yz_idx1);
6428 
6429     adcxq(tmp4, tmp);
6430     adoxq(tmp4, yz_idx2);
6431 
6432     movl(carry, 0); // does not affect flags
6433     adcxq(carry2, carry);
6434     adoxq(carry2, carry);
6435   } else {
6436     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
6437     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
6438   }
6439   movq(carry, carry2);
6440 
6441   movl(Address(z, idx, Address::times_4, 12), tmp3);
6442   shrq(tmp3, 32);
6443   movl(Address(z, idx, Address::times_4,  8), tmp3);
6444 
6445   movl(Address(z, idx, Address::times_4,  4), tmp4);
6446   shrq(tmp4, 32);
6447   movl(Address(z, idx, Address::times_4,  0), tmp4);
6448 
6449   jmp(L_third_loop);
6450 
6451   bind (L_third_loop_exit);
6452 
6453   andl (idx, 0x3);
6454   jcc(Assembler::zero, L_post_third_loop_done);
6455 
6456   Label L_check_1;
6457   subl(idx, 2);
6458   jcc(Assembler::negative, L_check_1);
6459 
6460   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
6461   rorxq(yz_idx1, yz_idx1, 32);
6462   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
6463   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
6464   rorxq(yz_idx2, yz_idx2, 32);
6465 
6466   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
6467 
6468   movl(Address(z, idx, Address::times_4,  4), tmp3);
6469   shrq(tmp3, 32);
6470   movl(Address(z, idx, Address::times_4,  0), tmp3);
6471   movq(carry, tmp4);
6472 
6473   bind (L_check_1);
6474   addl (idx, 0x2);
6475   andl (idx, 0x1);
6476   subl(idx, 1);
6477   jcc(Assembler::negative, L_post_third_loop_done);
6478   movl(tmp4, Address(y, idx, Address::times_4,  0));
6479   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
6480   movl(tmp4, Address(z, idx, Address::times_4,  0));
6481 
6482   add2_with_carry(carry2, tmp3, tmp4, carry);
6483 
6484   movl(Address(z, idx, Address::times_4,  0), tmp3);
6485   shrq(tmp3, 32);
6486 
6487   shlq(carry2, 32);
6488   orq(tmp3, carry2);
6489   movq(carry, tmp3);
6490 
6491   bind(L_post_third_loop_done);
6492 }
6493 
6494 /**
6495  * Code for BigInteger::multiplyToLen() intrinsic.
6496  *
6497  * rdi: x
6498  * rax: xlen
6499  * rsi: y
6500  * rcx: ylen
6501  * r8:  z
6502  * r11: tmp0
6503  * r12: tmp1
6504  * r13: tmp2
6505  * r14: tmp3
6506  * r15: tmp4
6507  * rbx: tmp5
6508  *
6509  */
6510 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
6511                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
6512   ShortBranchVerifier sbv(this);
6513   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
6514 
6515   push(tmp0);
6516   push(tmp1);
6517   push(tmp2);
6518   push(tmp3);
6519   push(tmp4);
6520   push(tmp5);
6521 
6522   push(xlen);
6523 
6524   const Register idx = tmp1;
6525   const Register kdx = tmp2;
6526   const Register xstart = tmp3;
6527 
6528   const Register y_idx = tmp4;
6529   const Register carry = tmp5;
6530   const Register product  = xlen;
6531   const Register x_xstart = tmp0;
6532 
6533   // First Loop.
6534   //
6535   //  final static long LONG_MASK = 0xffffffffL;
6536   //  int xstart = xlen - 1;
6537   //  int ystart = ylen - 1;
6538   //  long carry = 0;
6539   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
6540   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
6541   //    z[kdx] = (int)product;
6542   //    carry = product >>> 32;
6543   //  }
6544   //  z[xstart] = (int)carry;
6545   //
6546 
6547   movl(idx, ylen);               // idx = ylen;
6548   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
6549   xorq(carry, carry);            // carry = 0;
6550 
6551   Label L_done;
6552 
6553   movl(xstart, xlen);
6554   decrementl(xstart);
6555   jcc(Assembler::negative, L_done);
6556 
6557   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
6558 
6559   Label L_second_loop;
6560   testl(kdx, kdx);
6561   jcc(Assembler::zero, L_second_loop);
6562 
6563   Label L_carry;
6564   subl(kdx, 1);
6565   jcc(Assembler::zero, L_carry);
6566 
6567   movl(Address(z, kdx, Address::times_4,  0), carry);
6568   shrq(carry, 32);
6569   subl(kdx, 1);
6570 
6571   bind(L_carry);
6572   movl(Address(z, kdx, Address::times_4,  0), carry);
6573 
6574   // Second and third (nested) loops.
6575   //
6576   // for (int i = xstart-1; i >= 0; i--) { // Second loop
6577   //   carry = 0;
6578   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
6579   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
6580   //                    (z[k] & LONG_MASK) + carry;
6581   //     z[k] = (int)product;
6582   //     carry = product >>> 32;
6583   //   }
6584   //   z[i] = (int)carry;
6585   // }
6586   //
6587   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
6588 
6589   const Register jdx = tmp1;
6590 
6591   bind(L_second_loop);
6592   xorl(carry, carry);    // carry = 0;
6593   movl(jdx, ylen);       // j = ystart+1
6594 
6595   subl(xstart, 1);       // i = xstart-1;
6596   jcc(Assembler::negative, L_done);
6597 
6598   push (z);
6599 
6600   Label L_last_x;
6601   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
6602   subl(xstart, 1);       // i = xstart-1;
6603   jcc(Assembler::negative, L_last_x);
6604 
6605   if (UseBMI2Instructions) {
6606     movq(rdx,  Address(x, xstart, Address::times_4,  0));
6607     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
6608   } else {
6609     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
6610     rorq(x_xstart, 32);  // convert big-endian to little-endian
6611   }
6612 
6613   Label L_third_loop_prologue;
6614   bind(L_third_loop_prologue);
6615 
6616   push (x);
6617   push (xstart);
6618   push (ylen);
6619 
6620 
6621   if (UseBMI2Instructions) {
6622     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
6623   } else { // !UseBMI2Instructions
6624     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
6625   }
6626 
6627   pop(ylen);
6628   pop(xlen);
6629   pop(x);
6630   pop(z);
6631 
6632   movl(tmp3, xlen);
6633   addl(tmp3, 1);
6634   movl(Address(z, tmp3, Address::times_4,  0), carry);
6635   subl(tmp3, 1);
6636   jccb(Assembler::negative, L_done);
6637 
6638   shrq(carry, 32);
6639   movl(Address(z, tmp3, Address::times_4,  0), carry);
6640   jmp(L_second_loop);
6641 
6642   // Next infrequent code is moved outside loops.
6643   bind(L_last_x);
6644   if (UseBMI2Instructions) {
6645     movl(rdx, Address(x,  0));
6646   } else {
6647     movl(x_xstart, Address(x,  0));
6648   }
6649   jmp(L_third_loop_prologue);
6650 
6651   bind(L_done);
6652 
6653   pop(xlen);
6654 
6655   pop(tmp5);
6656   pop(tmp4);
6657   pop(tmp3);
6658   pop(tmp2);
6659   pop(tmp1);
6660   pop(tmp0);
6661 }
6662 
6663 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
6664   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
6665   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
6666   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
6667   Label VECTOR8_TAIL, VECTOR4_TAIL;
6668   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
6669   Label SAME_TILL_END, DONE;
6670   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
6671 
6672   //scale is in rcx in both Win64 and Unix
6673   ShortBranchVerifier sbv(this);
6674 
6675   shlq(length);
6676   xorq(result, result);
6677 
6678   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
6679       VM_Version::supports_avx512vlbw()) {
6680     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
6681 
6682     cmpq(length, 64);
6683     jcc(Assembler::less, VECTOR32_TAIL);
6684 
6685     movq(tmp1, length);
6686     andq(tmp1, 0x3F);      // tail count
6687     andq(length, ~(0x3F)); //vector count
6688 
6689     bind(VECTOR64_LOOP);
6690     // AVX512 code to compare 64 byte vectors.
6691     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
6692     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
6693     kortestql(k7, k7);
6694     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
6695     addq(result, 64);
6696     subq(length, 64);
6697     jccb(Assembler::notZero, VECTOR64_LOOP);
6698 
6699     //bind(VECTOR64_TAIL);
6700     testq(tmp1, tmp1);
6701     jcc(Assembler::zero, SAME_TILL_END);
6702 
6703     //bind(VECTOR64_TAIL);
6704     // AVX512 code to compare up to 63 byte vectors.
6705     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
6706     shlxq(tmp2, tmp2, tmp1);
6707     notq(tmp2);
6708     kmovql(k3, tmp2);
6709 
6710     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
6711     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
6712 
6713     ktestql(k7, k3);
6714     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
6715 
6716     bind(VECTOR64_NOT_EQUAL);
6717     kmovql(tmp1, k7);
6718     notq(tmp1);
6719     tzcntq(tmp1, tmp1);
6720     addq(result, tmp1);
6721     shrq(result);
6722     jmp(DONE);
6723     bind(VECTOR32_TAIL);
6724   }
6725 
6726   cmpq(length, 8);
6727   jcc(Assembler::equal, VECTOR8_LOOP);
6728   jcc(Assembler::less, VECTOR4_TAIL);
6729 
6730   if (UseAVX >= 2) {
6731     Label VECTOR16_TAIL, VECTOR32_LOOP;
6732 
6733     cmpq(length, 16);
6734     jcc(Assembler::equal, VECTOR16_LOOP);
6735     jcc(Assembler::less, VECTOR8_LOOP);
6736 
6737     cmpq(length, 32);
6738     jccb(Assembler::less, VECTOR16_TAIL);
6739 
6740     subq(length, 32);
6741     bind(VECTOR32_LOOP);
6742     vmovdqu(rymm0, Address(obja, result));
6743     vmovdqu(rymm1, Address(objb, result));
6744     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
6745     vptest(rymm2, rymm2);
6746     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
6747     addq(result, 32);
6748     subq(length, 32);
6749     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
6750     addq(length, 32);
6751     jcc(Assembler::equal, SAME_TILL_END);
6752     //falling through if less than 32 bytes left //close the branch here.
6753 
6754     bind(VECTOR16_TAIL);
6755     cmpq(length, 16);
6756     jccb(Assembler::less, VECTOR8_TAIL);
6757     bind(VECTOR16_LOOP);
6758     movdqu(rymm0, Address(obja, result));
6759     movdqu(rymm1, Address(objb, result));
6760     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
6761     ptest(rymm2, rymm2);
6762     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6763     addq(result, 16);
6764     subq(length, 16);
6765     jcc(Assembler::equal, SAME_TILL_END);
6766     //falling through if less than 16 bytes left
6767   } else {//regular intrinsics
6768 
6769     cmpq(length, 16);
6770     jccb(Assembler::less, VECTOR8_TAIL);
6771 
6772     subq(length, 16);
6773     bind(VECTOR16_LOOP);
6774     movdqu(rymm0, Address(obja, result));
6775     movdqu(rymm1, Address(objb, result));
6776     pxor(rymm0, rymm1);
6777     ptest(rymm0, rymm0);
6778     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6779     addq(result, 16);
6780     subq(length, 16);
6781     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
6782     addq(length, 16);
6783     jcc(Assembler::equal, SAME_TILL_END);
6784     //falling through if less than 16 bytes left
6785   }
6786 
6787   bind(VECTOR8_TAIL);
6788   cmpq(length, 8);
6789   jccb(Assembler::less, VECTOR4_TAIL);
6790   bind(VECTOR8_LOOP);
6791   movq(tmp1, Address(obja, result));
6792   movq(tmp2, Address(objb, result));
6793   xorq(tmp1, tmp2);
6794   testq(tmp1, tmp1);
6795   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
6796   addq(result, 8);
6797   subq(length, 8);
6798   jcc(Assembler::equal, SAME_TILL_END);
6799   //falling through if less than 8 bytes left
6800 
6801   bind(VECTOR4_TAIL);
6802   cmpq(length, 4);
6803   jccb(Assembler::less, BYTES_TAIL);
6804   bind(VECTOR4_LOOP);
6805   movl(tmp1, Address(obja, result));
6806   xorl(tmp1, Address(objb, result));
6807   testl(tmp1, tmp1);
6808   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
6809   addq(result, 4);
6810   subq(length, 4);
6811   jcc(Assembler::equal, SAME_TILL_END);
6812   //falling through if less than 4 bytes left
6813 
6814   bind(BYTES_TAIL);
6815   bind(BYTES_LOOP);
6816   load_unsigned_byte(tmp1, Address(obja, result));
6817   load_unsigned_byte(tmp2, Address(objb, result));
6818   xorl(tmp1, tmp2);
6819   testl(tmp1, tmp1);
6820   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6821   decq(length);
6822   jcc(Assembler::zero, SAME_TILL_END);
6823   incq(result);
6824   load_unsigned_byte(tmp1, Address(obja, result));
6825   load_unsigned_byte(tmp2, Address(objb, result));
6826   xorl(tmp1, tmp2);
6827   testl(tmp1, tmp1);
6828   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6829   decq(length);
6830   jcc(Assembler::zero, SAME_TILL_END);
6831   incq(result);
6832   load_unsigned_byte(tmp1, Address(obja, result));
6833   load_unsigned_byte(tmp2, Address(objb, result));
6834   xorl(tmp1, tmp2);
6835   testl(tmp1, tmp1);
6836   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6837   jmp(SAME_TILL_END);
6838 
6839   if (UseAVX >= 2) {
6840     bind(VECTOR32_NOT_EQUAL);
6841     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
6842     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
6843     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
6844     vpmovmskb(tmp1, rymm0);
6845     bsfq(tmp1, tmp1);
6846     addq(result, tmp1);
6847     shrq(result);
6848     jmp(DONE);
6849   }
6850 
6851   bind(VECTOR16_NOT_EQUAL);
6852   if (UseAVX >= 2) {
6853     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
6854     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
6855     pxor(rymm0, rymm2);
6856   } else {
6857     pcmpeqb(rymm2, rymm2);
6858     pxor(rymm0, rymm1);
6859     pcmpeqb(rymm0, rymm1);
6860     pxor(rymm0, rymm2);
6861   }
6862   pmovmskb(tmp1, rymm0);
6863   bsfq(tmp1, tmp1);
6864   addq(result, tmp1);
6865   shrq(result);
6866   jmpb(DONE);
6867 
6868   bind(VECTOR8_NOT_EQUAL);
6869   bind(VECTOR4_NOT_EQUAL);
6870   bsfq(tmp1, tmp1);
6871   shrq(tmp1, 3);
6872   addq(result, tmp1);
6873   bind(BYTES_NOT_EQUAL);
6874   shrq(result);
6875   jmpb(DONE);
6876 
6877   bind(SAME_TILL_END);
6878   mov64(result, -1);
6879 
6880   bind(DONE);
6881 }
6882 
6883 //Helper functions for square_to_len()
6884 
6885 /**
6886  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
6887  * Preserves x and z and modifies rest of the registers.
6888  */
6889 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6890   // Perform square and right shift by 1
6891   // Handle odd xlen case first, then for even xlen do the following
6892   // jlong carry = 0;
6893   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
6894   //     huge_128 product = x[j:j+1] * x[j:j+1];
6895   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
6896   //     z[i+2:i+3] = (jlong)(product >>> 1);
6897   //     carry = (jlong)product;
6898   // }
6899 
6900   xorq(tmp5, tmp5);     // carry
6901   xorq(rdxReg, rdxReg);
6902   xorl(tmp1, tmp1);     // index for x
6903   xorl(tmp4, tmp4);     // index for z
6904 
6905   Label L_first_loop, L_first_loop_exit;
6906 
6907   testl(xlen, 1);
6908   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
6909 
6910   // Square and right shift by 1 the odd element using 32 bit multiply
6911   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
6912   imulq(raxReg, raxReg);
6913   shrq(raxReg, 1);
6914   adcq(tmp5, 0);
6915   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
6916   incrementl(tmp1);
6917   addl(tmp4, 2);
6918 
6919   // Square and  right shift by 1 the rest using 64 bit multiply
6920   bind(L_first_loop);
6921   cmpptr(tmp1, xlen);
6922   jccb(Assembler::equal, L_first_loop_exit);
6923 
6924   // Square
6925   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
6926   rorq(raxReg, 32);    // convert big-endian to little-endian
6927   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
6928 
6929   // Right shift by 1 and save carry
6930   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
6931   rcrq(rdxReg, 1);
6932   rcrq(raxReg, 1);
6933   adcq(tmp5, 0);
6934 
6935   // Store result in z
6936   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
6937   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
6938 
6939   // Update indices for x and z
6940   addl(tmp1, 2);
6941   addl(tmp4, 4);
6942   jmp(L_first_loop);
6943 
6944   bind(L_first_loop_exit);
6945 }
6946 
6947 
6948 /**
6949  * Perform the following multiply add operation using BMI2 instructions
6950  * carry:sum = sum + op1*op2 + carry
6951  * op2 should be in rdx
6952  * op2 is preserved, all other registers are modified
6953  */
6954 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
6955   // assert op2 is rdx
6956   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
6957   addq(sum, carry);
6958   adcq(tmp2, 0);
6959   addq(sum, op1);
6960   adcq(tmp2, 0);
6961   movq(carry, tmp2);
6962 }
6963 
6964 /**
6965  * Perform the following multiply add operation:
6966  * carry:sum = sum + op1*op2 + carry
6967  * Preserves op1, op2 and modifies rest of registers
6968  */
6969 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
6970   // rdx:rax = op1 * op2
6971   movq(raxReg, op2);
6972   mulq(op1);
6973 
6974   //  rdx:rax = sum + carry + rdx:rax
6975   addq(sum, carry);
6976   adcq(rdxReg, 0);
6977   addq(sum, raxReg);
6978   adcq(rdxReg, 0);
6979 
6980   // carry:sum = rdx:sum
6981   movq(carry, rdxReg);
6982 }
6983 
6984 /**
6985  * Add 64 bit long carry into z[] with carry propagation.
6986  * Preserves z and carry register values and modifies rest of registers.
6987  *
6988  */
6989 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
6990   Label L_fourth_loop, L_fourth_loop_exit;
6991 
6992   movl(tmp1, 1);
6993   subl(zlen, 2);
6994   addq(Address(z, zlen, Address::times_4, 0), carry);
6995 
6996   bind(L_fourth_loop);
6997   jccb(Assembler::carryClear, L_fourth_loop_exit);
6998   subl(zlen, 2);
6999   jccb(Assembler::negative, L_fourth_loop_exit);
7000   addq(Address(z, zlen, Address::times_4, 0), tmp1);
7001   jmp(L_fourth_loop);
7002   bind(L_fourth_loop_exit);
7003 }
7004 
7005 /**
7006  * Shift z[] left by 1 bit.
7007  * Preserves x, len, z and zlen registers and modifies rest of the registers.
7008  *
7009  */
7010 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
7011 
7012   Label L_fifth_loop, L_fifth_loop_exit;
7013 
7014   // Fifth loop
7015   // Perform primitiveLeftShift(z, zlen, 1)
7016 
7017   const Register prev_carry = tmp1;
7018   const Register new_carry = tmp4;
7019   const Register value = tmp2;
7020   const Register zidx = tmp3;
7021 
7022   // int zidx, carry;
7023   // long value;
7024   // carry = 0;
7025   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
7026   //    (carry:value)  = (z[i] << 1) | carry ;
7027   //    z[i] = value;
7028   // }
7029 
7030   movl(zidx, zlen);
7031   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
7032 
7033   bind(L_fifth_loop);
7034   decl(zidx);  // Use decl to preserve carry flag
7035   decl(zidx);
7036   jccb(Assembler::negative, L_fifth_loop_exit);
7037 
7038   if (UseBMI2Instructions) {
7039      movq(value, Address(z, zidx, Address::times_4, 0));
7040      rclq(value, 1);
7041      rorxq(value, value, 32);
7042      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
7043   }
7044   else {
7045     // clear new_carry
7046     xorl(new_carry, new_carry);
7047 
7048     // Shift z[i] by 1, or in previous carry and save new carry
7049     movq(value, Address(z, zidx, Address::times_4, 0));
7050     shlq(value, 1);
7051     adcl(new_carry, 0);
7052 
7053     orq(value, prev_carry);
7054     rorq(value, 0x20);
7055     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
7056 
7057     // Set previous carry = new carry
7058     movl(prev_carry, new_carry);
7059   }
7060   jmp(L_fifth_loop);
7061 
7062   bind(L_fifth_loop_exit);
7063 }
7064 
7065 
7066 /**
7067  * Code for BigInteger::squareToLen() intrinsic
7068  *
7069  * rdi: x
7070  * rsi: len
7071  * r8:  z
7072  * rcx: zlen
7073  * r12: tmp1
7074  * r13: tmp2
7075  * r14: tmp3
7076  * r15: tmp4
7077  * rbx: tmp5
7078  *
7079  */
7080 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7081 
7082   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
7083   push(tmp1);
7084   push(tmp2);
7085   push(tmp3);
7086   push(tmp4);
7087   push(tmp5);
7088 
7089   // First loop
7090   // Store the squares, right shifted one bit (i.e., divided by 2).
7091   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
7092 
7093   // Add in off-diagonal sums.
7094   //
7095   // Second, third (nested) and fourth loops.
7096   // zlen +=2;
7097   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
7098   //    carry = 0;
7099   //    long op2 = x[xidx:xidx+1];
7100   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
7101   //       k -= 2;
7102   //       long op1 = x[j:j+1];
7103   //       long sum = z[k:k+1];
7104   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
7105   //       z[k:k+1] = sum;
7106   //    }
7107   //    add_one_64(z, k, carry, tmp_regs);
7108   // }
7109 
7110   const Register carry = tmp5;
7111   const Register sum = tmp3;
7112   const Register op1 = tmp4;
7113   Register op2 = tmp2;
7114 
7115   push(zlen);
7116   push(len);
7117   addl(zlen,2);
7118   bind(L_second_loop);
7119   xorq(carry, carry);
7120   subl(zlen, 4);
7121   subl(len, 2);
7122   push(zlen);
7123   push(len);
7124   cmpl(len, 0);
7125   jccb(Assembler::lessEqual, L_second_loop_exit);
7126 
7127   // Multiply an array by one 64 bit long.
7128   if (UseBMI2Instructions) {
7129     op2 = rdxReg;
7130     movq(op2, Address(x, len, Address::times_4,  0));
7131     rorxq(op2, op2, 32);
7132   }
7133   else {
7134     movq(op2, Address(x, len, Address::times_4,  0));
7135     rorq(op2, 32);
7136   }
7137 
7138   bind(L_third_loop);
7139   decrementl(len);
7140   jccb(Assembler::negative, L_third_loop_exit);
7141   decrementl(len);
7142   jccb(Assembler::negative, L_last_x);
7143 
7144   movq(op1, Address(x, len, Address::times_4,  0));
7145   rorq(op1, 32);
7146 
7147   bind(L_multiply);
7148   subl(zlen, 2);
7149   movq(sum, Address(z, zlen, Address::times_4,  0));
7150 
7151   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
7152   if (UseBMI2Instructions) {
7153     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
7154   }
7155   else {
7156     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7157   }
7158 
7159   movq(Address(z, zlen, Address::times_4, 0), sum);
7160 
7161   jmp(L_third_loop);
7162   bind(L_third_loop_exit);
7163 
7164   // Fourth loop
7165   // Add 64 bit long carry into z with carry propagation.
7166   // Uses offsetted zlen.
7167   add_one_64(z, zlen, carry, tmp1);
7168 
7169   pop(len);
7170   pop(zlen);
7171   jmp(L_second_loop);
7172 
7173   // Next infrequent code is moved outside loops.
7174   bind(L_last_x);
7175   movl(op1, Address(x, 0));
7176   jmp(L_multiply);
7177 
7178   bind(L_second_loop_exit);
7179   pop(len);
7180   pop(zlen);
7181   pop(len);
7182   pop(zlen);
7183 
7184   // Fifth loop
7185   // Shift z left 1 bit.
7186   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
7187 
7188   // z[zlen-1] |= x[len-1] & 1;
7189   movl(tmp3, Address(x, len, Address::times_4, -4));
7190   andl(tmp3, 1);
7191   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
7192 
7193   pop(tmp5);
7194   pop(tmp4);
7195   pop(tmp3);
7196   pop(tmp2);
7197   pop(tmp1);
7198 }
7199 
7200 /**
7201  * Helper function for mul_add()
7202  * Multiply the in[] by int k and add to out[] starting at offset offs using
7203  * 128 bit by 32 bit multiply and return the carry in tmp5.
7204  * Only quad int aligned length of in[] is operated on in this function.
7205  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
7206  * This function preserves out, in and k registers.
7207  * len and offset point to the appropriate index in "in" & "out" correspondingly
7208  * tmp5 has the carry.
7209  * other registers are temporary and are modified.
7210  *
7211  */
7212 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
7213   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
7214   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7215 
7216   Label L_first_loop, L_first_loop_exit;
7217 
7218   movl(tmp1, len);
7219   shrl(tmp1, 2);
7220 
7221   bind(L_first_loop);
7222   subl(tmp1, 1);
7223   jccb(Assembler::negative, L_first_loop_exit);
7224 
7225   subl(len, 4);
7226   subl(offset, 4);
7227 
7228   Register op2 = tmp2;
7229   const Register sum = tmp3;
7230   const Register op1 = tmp4;
7231   const Register carry = tmp5;
7232 
7233   if (UseBMI2Instructions) {
7234     op2 = rdxReg;
7235   }
7236 
7237   movq(op1, Address(in, len, Address::times_4,  8));
7238   rorq(op1, 32);
7239   movq(sum, Address(out, offset, Address::times_4,  8));
7240   rorq(sum, 32);
7241   if (UseBMI2Instructions) {
7242     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7243   }
7244   else {
7245     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7246   }
7247   // Store back in big endian from little endian
7248   rorq(sum, 0x20);
7249   movq(Address(out, offset, Address::times_4,  8), sum);
7250 
7251   movq(op1, Address(in, len, Address::times_4,  0));
7252   rorq(op1, 32);
7253   movq(sum, Address(out, offset, Address::times_4,  0));
7254   rorq(sum, 32);
7255   if (UseBMI2Instructions) {
7256     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7257   }
7258   else {
7259     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7260   }
7261   // Store back in big endian from little endian
7262   rorq(sum, 0x20);
7263   movq(Address(out, offset, Address::times_4,  0), sum);
7264 
7265   jmp(L_first_loop);
7266   bind(L_first_loop_exit);
7267 }
7268 
7269 /**
7270  * Code for BigInteger::mulAdd() intrinsic
7271  *
7272  * rdi: out
7273  * rsi: in
7274  * r11: offs (out.length - offset)
7275  * rcx: len
7276  * r8:  k
7277  * r12: tmp1
7278  * r13: tmp2
7279  * r14: tmp3
7280  * r15: tmp4
7281  * rbx: tmp5
7282  * Multiply the in[] by word k and add to out[], return the carry in rax
7283  */
7284 void MacroAssembler::mul_add(Register out, Register in, Register offs,
7285    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
7286    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7287 
7288   Label L_carry, L_last_in, L_done;
7289 
7290 // carry = 0;
7291 // for (int j=len-1; j >= 0; j--) {
7292 //    long product = (in[j] & LONG_MASK) * kLong +
7293 //                   (out[offs] & LONG_MASK) + carry;
7294 //    out[offs--] = (int)product;
7295 //    carry = product >>> 32;
7296 // }
7297 //
7298   push(tmp1);
7299   push(tmp2);
7300   push(tmp3);
7301   push(tmp4);
7302   push(tmp5);
7303 
7304   Register op2 = tmp2;
7305   const Register sum = tmp3;
7306   const Register op1 = tmp4;
7307   const Register carry =  tmp5;
7308 
7309   if (UseBMI2Instructions) {
7310     op2 = rdxReg;
7311     movl(op2, k);
7312   }
7313   else {
7314     movl(op2, k);
7315   }
7316 
7317   xorq(carry, carry);
7318 
7319   //First loop
7320 
7321   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
7322   //The carry is in tmp5
7323   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
7324 
7325   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
7326   decrementl(len);
7327   jccb(Assembler::negative, L_carry);
7328   decrementl(len);
7329   jccb(Assembler::negative, L_last_in);
7330 
7331   movq(op1, Address(in, len, Address::times_4,  0));
7332   rorq(op1, 32);
7333 
7334   subl(offs, 2);
7335   movq(sum, Address(out, offs, Address::times_4,  0));
7336   rorq(sum, 32);
7337 
7338   if (UseBMI2Instructions) {
7339     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7340   }
7341   else {
7342     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7343   }
7344 
7345   // Store back in big endian from little endian
7346   rorq(sum, 0x20);
7347   movq(Address(out, offs, Address::times_4,  0), sum);
7348 
7349   testl(len, len);
7350   jccb(Assembler::zero, L_carry);
7351 
7352   //Multiply the last in[] entry, if any
7353   bind(L_last_in);
7354   movl(op1, Address(in, 0));
7355   movl(sum, Address(out, offs, Address::times_4,  -4));
7356 
7357   movl(raxReg, k);
7358   mull(op1); //tmp4 * eax -> edx:eax
7359   addl(sum, carry);
7360   adcl(rdxReg, 0);
7361   addl(sum, raxReg);
7362   adcl(rdxReg, 0);
7363   movl(carry, rdxReg);
7364 
7365   movl(Address(out, offs, Address::times_4,  -4), sum);
7366 
7367   bind(L_carry);
7368   //return tmp5/carry as carry in rax
7369   movl(rax, carry);
7370 
7371   bind(L_done);
7372   pop(tmp5);
7373   pop(tmp4);
7374   pop(tmp3);
7375   pop(tmp2);
7376   pop(tmp1);
7377 }
7378 
7379 /**
7380  * Emits code to update CRC-32 with a byte value according to constants in table
7381  *
7382  * @param [in,out]crc   Register containing the crc.
7383  * @param [in]val       Register containing the byte to fold into the CRC.
7384  * @param [in]table     Register containing the table of crc constants.
7385  *
7386  * uint32_t crc;
7387  * val = crc_table[(val ^ crc) & 0xFF];
7388  * crc = val ^ (crc >> 8);
7389  *
7390  */
7391 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
7392   xorl(val, crc);
7393   andl(val, 0xFF);
7394   shrl(crc, 8); // unsigned shift
7395   xorl(crc, Address(table, val, Address::times_4, 0));
7396 }
7397 
7398 /**
7399  * Fold 128-bit data chunk
7400  */
7401 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
7402   if (UseAVX > 0) {
7403     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
7404     vpclmulldq(xcrc, xK, xcrc); // [63:0]
7405     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
7406     pxor(xcrc, xtmp);
7407   } else {
7408     movdqa(xtmp, xcrc);
7409     pclmulhdq(xtmp, xK);   // [123:64]
7410     pclmulldq(xcrc, xK);   // [63:0]
7411     pxor(xcrc, xtmp);
7412     movdqu(xtmp, Address(buf, offset));
7413     pxor(xcrc, xtmp);
7414   }
7415 }
7416 
7417 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
7418   if (UseAVX > 0) {
7419     vpclmulhdq(xtmp, xK, xcrc);
7420     vpclmulldq(xcrc, xK, xcrc);
7421     pxor(xcrc, xbuf);
7422     pxor(xcrc, xtmp);
7423   } else {
7424     movdqa(xtmp, xcrc);
7425     pclmulhdq(xtmp, xK);
7426     pclmulldq(xcrc, xK);
7427     pxor(xcrc, xbuf);
7428     pxor(xcrc, xtmp);
7429   }
7430 }
7431 
7432 /**
7433  * 8-bit folds to compute 32-bit CRC
7434  *
7435  * uint64_t xcrc;
7436  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
7437  */
7438 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
7439   movdl(tmp, xcrc);
7440   andl(tmp, 0xFF);
7441   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
7442   psrldq(xcrc, 1); // unsigned shift one byte
7443   pxor(xcrc, xtmp);
7444 }
7445 
7446 /**
7447  * uint32_t crc;
7448  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
7449  */
7450 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
7451   movl(tmp, crc);
7452   andl(tmp, 0xFF);
7453   shrl(crc, 8);
7454   xorl(crc, Address(table, tmp, Address::times_4, 0));
7455 }
7456 
7457 /**
7458  * @param crc   register containing existing CRC (32-bit)
7459  * @param buf   register pointing to input byte buffer (byte*)
7460  * @param len   register containing number of bytes
7461  * @param table register that will contain address of CRC table
7462  * @param tmp   scratch register
7463  */
7464 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
7465   assert_different_registers(crc, buf, len, table, tmp, rax);
7466 
7467   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
7468   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
7469 
7470   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
7471   // context for the registers used, where all instructions below are using 128-bit mode
7472   // On EVEX without VL and BW, these instructions will all be AVX.
7473   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
7474   notl(crc); // ~crc
7475   cmpl(len, 16);
7476   jcc(Assembler::less, L_tail);
7477 
7478   // Align buffer to 16 bytes
7479   movl(tmp, buf);
7480   andl(tmp, 0xF);
7481   jccb(Assembler::zero, L_aligned);
7482   subl(tmp,  16);
7483   addl(len, tmp);
7484 
7485   align(4);
7486   BIND(L_align_loop);
7487   movsbl(rax, Address(buf, 0)); // load byte with sign extension
7488   update_byte_crc32(crc, rax, table);
7489   increment(buf);
7490   incrementl(tmp);
7491   jccb(Assembler::less, L_align_loop);
7492 
7493   BIND(L_aligned);
7494   movl(tmp, len); // save
7495   shrl(len, 4);
7496   jcc(Assembler::zero, L_tail_restore);
7497 
7498   // Fold crc into first bytes of vector
7499   movdqa(xmm1, Address(buf, 0));
7500   movdl(rax, xmm1);
7501   xorl(crc, rax);
7502   if (VM_Version::supports_sse4_1()) {
7503     pinsrd(xmm1, crc, 0);
7504   } else {
7505     pinsrw(xmm1, crc, 0);
7506     shrl(crc, 16);
7507     pinsrw(xmm1, crc, 1);
7508   }
7509   addptr(buf, 16);
7510   subl(len, 4); // len > 0
7511   jcc(Assembler::less, L_fold_tail);
7512 
7513   movdqa(xmm2, Address(buf,  0));
7514   movdqa(xmm3, Address(buf, 16));
7515   movdqa(xmm4, Address(buf, 32));
7516   addptr(buf, 48);
7517   subl(len, 3);
7518   jcc(Assembler::lessEqual, L_fold_512b);
7519 
7520   // Fold total 512 bits of polynomial on each iteration,
7521   // 128 bits per each of 4 parallel streams.
7522   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
7523 
7524   align32();
7525   BIND(L_fold_512b_loop);
7526   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7527   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
7528   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
7529   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
7530   addptr(buf, 64);
7531   subl(len, 4);
7532   jcc(Assembler::greater, L_fold_512b_loop);
7533 
7534   // Fold 512 bits to 128 bits.
7535   BIND(L_fold_512b);
7536   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
7537   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
7538   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
7539   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
7540 
7541   // Fold the rest of 128 bits data chunks
7542   BIND(L_fold_tail);
7543   addl(len, 3);
7544   jccb(Assembler::lessEqual, L_fold_128b);
7545   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
7546 
7547   BIND(L_fold_tail_loop);
7548   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7549   addptr(buf, 16);
7550   decrementl(len);
7551   jccb(Assembler::greater, L_fold_tail_loop);
7552 
7553   // Fold 128 bits in xmm1 down into 32 bits in crc register.
7554   BIND(L_fold_128b);
7555   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
7556   if (UseAVX > 0) {
7557     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
7558     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
7559     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
7560   } else {
7561     movdqa(xmm2, xmm0);
7562     pclmulqdq(xmm2, xmm1, 0x1);
7563     movdqa(xmm3, xmm0);
7564     pand(xmm3, xmm2);
7565     pclmulqdq(xmm0, xmm3, 0x1);
7566   }
7567   psrldq(xmm1, 8);
7568   psrldq(xmm2, 4);
7569   pxor(xmm0, xmm1);
7570   pxor(xmm0, xmm2);
7571 
7572   // 8 8-bit folds to compute 32-bit CRC.
7573   for (int j = 0; j < 4; j++) {
7574     fold_8bit_crc32(xmm0, table, xmm1, rax);
7575   }
7576   movdl(crc, xmm0); // mov 32 bits to general register
7577   for (int j = 0; j < 4; j++) {
7578     fold_8bit_crc32(crc, table, rax);
7579   }
7580 
7581   BIND(L_tail_restore);
7582   movl(len, tmp); // restore
7583   BIND(L_tail);
7584   andl(len, 0xf);
7585   jccb(Assembler::zero, L_exit);
7586 
7587   // Fold the rest of bytes
7588   align(4);
7589   BIND(L_tail_loop);
7590   movsbl(rax, Address(buf, 0)); // load byte with sign extension
7591   update_byte_crc32(crc, rax, table);
7592   increment(buf);
7593   decrementl(len);
7594   jccb(Assembler::greater, L_tail_loop);
7595 
7596   BIND(L_exit);
7597   notl(crc); // ~c
7598 }
7599 
7600 // Helper function for AVX 512 CRC32
7601 // Fold 512-bit data chunks
7602 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
7603                                              Register pos, int offset) {
7604   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
7605   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
7606   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
7607   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
7608   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
7609 }
7610 
7611 // Helper function for AVX 512 CRC32
7612 // Compute CRC32 for < 256B buffers
7613 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
7614                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
7615                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
7616 
7617   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
7618   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
7619   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
7620 
7621   // check if there is enough buffer to be able to fold 16B at a time
7622   cmpl(len, 32);
7623   jcc(Assembler::less, L_less_than_32);
7624 
7625   // if there is, load the constants
7626   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
7627   movdl(xmm0, crc);                        // get the initial crc value
7628   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7629   pxor(xmm7, xmm0);
7630 
7631   // update the buffer pointer
7632   addl(pos, 16);
7633   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
7634   subl(len, 32);
7635   jmp(L_16B_reduction_loop);
7636 
7637   bind(L_less_than_32);
7638   //mov initial crc to the return value. this is necessary for zero - length buffers.
7639   movl(rax, crc);
7640   testl(len, len);
7641   jcc(Assembler::equal, L_cleanup);
7642 
7643   movdl(xmm0, crc);                        //get the initial crc value
7644 
7645   cmpl(len, 16);
7646   jcc(Assembler::equal, L_exact_16_left);
7647   jcc(Assembler::less, L_less_than_16_left);
7648 
7649   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7650   pxor(xmm7, xmm0);                       //xor the initial crc value
7651   addl(pos, 16);
7652   subl(len, 16);
7653   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
7654   jmp(L_get_last_two_xmms);
7655 
7656   bind(L_less_than_16_left);
7657   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
7658   pxor(xmm1, xmm1);
7659   movptr(tmp1, rsp);
7660   movdqu(Address(tmp1, 0 * 16), xmm1);
7661 
7662   cmpl(len, 4);
7663   jcc(Assembler::less, L_only_less_than_4);
7664 
7665   //backup the counter value
7666   movl(tmp2, len);
7667   cmpl(len, 8);
7668   jcc(Assembler::less, L_less_than_8_left);
7669 
7670   //load 8 Bytes
7671   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
7672   movq(Address(tmp1, 0 * 16), rax);
7673   addptr(tmp1, 8);
7674   subl(len, 8);
7675   addl(pos, 8);
7676 
7677   bind(L_less_than_8_left);
7678   cmpl(len, 4);
7679   jcc(Assembler::less, L_less_than_4_left);
7680 
7681   //load 4 Bytes
7682   movl(rax, Address(buf, pos, Address::times_1, 0));
7683   movl(Address(tmp1, 0 * 16), rax);
7684   addptr(tmp1, 4);
7685   subl(len, 4);
7686   addl(pos, 4);
7687 
7688   bind(L_less_than_4_left);
7689   cmpl(len, 2);
7690   jcc(Assembler::less, L_less_than_2_left);
7691 
7692   // load 2 Bytes
7693   movw(rax, Address(buf, pos, Address::times_1, 0));
7694   movl(Address(tmp1, 0 * 16), rax);
7695   addptr(tmp1, 2);
7696   subl(len, 2);
7697   addl(pos, 2);
7698 
7699   bind(L_less_than_2_left);
7700   cmpl(len, 1);
7701   jcc(Assembler::less, L_zero_left);
7702 
7703   // load 1 Byte
7704   movb(rax, Address(buf, pos, Address::times_1, 0));
7705   movb(Address(tmp1, 0 * 16), rax);
7706 
7707   bind(L_zero_left);
7708   movdqu(xmm7, Address(rsp, 0));
7709   pxor(xmm7, xmm0);                       //xor the initial crc value
7710 
7711   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7712   movdqu(xmm0, Address(rax, tmp2));
7713   pshufb(xmm7, xmm0);
7714   jmp(L_128_done);
7715 
7716   bind(L_exact_16_left);
7717   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
7718   pxor(xmm7, xmm0);                       //xor the initial crc value
7719   jmp(L_128_done);
7720 
7721   bind(L_only_less_than_4);
7722   cmpl(len, 3);
7723   jcc(Assembler::less, L_only_less_than_3);
7724 
7725   // load 3 Bytes
7726   movb(rax, Address(buf, pos, Address::times_1, 0));
7727   movb(Address(tmp1, 0), rax);
7728 
7729   movb(rax, Address(buf, pos, Address::times_1, 1));
7730   movb(Address(tmp1, 1), rax);
7731 
7732   movb(rax, Address(buf, pos, Address::times_1, 2));
7733   movb(Address(tmp1, 2), rax);
7734 
7735   movdqu(xmm7, Address(rsp, 0));
7736   pxor(xmm7, xmm0);                     //xor the initial crc value
7737 
7738   pslldq(xmm7, 0x5);
7739   jmp(L_barrett);
7740   bind(L_only_less_than_3);
7741   cmpl(len, 2);
7742   jcc(Assembler::less, L_only_less_than_2);
7743 
7744   // load 2 Bytes
7745   movb(rax, Address(buf, pos, Address::times_1, 0));
7746   movb(Address(tmp1, 0), rax);
7747 
7748   movb(rax, Address(buf, pos, Address::times_1, 1));
7749   movb(Address(tmp1, 1), rax);
7750 
7751   movdqu(xmm7, Address(rsp, 0));
7752   pxor(xmm7, xmm0);                     //xor the initial crc value
7753 
7754   pslldq(xmm7, 0x6);
7755   jmp(L_barrett);
7756 
7757   bind(L_only_less_than_2);
7758   //load 1 Byte
7759   movb(rax, Address(buf, pos, Address::times_1, 0));
7760   movb(Address(tmp1, 0), rax);
7761 
7762   movdqu(xmm7, Address(rsp, 0));
7763   pxor(xmm7, xmm0);                     //xor the initial crc value
7764 
7765   pslldq(xmm7, 0x7);
7766 }
7767 
7768 /**
7769 * Compute CRC32 using AVX512 instructions
7770 * param crc   register containing existing CRC (32-bit)
7771 * param buf   register pointing to input byte buffer (byte*)
7772 * param len   register containing number of bytes
7773 * param table address of crc or crc32c table
7774 * param tmp1  scratch register
7775 * param tmp2  scratch register
7776 * return rax  result register
7777 *
7778 * This routine is identical for crc32c with the exception of the precomputed constant
7779 * table which will be passed as the table argument.  The calculation steps are
7780 * the same for both variants.
7781 */
7782 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
7783   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
7784 
7785   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
7786   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
7787   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
7788   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
7789   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
7790 
7791   const Register pos = r12;
7792   push(r12);
7793   subptr(rsp, 16 * 2 + 8);
7794 
7795   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
7796   // context for the registers used, where all instructions below are using 128-bit mode
7797   // On EVEX without VL and BW, these instructions will all be AVX.
7798   movl(pos, 0);
7799 
7800   // check if smaller than 256B
7801   cmpl(len, 256);
7802   jcc(Assembler::less, L_less_than_256);
7803 
7804   // load the initial crc value
7805   movdl(xmm10, crc);
7806 
7807   // receive the initial 64B data, xor the initial crc value
7808   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
7809   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
7810   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
7811   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
7812 
7813   subl(len, 256);
7814   cmpl(len, 256);
7815   jcc(Assembler::less, L_fold_128_B_loop);
7816 
7817   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
7818   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
7819   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
7820   subl(len, 256);
7821 
7822   bind(L_fold_256_B_loop);
7823   addl(pos, 256);
7824   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
7825   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
7826   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
7827   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
7828 
7829   subl(len, 256);
7830   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
7831 
7832   // Fold 256 into 128
7833   addl(pos, 256);
7834   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
7835   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
7836   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
7837 
7838   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
7839   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
7840   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
7841 
7842   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
7843   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
7844 
7845   addl(len, 128);
7846   jmp(L_fold_128_B_register);
7847 
7848   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
7849   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
7850 
7851   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
7852   bind(L_fold_128_B_loop);
7853   addl(pos, 128);
7854   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
7855   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
7856 
7857   subl(len, 128);
7858   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
7859 
7860   addl(pos, 128);
7861 
7862   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
7863   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
7864   bind(L_fold_128_B_register);
7865   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
7866   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
7867   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
7868   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
7869   // save last that has no multiplicand
7870   vextracti64x2(xmm7, xmm4, 3);
7871 
7872   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
7873   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
7874   // Needed later in reduction loop
7875   movdqu(xmm10, Address(table, 1 * 16));
7876   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
7877   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
7878 
7879   // Swap 1,0,3,2 - 01 00 11 10
7880   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
7881   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
7882   vextracti128(xmm5, xmm8, 1);
7883   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
7884 
7885   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
7886   // instead of a cmp instruction, we use the negative flag with the jl instruction
7887   addl(len, 128 - 16);
7888   jcc(Assembler::less, L_final_reduction_for_128);
7889 
7890   bind(L_16B_reduction_loop);
7891   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
7892   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7893   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7894   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
7895   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7896   addl(pos, 16);
7897   subl(len, 16);
7898   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
7899 
7900   bind(L_final_reduction_for_128);
7901   addl(len, 16);
7902   jcc(Assembler::equal, L_128_done);
7903 
7904   bind(L_get_last_two_xmms);
7905   movdqu(xmm2, xmm7);
7906   addl(pos, len);
7907   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
7908   subl(pos, len);
7909 
7910   // get rid of the extra data that was loaded before
7911   // load the shift constant
7912   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7913   movdqu(xmm0, Address(rax, len));
7914   addl(rax, len);
7915 
7916   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7917   //Change mask to 512
7918   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
7919   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
7920 
7921   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
7922   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
7923   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7924   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7925   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
7926 
7927   bind(L_128_done);
7928   // compute crc of a 128-bit value
7929   movdqu(xmm10, Address(table, 3 * 16));
7930   movdqu(xmm0, xmm7);
7931 
7932   // 64b fold
7933   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
7934   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
7935   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7936 
7937   // 32b fold
7938   movdqu(xmm0, xmm7);
7939   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
7940   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7941   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7942   jmp(L_barrett);
7943 
7944   bind(L_less_than_256);
7945   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
7946 
7947   //barrett reduction
7948   bind(L_barrett);
7949   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
7950   movdqu(xmm1, xmm7);
7951   movdqu(xmm2, xmm7);
7952   movdqu(xmm10, Address(table, 4 * 16));
7953 
7954   pclmulqdq(xmm7, xmm10, 0x0);
7955   pxor(xmm7, xmm2);
7956   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
7957   movdqu(xmm2, xmm7);
7958   pclmulqdq(xmm7, xmm10, 0x10);
7959   pxor(xmm7, xmm2);
7960   pxor(xmm7, xmm1);
7961   pextrd(crc, xmm7, 2);
7962 
7963   bind(L_cleanup);
7964   addptr(rsp, 16 * 2 + 8);
7965   pop(r12);
7966 }
7967 
7968 // S. Gueron / Information Processing Letters 112 (2012) 184
7969 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
7970 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
7971 // Output: the 64-bit carry-less product of B * CONST
7972 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
7973                                      Register tmp1, Register tmp2, Register tmp3) {
7974   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
7975   if (n > 0) {
7976     addq(tmp3, n * 256 * 8);
7977   }
7978   //    Q1 = TABLEExt[n][B & 0xFF];
7979   movl(tmp1, in);
7980   andl(tmp1, 0x000000FF);
7981   shll(tmp1, 3);
7982   addq(tmp1, tmp3);
7983   movq(tmp1, Address(tmp1, 0));
7984 
7985   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
7986   movl(tmp2, in);
7987   shrl(tmp2, 8);
7988   andl(tmp2, 0x000000FF);
7989   shll(tmp2, 3);
7990   addq(tmp2, tmp3);
7991   movq(tmp2, Address(tmp2, 0));
7992 
7993   shlq(tmp2, 8);
7994   xorq(tmp1, tmp2);
7995 
7996   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
7997   movl(tmp2, in);
7998   shrl(tmp2, 16);
7999   andl(tmp2, 0x000000FF);
8000   shll(tmp2, 3);
8001   addq(tmp2, tmp3);
8002   movq(tmp2, Address(tmp2, 0));
8003 
8004   shlq(tmp2, 16);
8005   xorq(tmp1, tmp2);
8006 
8007   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
8008   shrl(in, 24);
8009   andl(in, 0x000000FF);
8010   shll(in, 3);
8011   addq(in, tmp3);
8012   movq(in, Address(in, 0));
8013 
8014   shlq(in, 24);
8015   xorq(in, tmp1);
8016   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
8017 }
8018 
8019 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
8020                                       Register in_out,
8021                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
8022                                       XMMRegister w_xtmp2,
8023                                       Register tmp1,
8024                                       Register n_tmp2, Register n_tmp3) {
8025   if (is_pclmulqdq_supported) {
8026     movdl(w_xtmp1, in_out); // modified blindly
8027 
8028     movl(tmp1, const_or_pre_comp_const_index);
8029     movdl(w_xtmp2, tmp1);
8030     pclmulqdq(w_xtmp1, w_xtmp2, 0);
8031 
8032     movdq(in_out, w_xtmp1);
8033   } else {
8034     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
8035   }
8036 }
8037 
8038 // Recombination Alternative 2: No bit-reflections
8039 // T1 = (CRC_A * U1) << 1
8040 // T2 = (CRC_B * U2) << 1
8041 // C1 = T1 >> 32
8042 // C2 = T2 >> 32
8043 // T1 = T1 & 0xFFFFFFFF
8044 // T2 = T2 & 0xFFFFFFFF
8045 // T1 = CRC32(0, T1)
8046 // T2 = CRC32(0, T2)
8047 // C1 = C1 ^ T1
8048 // C2 = C2 ^ T2
8049 // CRC = C1 ^ C2 ^ CRC_C
8050 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
8051                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8052                                      Register tmp1, Register tmp2,
8053                                      Register n_tmp3) {
8054   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
8055   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
8056   shlq(in_out, 1);
8057   movl(tmp1, in_out);
8058   shrq(in_out, 32);
8059   xorl(tmp2, tmp2);
8060   crc32(tmp2, tmp1, 4);
8061   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
8062   shlq(in1, 1);
8063   movl(tmp1, in1);
8064   shrq(in1, 32);
8065   xorl(tmp2, tmp2);
8066   crc32(tmp2, tmp1, 4);
8067   xorl(in1, tmp2);
8068   xorl(in_out, in1);
8069   xorl(in_out, in2);
8070 }
8071 
8072 // Set N to predefined value
8073 // Subtract from a length of a buffer
8074 // execute in a loop:
8075 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
8076 // for i = 1 to N do
8077 //  CRC_A = CRC32(CRC_A, A[i])
8078 //  CRC_B = CRC32(CRC_B, B[i])
8079 //  CRC_C = CRC32(CRC_C, C[i])
8080 // end for
8081 // Recombine
8082 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
8083                                        Register in_out1, Register in_out2, Register in_out3,
8084                                        Register tmp1, Register tmp2, Register tmp3,
8085                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8086                                        Register tmp4, Register tmp5,
8087                                        Register n_tmp6) {
8088   Label L_processPartitions;
8089   Label L_processPartition;
8090   Label L_exit;
8091 
8092   bind(L_processPartitions);
8093   cmpl(in_out1, 3 * size);
8094   jcc(Assembler::less, L_exit);
8095     xorl(tmp1, tmp1);
8096     xorl(tmp2, tmp2);
8097     movq(tmp3, in_out2);
8098     addq(tmp3, size);
8099 
8100     bind(L_processPartition);
8101       crc32(in_out3, Address(in_out2, 0), 8);
8102       crc32(tmp1, Address(in_out2, size), 8);
8103       crc32(tmp2, Address(in_out2, size * 2), 8);
8104       addq(in_out2, 8);
8105       cmpq(in_out2, tmp3);
8106       jcc(Assembler::less, L_processPartition);
8107     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
8108             w_xtmp1, w_xtmp2, w_xtmp3,
8109             tmp4, tmp5,
8110             n_tmp6);
8111     addq(in_out2, 2 * size);
8112     subl(in_out1, 3 * size);
8113     jmp(L_processPartitions);
8114 
8115   bind(L_exit);
8116 }
8117 
8118 // Algorithm 2: Pipelined usage of the CRC32 instruction.
8119 // Input: A buffer I of L bytes.
8120 // Output: the CRC32C value of the buffer.
8121 // Notations:
8122 // Write L = 24N + r, with N = floor (L/24).
8123 // r = L mod 24 (0 <= r < 24).
8124 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
8125 // N quadwords, and R consists of r bytes.
8126 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
8127 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
8128 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
8129 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
8130 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
8131                                           Register tmp1, Register tmp2, Register tmp3,
8132                                           Register tmp4, Register tmp5, Register tmp6,
8133                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8134                                           bool is_pclmulqdq_supported) {
8135   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
8136   Label L_wordByWord;
8137   Label L_byteByByteProlog;
8138   Label L_byteByByte;
8139   Label L_exit;
8140 
8141   if (is_pclmulqdq_supported ) {
8142     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
8143     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
8144 
8145     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
8146     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
8147 
8148     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
8149     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
8150     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
8151   } else {
8152     const_or_pre_comp_const_index[0] = 1;
8153     const_or_pre_comp_const_index[1] = 0;
8154 
8155     const_or_pre_comp_const_index[2] = 3;
8156     const_or_pre_comp_const_index[3] = 2;
8157 
8158     const_or_pre_comp_const_index[4] = 5;
8159     const_or_pre_comp_const_index[5] = 4;
8160    }
8161   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
8162                     in2, in1, in_out,
8163                     tmp1, tmp2, tmp3,
8164                     w_xtmp1, w_xtmp2, w_xtmp3,
8165                     tmp4, tmp5,
8166                     tmp6);
8167   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
8168                     in2, in1, in_out,
8169                     tmp1, tmp2, tmp3,
8170                     w_xtmp1, w_xtmp2, w_xtmp3,
8171                     tmp4, tmp5,
8172                     tmp6);
8173   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
8174                     in2, in1, in_out,
8175                     tmp1, tmp2, tmp3,
8176                     w_xtmp1, w_xtmp2, w_xtmp3,
8177                     tmp4, tmp5,
8178                     tmp6);
8179   movl(tmp1, in2);
8180   andl(tmp1, 0x00000007);
8181   negl(tmp1);
8182   addl(tmp1, in2);
8183   addq(tmp1, in1);
8184 
8185   cmpq(in1, tmp1);
8186   jccb(Assembler::greaterEqual, L_byteByByteProlog);
8187   align(16);
8188   BIND(L_wordByWord);
8189     crc32(in_out, Address(in1, 0), 8);
8190     addq(in1, 8);
8191     cmpq(in1, tmp1);
8192     jcc(Assembler::less, L_wordByWord);
8193 
8194   BIND(L_byteByByteProlog);
8195   andl(in2, 0x00000007);
8196   movl(tmp2, 1);
8197 
8198   cmpl(tmp2, in2);
8199   jccb(Assembler::greater, L_exit);
8200   BIND(L_byteByByte);
8201     crc32(in_out, Address(in1, 0), 1);
8202     incq(in1);
8203     incl(tmp2);
8204     cmpl(tmp2, in2);
8205     jcc(Assembler::lessEqual, L_byteByByte);
8206 
8207   BIND(L_exit);
8208 }
8209 #undef BIND
8210 #undef BLOCK_COMMENT
8211 
8212 // Compress char[] array to byte[].
8213 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
8214 // Return the array length if every element in array can be encoded,
8215 // otherwise, the index of first non-latin1 (> 0xff) character.
8216 //   @IntrinsicCandidate
8217 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
8218 //     for (int i = 0; i < len; i++) {
8219 //       char c = src[srcOff];
8220 //       if (c > 0xff) {
8221 //           return i;  // return index of non-latin1 char
8222 //       }
8223 //       dst[dstOff] = (byte)c;
8224 //       srcOff++;
8225 //       dstOff++;
8226 //     }
8227 //     return len;
8228 //   }
8229 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
8230   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8231   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8232   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
8233   Label copy_chars_loop, done, reset_sp, copy_tail;
8234 
8235   // rsi: src
8236   // rdi: dst
8237   // rdx: len
8238   // rcx: tmp5
8239   // rax: result
8240 
8241   // rsi holds start addr of source char[] to be compressed
8242   // rdi holds start addr of destination byte[]
8243   // rdx holds length
8244 
8245   assert(len != result, "");
8246 
8247   // save length for return
8248   movl(result, len);
8249 
8250   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
8251     VM_Version::supports_avx512vlbw() &&
8252     VM_Version::supports_bmi2()) {
8253 
8254     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
8255 
8256     // alignment
8257     Label post_alignment;
8258 
8259     // if length of the string is less than 32, handle it the old fashioned way
8260     testl(len, -32);
8261     jcc(Assembler::zero, below_threshold);
8262 
8263     // First check whether a character is compressible ( <= 0xFF).
8264     // Create mask to test for Unicode chars inside zmm vector
8265     movl(tmp5, 0x00FF);
8266     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
8267 
8268     testl(len, -64);
8269     jccb(Assembler::zero, post_alignment);
8270 
8271     movl(tmp5, dst);
8272     andl(tmp5, (32 - 1));
8273     negl(tmp5);
8274     andl(tmp5, (32 - 1));
8275 
8276     // bail out when there is nothing to be done
8277     testl(tmp5, 0xFFFFFFFF);
8278     jccb(Assembler::zero, post_alignment);
8279 
8280     // ~(~0 << len), where len is the # of remaining elements to process
8281     movl(len, 0xFFFFFFFF);
8282     shlxl(len, len, tmp5);
8283     notl(len);
8284     kmovdl(mask2, len);
8285     movl(len, result);
8286 
8287     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
8288     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
8289     ktestd(mask1, mask2);
8290     jcc(Assembler::carryClear, copy_tail);
8291 
8292     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
8293 
8294     addptr(src, tmp5);
8295     addptr(src, tmp5);
8296     addptr(dst, tmp5);
8297     subl(len, tmp5);
8298 
8299     bind(post_alignment);
8300     // end of alignment
8301 
8302     movl(tmp5, len);
8303     andl(tmp5, (32 - 1));    // tail count (in chars)
8304     andl(len, ~(32 - 1));    // vector count (in chars)
8305     jccb(Assembler::zero, copy_loop_tail);
8306 
8307     lea(src, Address(src, len, Address::times_2));
8308     lea(dst, Address(dst, len, Address::times_1));
8309     negptr(len);
8310 
8311     bind(copy_32_loop);
8312     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
8313     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
8314     kortestdl(mask1, mask1);
8315     jccb(Assembler::carryClear, reset_for_copy_tail);
8316 
8317     // All elements in current processed chunk are valid candidates for
8318     // compression. Write a truncated byte elements to the memory.
8319     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
8320     addptr(len, 32);
8321     jccb(Assembler::notZero, copy_32_loop);
8322 
8323     bind(copy_loop_tail);
8324     // bail out when there is nothing to be done
8325     testl(tmp5, 0xFFFFFFFF);
8326     jcc(Assembler::zero, done);
8327 
8328     movl(len, tmp5);
8329 
8330     // ~(~0 << len), where len is the # of remaining elements to process
8331     movl(tmp5, 0xFFFFFFFF);
8332     shlxl(tmp5, tmp5, len);
8333     notl(tmp5);
8334 
8335     kmovdl(mask2, tmp5);
8336 
8337     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
8338     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
8339     ktestd(mask1, mask2);
8340     jcc(Assembler::carryClear, copy_tail);
8341 
8342     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
8343     jmp(done);
8344 
8345     bind(reset_for_copy_tail);
8346     lea(src, Address(src, tmp5, Address::times_2));
8347     lea(dst, Address(dst, tmp5, Address::times_1));
8348     subptr(len, tmp5);
8349     jmp(copy_chars_loop);
8350 
8351     bind(below_threshold);
8352   }
8353 
8354   if (UseSSE42Intrinsics) {
8355     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
8356 
8357     // vectored compression
8358     testl(len, 0xfffffff8);
8359     jcc(Assembler::zero, copy_tail);
8360 
8361     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
8362     movdl(tmp1Reg, tmp5);
8363     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
8364 
8365     andl(len, 0xfffffff0);
8366     jccb(Assembler::zero, copy_16);
8367 
8368     // compress 16 chars per iter
8369     pxor(tmp4Reg, tmp4Reg);
8370 
8371     lea(src, Address(src, len, Address::times_2));
8372     lea(dst, Address(dst, len, Address::times_1));
8373     negptr(len);
8374 
8375     bind(copy_32_loop);
8376     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
8377     por(tmp4Reg, tmp2Reg);
8378     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
8379     por(tmp4Reg, tmp3Reg);
8380     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
8381     jccb(Assembler::notZero, reset_for_copy_tail);
8382     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
8383     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
8384     addptr(len, 16);
8385     jccb(Assembler::notZero, copy_32_loop);
8386 
8387     // compress next vector of 8 chars (if any)
8388     bind(copy_16);
8389     // len = 0
8390     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
8391     jccb(Assembler::zero, copy_tail_sse);
8392 
8393     pxor(tmp3Reg, tmp3Reg);
8394 
8395     movdqu(tmp2Reg, Address(src, 0));
8396     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
8397     jccb(Assembler::notZero, reset_for_copy_tail);
8398     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
8399     movq(Address(dst, 0), tmp2Reg);
8400     addptr(src, 16);
8401     addptr(dst, 8);
8402     jmpb(copy_tail_sse);
8403 
8404     bind(reset_for_copy_tail);
8405     movl(tmp5, result);
8406     andl(tmp5, 0x0000000f);
8407     lea(src, Address(src, tmp5, Address::times_2));
8408     lea(dst, Address(dst, tmp5, Address::times_1));
8409     subptr(len, tmp5);
8410     jmpb(copy_chars_loop);
8411 
8412     bind(copy_tail_sse);
8413     movl(len, result);
8414     andl(len, 0x00000007);    // tail count (in chars)
8415   }
8416   // compress 1 char per iter
8417   bind(copy_tail);
8418   testl(len, len);
8419   jccb(Assembler::zero, done);
8420   lea(src, Address(src, len, Address::times_2));
8421   lea(dst, Address(dst, len, Address::times_1));
8422   negptr(len);
8423 
8424   bind(copy_chars_loop);
8425   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
8426   testl(tmp5, 0xff00);      // check if Unicode char
8427   jccb(Assembler::notZero, reset_sp);
8428   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
8429   increment(len);
8430   jccb(Assembler::notZero, copy_chars_loop);
8431 
8432   // add len then return (len will be zero if compress succeeded, otherwise negative)
8433   bind(reset_sp);
8434   addl(result, len);
8435 
8436   bind(done);
8437 }
8438 
8439 // Inflate byte[] array to char[].
8440 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
8441 //   @IntrinsicCandidate
8442 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
8443 //     for (int i = 0; i < len; i++) {
8444 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
8445 //     }
8446 //   }
8447 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
8448   XMMRegister tmp1, Register tmp2, KRegister mask) {
8449   Label copy_chars_loop, done, below_threshold, avx3_threshold;
8450   // rsi: src
8451   // rdi: dst
8452   // rdx: len
8453   // rcx: tmp2
8454 
8455   // rsi holds start addr of source byte[] to be inflated
8456   // rdi holds start addr of destination char[]
8457   // rdx holds length
8458   assert_different_registers(src, dst, len, tmp2);
8459   movl(tmp2, len);
8460   if ((UseAVX > 2) && // AVX512
8461     VM_Version::supports_avx512vlbw() &&
8462     VM_Version::supports_bmi2()) {
8463 
8464     Label copy_32_loop, copy_tail;
8465     Register tmp3_aliased = len;
8466 
8467     // if length of the string is less than 16, handle it in an old fashioned way
8468     testl(len, -16);
8469     jcc(Assembler::zero, below_threshold);
8470 
8471     testl(len, -1 * AVX3Threshold);
8472     jcc(Assembler::zero, avx3_threshold);
8473 
8474     // In order to use only one arithmetic operation for the main loop we use
8475     // this pre-calculation
8476     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
8477     andl(len, -32);     // vector count
8478     jccb(Assembler::zero, copy_tail);
8479 
8480     lea(src, Address(src, len, Address::times_1));
8481     lea(dst, Address(dst, len, Address::times_2));
8482     negptr(len);
8483 
8484 
8485     // inflate 32 chars per iter
8486     bind(copy_32_loop);
8487     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
8488     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
8489     addptr(len, 32);
8490     jcc(Assembler::notZero, copy_32_loop);
8491 
8492     bind(copy_tail);
8493     // bail out when there is nothing to be done
8494     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
8495     jcc(Assembler::zero, done);
8496 
8497     // ~(~0 << length), where length is the # of remaining elements to process
8498     movl(tmp3_aliased, -1);
8499     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
8500     notl(tmp3_aliased);
8501     kmovdl(mask, tmp3_aliased);
8502     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
8503     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
8504 
8505     jmp(done);
8506     bind(avx3_threshold);
8507   }
8508   if (UseSSE42Intrinsics) {
8509     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
8510 
8511     if (UseAVX > 1) {
8512       andl(tmp2, (16 - 1));
8513       andl(len, -16);
8514       jccb(Assembler::zero, copy_new_tail);
8515     } else {
8516       andl(tmp2, 0x00000007);   // tail count (in chars)
8517       andl(len, 0xfffffff8);    // vector count (in chars)
8518       jccb(Assembler::zero, copy_tail);
8519     }
8520 
8521     // vectored inflation
8522     lea(src, Address(src, len, Address::times_1));
8523     lea(dst, Address(dst, len, Address::times_2));
8524     negptr(len);
8525 
8526     if (UseAVX > 1) {
8527       bind(copy_16_loop);
8528       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
8529       vmovdqu(Address(dst, len, Address::times_2), tmp1);
8530       addptr(len, 16);
8531       jcc(Assembler::notZero, copy_16_loop);
8532 
8533       bind(below_threshold);
8534       bind(copy_new_tail);
8535       movl(len, tmp2);
8536       andl(tmp2, 0x00000007);
8537       andl(len, 0xFFFFFFF8);
8538       jccb(Assembler::zero, copy_tail);
8539 
8540       pmovzxbw(tmp1, Address(src, 0));
8541       movdqu(Address(dst, 0), tmp1);
8542       addptr(src, 8);
8543       addptr(dst, 2 * 8);
8544 
8545       jmp(copy_tail, true);
8546     }
8547 
8548     // inflate 8 chars per iter
8549     bind(copy_8_loop);
8550     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
8551     movdqu(Address(dst, len, Address::times_2), tmp1);
8552     addptr(len, 8);
8553     jcc(Assembler::notZero, copy_8_loop);
8554 
8555     bind(copy_tail);
8556     movl(len, tmp2);
8557 
8558     cmpl(len, 4);
8559     jccb(Assembler::less, copy_bytes);
8560 
8561     movdl(tmp1, Address(src, 0));  // load 4 byte chars
8562     pmovzxbw(tmp1, tmp1);
8563     movq(Address(dst, 0), tmp1);
8564     subptr(len, 4);
8565     addptr(src, 4);
8566     addptr(dst, 8);
8567 
8568     bind(copy_bytes);
8569   } else {
8570     bind(below_threshold);
8571   }
8572 
8573   testl(len, len);
8574   jccb(Assembler::zero, done);
8575   lea(src, Address(src, len, Address::times_1));
8576   lea(dst, Address(dst, len, Address::times_2));
8577   negptr(len);
8578 
8579   // inflate 1 char per iter
8580   bind(copy_chars_loop);
8581   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
8582   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
8583   increment(len);
8584   jcc(Assembler::notZero, copy_chars_loop);
8585 
8586   bind(done);
8587 }
8588 
8589 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
8590   switch(type) {
8591     case T_BYTE:
8592     case T_BOOLEAN:
8593       evmovdqub(dst, kmask, src, merge, vector_len);
8594       break;
8595     case T_CHAR:
8596     case T_SHORT:
8597       evmovdquw(dst, kmask, src, merge, vector_len);
8598       break;
8599     case T_INT:
8600     case T_FLOAT:
8601       evmovdqul(dst, kmask, src, merge, vector_len);
8602       break;
8603     case T_LONG:
8604     case T_DOUBLE:
8605       evmovdquq(dst, kmask, src, merge, vector_len);
8606       break;
8607     default:
8608       fatal("Unexpected type argument %s", type2name(type));
8609       break;
8610   }
8611 }
8612 
8613 
8614 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
8615   switch(type) {
8616     case T_BYTE:
8617     case T_BOOLEAN:
8618       evmovdqub(dst, kmask, src, merge, vector_len);
8619       break;
8620     case T_CHAR:
8621     case T_SHORT:
8622       evmovdquw(dst, kmask, src, merge, vector_len);
8623       break;
8624     case T_INT:
8625     case T_FLOAT:
8626       evmovdqul(dst, kmask, src, merge, vector_len);
8627       break;
8628     case T_LONG:
8629     case T_DOUBLE:
8630       evmovdquq(dst, kmask, src, merge, vector_len);
8631       break;
8632     default:
8633       fatal("Unexpected type argument %s", type2name(type));
8634       break;
8635   }
8636 }
8637 
8638 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
8639   switch(type) {
8640     case T_BYTE:
8641     case T_BOOLEAN:
8642       evmovdqub(dst, kmask, src, merge, vector_len);
8643       break;
8644     case T_CHAR:
8645     case T_SHORT:
8646       evmovdquw(dst, kmask, src, merge, vector_len);
8647       break;
8648     case T_INT:
8649     case T_FLOAT:
8650       evmovdqul(dst, kmask, src, merge, vector_len);
8651       break;
8652     case T_LONG:
8653     case T_DOUBLE:
8654       evmovdquq(dst, kmask, src, merge, vector_len);
8655       break;
8656     default:
8657       fatal("Unexpected type argument %s", type2name(type));
8658       break;
8659   }
8660 }
8661 
8662 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
8663   switch(masklen) {
8664     case 2:
8665        knotbl(dst, src);
8666        movl(rtmp, 3);
8667        kmovbl(ktmp, rtmp);
8668        kandbl(dst, ktmp, dst);
8669        break;
8670     case 4:
8671        knotbl(dst, src);
8672        movl(rtmp, 15);
8673        kmovbl(ktmp, rtmp);
8674        kandbl(dst, ktmp, dst);
8675        break;
8676     case 8:
8677        knotbl(dst, src);
8678        break;
8679     case 16:
8680        knotwl(dst, src);
8681        break;
8682     case 32:
8683        knotdl(dst, src);
8684        break;
8685     case 64:
8686        knotql(dst, src);
8687        break;
8688     default:
8689       fatal("Unexpected vector length %d", masklen);
8690       break;
8691   }
8692 }
8693 
8694 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8695   switch(type) {
8696     case T_BOOLEAN:
8697     case T_BYTE:
8698        kandbl(dst, src1, src2);
8699        break;
8700     case T_CHAR:
8701     case T_SHORT:
8702        kandwl(dst, src1, src2);
8703        break;
8704     case T_INT:
8705     case T_FLOAT:
8706        kanddl(dst, src1, src2);
8707        break;
8708     case T_LONG:
8709     case T_DOUBLE:
8710        kandql(dst, src1, src2);
8711        break;
8712     default:
8713       fatal("Unexpected type argument %s", type2name(type));
8714       break;
8715   }
8716 }
8717 
8718 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8719   switch(type) {
8720     case T_BOOLEAN:
8721     case T_BYTE:
8722        korbl(dst, src1, src2);
8723        break;
8724     case T_CHAR:
8725     case T_SHORT:
8726        korwl(dst, src1, src2);
8727        break;
8728     case T_INT:
8729     case T_FLOAT:
8730        kordl(dst, src1, src2);
8731        break;
8732     case T_LONG:
8733     case T_DOUBLE:
8734        korql(dst, src1, src2);
8735        break;
8736     default:
8737       fatal("Unexpected type argument %s", type2name(type));
8738       break;
8739   }
8740 }
8741 
8742 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8743   switch(type) {
8744     case T_BOOLEAN:
8745     case T_BYTE:
8746        kxorbl(dst, src1, src2);
8747        break;
8748     case T_CHAR:
8749     case T_SHORT:
8750        kxorwl(dst, src1, src2);
8751        break;
8752     case T_INT:
8753     case T_FLOAT:
8754        kxordl(dst, src1, src2);
8755        break;
8756     case T_LONG:
8757     case T_DOUBLE:
8758        kxorql(dst, src1, src2);
8759        break;
8760     default:
8761       fatal("Unexpected type argument %s", type2name(type));
8762       break;
8763   }
8764 }
8765 
8766 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8767   switch(type) {
8768     case T_BOOLEAN:
8769     case T_BYTE:
8770       evpermb(dst, mask, nds, src, merge, vector_len); break;
8771     case T_CHAR:
8772     case T_SHORT:
8773       evpermw(dst, mask, nds, src, merge, vector_len); break;
8774     case T_INT:
8775     case T_FLOAT:
8776       evpermd(dst, mask, nds, src, merge, vector_len); break;
8777     case T_LONG:
8778     case T_DOUBLE:
8779       evpermq(dst, mask, nds, src, merge, vector_len); break;
8780     default:
8781       fatal("Unexpected type argument %s", type2name(type)); break;
8782   }
8783 }
8784 
8785 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8786   switch(type) {
8787     case T_BOOLEAN:
8788     case T_BYTE:
8789       evpermb(dst, mask, nds, src, merge, vector_len); break;
8790     case T_CHAR:
8791     case T_SHORT:
8792       evpermw(dst, mask, nds, src, merge, vector_len); break;
8793     case T_INT:
8794     case T_FLOAT:
8795       evpermd(dst, mask, nds, src, merge, vector_len); break;
8796     case T_LONG:
8797     case T_DOUBLE:
8798       evpermq(dst, mask, nds, src, merge, vector_len); break;
8799     default:
8800       fatal("Unexpected type argument %s", type2name(type)); break;
8801   }
8802 }
8803 
8804 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8805   switch(type) {
8806     case T_BYTE:
8807       evpminub(dst, mask, nds, src, merge, vector_len); break;
8808     case T_SHORT:
8809       evpminuw(dst, mask, nds, src, merge, vector_len); break;
8810     case T_INT:
8811       evpminud(dst, mask, nds, src, merge, vector_len); break;
8812     case T_LONG:
8813       evpminuq(dst, mask, nds, src, merge, vector_len); break;
8814     default:
8815       fatal("Unexpected type argument %s", type2name(type)); break;
8816   }
8817 }
8818 
8819 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8820   switch(type) {
8821     case T_BYTE:
8822       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
8823     case T_SHORT:
8824       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
8825     case T_INT:
8826       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
8827     case T_LONG:
8828       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
8829     default:
8830       fatal("Unexpected type argument %s", type2name(type)); break;
8831   }
8832 }
8833 
8834 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8835   switch(type) {
8836     case T_BYTE:
8837       evpminub(dst, mask, nds, src, merge, vector_len); break;
8838     case T_SHORT:
8839       evpminuw(dst, mask, nds, src, merge, vector_len); break;
8840     case T_INT:
8841       evpminud(dst, mask, nds, src, merge, vector_len); break;
8842     case T_LONG:
8843       evpminuq(dst, mask, nds, src, merge, vector_len); break;
8844     default:
8845       fatal("Unexpected type argument %s", type2name(type)); break;
8846   }
8847 }
8848 
8849 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8850   switch(type) {
8851     case T_BYTE:
8852       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
8853     case T_SHORT:
8854       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
8855     case T_INT:
8856       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
8857     case T_LONG:
8858       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
8859     default:
8860       fatal("Unexpected type argument %s", type2name(type)); break;
8861   }
8862 }
8863 
8864 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8865   switch(type) {
8866     case T_BYTE:
8867       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8868     case T_SHORT:
8869       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8870     case T_INT:
8871       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8872     case T_LONG:
8873       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8874     case T_FLOAT:
8875       evminmaxps(dst, mask, nds, src, merge, AVX10_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8876     case T_DOUBLE:
8877       evminmaxpd(dst, mask, nds, src, merge, AVX10_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8878     default:
8879       fatal("Unexpected type argument %s", type2name(type)); break;
8880   }
8881 }
8882 
8883 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8884   switch(type) {
8885     case T_BYTE:
8886       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8887     case T_SHORT:
8888       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8889     case T_INT:
8890       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8891     case T_LONG:
8892       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8893     case T_FLOAT:
8894       evminmaxps(dst, mask, nds, src, merge, AVX10_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8895     case T_DOUBLE:
8896       evminmaxpd(dst, mask, nds, src, merge, AVX10_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8897     default:
8898       fatal("Unexpected type argument %s", type2name(type)); break;
8899   }
8900 }
8901 
8902 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8903   switch(type) {
8904     case T_BYTE:
8905       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8906     case T_SHORT:
8907       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8908     case T_INT:
8909       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8910     case T_LONG:
8911       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8912     case T_FLOAT:
8913       evminmaxps(dst, mask, nds, src, merge, AVX10_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8914     case T_DOUBLE:
8915       evminmaxpd(dst, mask, nds, src, merge, AVX10_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8916     default:
8917       fatal("Unexpected type argument %s", type2name(type)); break;
8918   }
8919 }
8920 
8921 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8922   switch(type) {
8923     case T_BYTE:
8924       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8925     case T_SHORT:
8926       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8927     case T_INT:
8928       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8929     case T_LONG:
8930       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8931     case T_FLOAT:
8932       evminmaxps(dst, mask, nds, src, merge, AVX10_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8933     case T_DOUBLE:
8934       evminmaxps(dst, mask, nds, src, merge, AVX10_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8935     default:
8936       fatal("Unexpected type argument %s", type2name(type)); break;
8937   }
8938 }
8939 
8940 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8941   switch(type) {
8942     case T_INT:
8943       evpxord(dst, mask, nds, src, merge, vector_len); break;
8944     case T_LONG:
8945       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8946     default:
8947       fatal("Unexpected type argument %s", type2name(type)); break;
8948   }
8949 }
8950 
8951 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8952   switch(type) {
8953     case T_INT:
8954       evpxord(dst, mask, nds, src, merge, vector_len); break;
8955     case T_LONG:
8956       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8957     default:
8958       fatal("Unexpected type argument %s", type2name(type)); break;
8959   }
8960 }
8961 
8962 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8963   switch(type) {
8964     case T_INT:
8965       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8966     case T_LONG:
8967       evporq(dst, mask, nds, src, merge, vector_len); break;
8968     default:
8969       fatal("Unexpected type argument %s", type2name(type)); break;
8970   }
8971 }
8972 
8973 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8974   switch(type) {
8975     case T_INT:
8976       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8977     case T_LONG:
8978       evporq(dst, mask, nds, src, merge, vector_len); break;
8979     default:
8980       fatal("Unexpected type argument %s", type2name(type)); break;
8981   }
8982 }
8983 
8984 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8985   switch(type) {
8986     case T_INT:
8987       evpandd(dst, mask, nds, src, merge, vector_len); break;
8988     case T_LONG:
8989       evpandq(dst, mask, nds, src, merge, vector_len); break;
8990     default:
8991       fatal("Unexpected type argument %s", type2name(type)); break;
8992   }
8993 }
8994 
8995 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8996   switch(type) {
8997     case T_INT:
8998       evpandd(dst, mask, nds, src, merge, vector_len); break;
8999     case T_LONG:
9000       evpandq(dst, mask, nds, src, merge, vector_len); break;
9001     default:
9002       fatal("Unexpected type argument %s", type2name(type)); break;
9003   }
9004 }
9005 
9006 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
9007   switch(masklen) {
9008     case 8:
9009        kortestbl(src1, src2);
9010        break;
9011     case 16:
9012        kortestwl(src1, src2);
9013        break;
9014     case 32:
9015        kortestdl(src1, src2);
9016        break;
9017     case 64:
9018        kortestql(src1, src2);
9019        break;
9020     default:
9021       fatal("Unexpected mask length %d", masklen);
9022       break;
9023   }
9024 }
9025 
9026 
9027 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
9028   switch(masklen)  {
9029     case 8:
9030        ktestbl(src1, src2);
9031        break;
9032     case 16:
9033        ktestwl(src1, src2);
9034        break;
9035     case 32:
9036        ktestdl(src1, src2);
9037        break;
9038     case 64:
9039        ktestql(src1, src2);
9040        break;
9041     default:
9042       fatal("Unexpected mask length %d", masklen);
9043       break;
9044   }
9045 }
9046 
9047 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
9048   switch(type) {
9049     case T_INT:
9050       evprold(dst, mask, src, shift, merge, vlen_enc); break;
9051     case T_LONG:
9052       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
9053     default:
9054       fatal("Unexpected type argument %s", type2name(type)); break;
9055       break;
9056   }
9057 }
9058 
9059 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
9060   switch(type) {
9061     case T_INT:
9062       evprord(dst, mask, src, shift, merge, vlen_enc); break;
9063     case T_LONG:
9064       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
9065     default:
9066       fatal("Unexpected type argument %s", type2name(type)); break;
9067   }
9068 }
9069 
9070 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
9071   switch(type) {
9072     case T_INT:
9073       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
9074     case T_LONG:
9075       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
9076     default:
9077       fatal("Unexpected type argument %s", type2name(type)); break;
9078   }
9079 }
9080 
9081 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
9082   switch(type) {
9083     case T_INT:
9084       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
9085     case T_LONG:
9086       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
9087     default:
9088       fatal("Unexpected type argument %s", type2name(type)); break;
9089   }
9090 }
9091 
9092 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9093   assert(rscratch != noreg || always_reachable(src), "missing");
9094 
9095   if (reachable(src)) {
9096     evpandq(dst, nds, as_Address(src), vector_len);
9097   } else {
9098     lea(rscratch, src);
9099     evpandq(dst, nds, Address(rscratch, 0), vector_len);
9100   }
9101 }
9102 
9103 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
9104   assert(rscratch != noreg || always_reachable(src), "missing");
9105 
9106   if (reachable(src)) {
9107     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
9108   } else {
9109     lea(rscratch, src);
9110     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
9111   }
9112 }
9113 
9114 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9115   assert(rscratch != noreg || always_reachable(src), "missing");
9116 
9117   if (reachable(src)) {
9118     evporq(dst, nds, as_Address(src), vector_len);
9119   } else {
9120     lea(rscratch, src);
9121     evporq(dst, nds, Address(rscratch, 0), vector_len);
9122   }
9123 }
9124 
9125 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9126   assert(rscratch != noreg || always_reachable(src), "missing");
9127 
9128   if (reachable(src)) {
9129     vpshufb(dst, nds, as_Address(src), vector_len);
9130   } else {
9131     lea(rscratch, src);
9132     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
9133   }
9134 }
9135 
9136 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9137   assert(rscratch != noreg || always_reachable(src), "missing");
9138 
9139   if (reachable(src)) {
9140     Assembler::vpor(dst, nds, as_Address(src), vector_len);
9141   } else {
9142     lea(rscratch, src);
9143     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
9144   }
9145 }
9146 
9147 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
9148   assert(rscratch != noreg || always_reachable(src3), "missing");
9149 
9150   if (reachable(src3)) {
9151     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
9152   } else {
9153     lea(rscratch, src3);
9154     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
9155   }
9156 }
9157 
9158 #if COMPILER2_OR_JVMCI
9159 
9160 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
9161                                  Register length, Register temp, int vec_enc) {
9162   // Computing mask for predicated vector store.
9163   movptr(temp, -1);
9164   bzhiq(temp, temp, length);
9165   kmov(mask, temp);
9166   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
9167 }
9168 
9169 // Set memory operation for length "less than" 64 bytes.
9170 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
9171                                        XMMRegister xmm, KRegister mask, Register length,
9172                                        Register temp, bool use64byteVector) {
9173   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9174   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
9175   if (!use64byteVector) {
9176     fill32(dst, disp, xmm);
9177     subptr(length, 32 >> shift);
9178     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
9179   } else {
9180     assert(MaxVectorSize == 64, "vector length != 64");
9181     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
9182   }
9183 }
9184 
9185 
9186 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
9187                                        XMMRegister xmm, KRegister mask, Register length,
9188                                        Register temp) {
9189   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9190   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
9191   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
9192 }
9193 
9194 
9195 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
9196   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9197   vmovdqu(dst, xmm);
9198 }
9199 
9200 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
9201   fill32(Address(dst, disp), xmm);
9202 }
9203 
9204 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
9205   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9206   if (!use64byteVector) {
9207     fill32(dst, xmm);
9208     fill32(dst.plus_disp(32), xmm);
9209   } else {
9210     evmovdquq(dst, xmm, Assembler::AVX_512bit);
9211   }
9212 }
9213 
9214 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
9215   fill64(Address(dst, disp), xmm, use64byteVector);
9216 }
9217 
9218 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
9219                                         Register count, Register rtmp, XMMRegister xtmp) {
9220   Label L_exit;
9221   Label L_fill_start;
9222   Label L_fill_64_bytes;
9223   Label L_fill_96_bytes;
9224   Label L_fill_128_bytes;
9225   Label L_fill_128_bytes_loop;
9226   Label L_fill_128_loop_header;
9227   Label L_fill_128_bytes_loop_header;
9228   Label L_fill_128_bytes_loop_pre_header;
9229   Label L_fill_zmm_sequence;
9230 
9231   int shift = -1;
9232   int avx3threshold = VM_Version::avx3_threshold();
9233   switch(type) {
9234     case T_BYTE:  shift = 0;
9235       break;
9236     case T_SHORT: shift = 1;
9237       break;
9238     case T_INT:   shift = 2;
9239       break;
9240     /* Uncomment when LONG fill stubs are supported.
9241     case T_LONG:  shift = 3;
9242       break;
9243     */
9244     default:
9245       fatal("Unhandled type: %s\n", type2name(type));
9246   }
9247 
9248   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
9249 
9250     if (MaxVectorSize == 64) {
9251       cmpq(count, avx3threshold >> shift);
9252       jcc(Assembler::greater, L_fill_zmm_sequence);
9253     }
9254 
9255     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
9256 
9257     bind(L_fill_start);
9258 
9259     cmpq(count, 32 >> shift);
9260     jccb(Assembler::greater, L_fill_64_bytes);
9261     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
9262     jmp(L_exit);
9263 
9264     bind(L_fill_64_bytes);
9265     cmpq(count, 64 >> shift);
9266     jccb(Assembler::greater, L_fill_96_bytes);
9267     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
9268     jmp(L_exit);
9269 
9270     bind(L_fill_96_bytes);
9271     cmpq(count, 96 >> shift);
9272     jccb(Assembler::greater, L_fill_128_bytes);
9273     fill64(to, 0, xtmp);
9274     subq(count, 64 >> shift);
9275     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
9276     jmp(L_exit);
9277 
9278     bind(L_fill_128_bytes);
9279     cmpq(count, 128 >> shift);
9280     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
9281     fill64(to, 0, xtmp);
9282     fill32(to, 64, xtmp);
9283     subq(count, 96 >> shift);
9284     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
9285     jmp(L_exit);
9286 
9287     bind(L_fill_128_bytes_loop_pre_header);
9288     {
9289       mov(rtmp, to);
9290       andq(rtmp, 31);
9291       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
9292       negq(rtmp);
9293       addq(rtmp, 32);
9294       mov64(r8, -1L);
9295       bzhiq(r8, r8, rtmp);
9296       kmovql(k2, r8);
9297       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
9298       addq(to, rtmp);
9299       shrq(rtmp, shift);
9300       subq(count, rtmp);
9301     }
9302 
9303     cmpq(count, 128 >> shift);
9304     jcc(Assembler::less, L_fill_start);
9305 
9306     bind(L_fill_128_bytes_loop_header);
9307     subq(count, 128 >> shift);
9308 
9309     align32();
9310     bind(L_fill_128_bytes_loop);
9311       fill64(to, 0, xtmp);
9312       fill64(to, 64, xtmp);
9313       addq(to, 128);
9314       subq(count, 128 >> shift);
9315       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
9316 
9317     addq(count, 128 >> shift);
9318     jcc(Assembler::zero, L_exit);
9319     jmp(L_fill_start);
9320   }
9321 
9322   if (MaxVectorSize == 64) {
9323     // Sequence using 64 byte ZMM register.
9324     Label L_fill_128_bytes_zmm;
9325     Label L_fill_192_bytes_zmm;
9326     Label L_fill_192_bytes_loop_zmm;
9327     Label L_fill_192_bytes_loop_header_zmm;
9328     Label L_fill_192_bytes_loop_pre_header_zmm;
9329     Label L_fill_start_zmm_sequence;
9330 
9331     bind(L_fill_zmm_sequence);
9332     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
9333 
9334     bind(L_fill_start_zmm_sequence);
9335     cmpq(count, 64 >> shift);
9336     jccb(Assembler::greater, L_fill_128_bytes_zmm);
9337     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
9338     jmp(L_exit);
9339 
9340     bind(L_fill_128_bytes_zmm);
9341     cmpq(count, 128 >> shift);
9342     jccb(Assembler::greater, L_fill_192_bytes_zmm);
9343     fill64(to, 0, xtmp, true);
9344     subq(count, 64 >> shift);
9345     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
9346     jmp(L_exit);
9347 
9348     bind(L_fill_192_bytes_zmm);
9349     cmpq(count, 192 >> shift);
9350     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
9351     fill64(to, 0, xtmp, true);
9352     fill64(to, 64, xtmp, true);
9353     subq(count, 128 >> shift);
9354     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
9355     jmp(L_exit);
9356 
9357     bind(L_fill_192_bytes_loop_pre_header_zmm);
9358     {
9359       movq(rtmp, to);
9360       andq(rtmp, 63);
9361       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
9362       negq(rtmp);
9363       addq(rtmp, 64);
9364       mov64(r8, -1L);
9365       bzhiq(r8, r8, rtmp);
9366       kmovql(k2, r8);
9367       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
9368       addq(to, rtmp);
9369       shrq(rtmp, shift);
9370       subq(count, rtmp);
9371     }
9372 
9373     cmpq(count, 192 >> shift);
9374     jcc(Assembler::less, L_fill_start_zmm_sequence);
9375 
9376     bind(L_fill_192_bytes_loop_header_zmm);
9377     subq(count, 192 >> shift);
9378 
9379     align32();
9380     bind(L_fill_192_bytes_loop_zmm);
9381       fill64(to, 0, xtmp, true);
9382       fill64(to, 64, xtmp, true);
9383       fill64(to, 128, xtmp, true);
9384       addq(to, 192);
9385       subq(count, 192 >> shift);
9386       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
9387 
9388     addq(count, 192 >> shift);
9389     jcc(Assembler::zero, L_exit);
9390     jmp(L_fill_start_zmm_sequence);
9391   }
9392   bind(L_exit);
9393 }
9394 #endif //COMPILER2_OR_JVMCI
9395 
9396 
9397 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
9398   Label done;
9399   cvttss2sil(dst, src);
9400   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9401   cmpl(dst, 0x80000000); // float_sign_flip
9402   jccb(Assembler::notEqual, done);
9403   subptr(rsp, 8);
9404   movflt(Address(rsp, 0), src);
9405   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9406   pop(dst);
9407   bind(done);
9408 }
9409 
9410 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
9411   Label done;
9412   cvttsd2sil(dst, src);
9413   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9414   cmpl(dst, 0x80000000); // float_sign_flip
9415   jccb(Assembler::notEqual, done);
9416   subptr(rsp, 8);
9417   movdbl(Address(rsp, 0), src);
9418   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9419   pop(dst);
9420   bind(done);
9421 }
9422 
9423 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
9424   Label done;
9425   cvttss2siq(dst, src);
9426   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9427   jccb(Assembler::notEqual, done);
9428   subptr(rsp, 8);
9429   movflt(Address(rsp, 0), src);
9430   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9431   pop(dst);
9432   bind(done);
9433 }
9434 
9435 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
9436   // Following code is line by line assembly translation rounding algorithm.
9437   // Please refer to java.lang.Math.round(float) algorithm for details.
9438   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
9439   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
9440   const int32_t FloatConsts_EXP_BIAS = 127;
9441   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
9442   const int32_t MINUS_32 = 0xFFFFFFE0;
9443   Label L_special_case, L_block1, L_exit;
9444   movl(rtmp, FloatConsts_EXP_BIT_MASK);
9445   movdl(dst, src);
9446   andl(dst, rtmp);
9447   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
9448   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
9449   subl(rtmp, dst);
9450   movl(rcx, rtmp);
9451   movl(dst, MINUS_32);
9452   testl(rtmp, dst);
9453   jccb(Assembler::notEqual, L_special_case);
9454   movdl(dst, src);
9455   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
9456   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
9457   movdl(rtmp, src);
9458   testl(rtmp, rtmp);
9459   jccb(Assembler::greaterEqual, L_block1);
9460   negl(dst);
9461   bind(L_block1);
9462   sarl(dst);
9463   addl(dst, 0x1);
9464   sarl(dst, 0x1);
9465   jmp(L_exit);
9466   bind(L_special_case);
9467   convert_f2i(dst, src);
9468   bind(L_exit);
9469 }
9470 
9471 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
9472   // Following code is line by line assembly translation rounding algorithm.
9473   // Please refer to java.lang.Math.round(double) algorithm for details.
9474   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
9475   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
9476   const int64_t DoubleConsts_EXP_BIAS = 1023;
9477   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
9478   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
9479   Label L_special_case, L_block1, L_exit;
9480   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
9481   movq(dst, src);
9482   andq(dst, rtmp);
9483   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
9484   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
9485   subq(rtmp, dst);
9486   movq(rcx, rtmp);
9487   mov64(dst, MINUS_64);
9488   testq(rtmp, dst);
9489   jccb(Assembler::notEqual, L_special_case);
9490   movq(dst, src);
9491   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
9492   andq(dst, rtmp);
9493   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
9494   orq(dst, rtmp);
9495   movq(rtmp, src);
9496   testq(rtmp, rtmp);
9497   jccb(Assembler::greaterEqual, L_block1);
9498   negq(dst);
9499   bind(L_block1);
9500   sarq(dst);
9501   addq(dst, 0x1);
9502   sarq(dst, 0x1);
9503   jmp(L_exit);
9504   bind(L_special_case);
9505   convert_d2l(dst, src);
9506   bind(L_exit);
9507 }
9508 
9509 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
9510   Label done;
9511   cvttsd2siq(dst, src);
9512   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9513   jccb(Assembler::notEqual, done);
9514   subptr(rsp, 8);
9515   movdbl(Address(rsp, 0), src);
9516   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9517   pop(dst);
9518   bind(done);
9519 }
9520 
9521 void MacroAssembler::cache_wb(Address line)
9522 {
9523   // 64 bit cpus always support clflush
9524   assert(VM_Version::supports_clflush(), "clflush should be available");
9525   bool optimized = VM_Version::supports_clflushopt();
9526   bool no_evict = VM_Version::supports_clwb();
9527 
9528   // prefer clwb (writeback without evict) otherwise
9529   // prefer clflushopt (potentially parallel writeback with evict)
9530   // otherwise fallback on clflush (serial writeback with evict)
9531 
9532   if (optimized) {
9533     if (no_evict) {
9534       clwb(line);
9535     } else {
9536       clflushopt(line);
9537     }
9538   } else {
9539     // no need for fence when using CLFLUSH
9540     clflush(line);
9541   }
9542 }
9543 
9544 void MacroAssembler::cache_wbsync(bool is_pre)
9545 {
9546   assert(VM_Version::supports_clflush(), "clflush should be available");
9547   bool optimized = VM_Version::supports_clflushopt();
9548   bool no_evict = VM_Version::supports_clwb();
9549 
9550   // pick the correct implementation
9551 
9552   if (!is_pre && (optimized || no_evict)) {
9553     // need an sfence for post flush when using clflushopt or clwb
9554     // otherwise no no need for any synchroniaztion
9555 
9556     sfence();
9557   }
9558 }
9559 
9560 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
9561   switch (cond) {
9562     // Note some conditions are synonyms for others
9563     case Assembler::zero:         return Assembler::notZero;
9564     case Assembler::notZero:      return Assembler::zero;
9565     case Assembler::less:         return Assembler::greaterEqual;
9566     case Assembler::lessEqual:    return Assembler::greater;
9567     case Assembler::greater:      return Assembler::lessEqual;
9568     case Assembler::greaterEqual: return Assembler::less;
9569     case Assembler::below:        return Assembler::aboveEqual;
9570     case Assembler::belowEqual:   return Assembler::above;
9571     case Assembler::above:        return Assembler::belowEqual;
9572     case Assembler::aboveEqual:   return Assembler::below;
9573     case Assembler::overflow:     return Assembler::noOverflow;
9574     case Assembler::noOverflow:   return Assembler::overflow;
9575     case Assembler::negative:     return Assembler::positive;
9576     case Assembler::positive:     return Assembler::negative;
9577     case Assembler::parity:       return Assembler::noParity;
9578     case Assembler::noParity:     return Assembler::parity;
9579   }
9580   ShouldNotReachHere(); return Assembler::overflow;
9581 }
9582 
9583 // This is simply a call to Thread::current()
9584 void MacroAssembler::get_thread_slow(Register thread) {
9585   if (thread != rax) {
9586     push(rax);
9587   }
9588   push(rdi);
9589   push(rsi);
9590   push(rdx);
9591   push(rcx);
9592   push(r8);
9593   push(r9);
9594   push(r10);
9595   push(r11);
9596 
9597   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
9598 
9599   pop(r11);
9600   pop(r10);
9601   pop(r9);
9602   pop(r8);
9603   pop(rcx);
9604   pop(rdx);
9605   pop(rsi);
9606   pop(rdi);
9607   if (thread != rax) {
9608     mov(thread, rax);
9609     pop(rax);
9610   }
9611 }
9612 
9613 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
9614   Label L_stack_ok;
9615   if (bias == 0) {
9616     testptr(sp, 2 * wordSize - 1);
9617   } else {
9618     // lea(tmp, Address(rsp, bias);
9619     mov(tmp, sp);
9620     addptr(tmp, bias);
9621     testptr(tmp, 2 * wordSize - 1);
9622   }
9623   jcc(Assembler::equal, L_stack_ok);
9624   block_comment(msg);
9625   stop(msg);
9626   bind(L_stack_ok);
9627 }
9628 
9629 // Implements lightweight-locking.
9630 //
9631 // obj: the object to be locked
9632 // reg_rax: rax
9633 // thread: the thread which attempts to lock obj
9634 // tmp: a temporary register
9635 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register tmp, Label& slow) {
9636   Register thread = r15_thread;
9637 
9638   assert(reg_rax == rax, "");
9639   assert_different_registers(basic_lock, obj, reg_rax, thread, tmp);
9640 
9641   Label push;
9642   const Register top = tmp;
9643 
9644   // Preload the markWord. It is important that this is the first
9645   // instruction emitted as it is part of C1's null check semantics.
9646   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
9647 
9648   if (UseObjectMonitorTable) {
9649     // Clear cache in case fast locking succeeds or we need to take the slow-path.
9650     movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0);
9651   }
9652 
9653   if (DiagnoseSyncOnValueBasedClasses != 0) {
9654     load_klass(tmp, obj, rscratch1);
9655     testb(Address(tmp, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
9656     jcc(Assembler::notZero, slow);
9657   }
9658 
9659   // Load top.
9660   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9661 
9662   // Check if the lock-stack is full.
9663   cmpl(top, LockStack::end_offset());
9664   jcc(Assembler::greaterEqual, slow);
9665 
9666   // Check for recursion.
9667   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
9668   jcc(Assembler::equal, push);
9669 
9670   // Check header for monitor (0b10).
9671   testptr(reg_rax, markWord::monitor_value);
9672   jcc(Assembler::notZero, slow);
9673 
9674   // Try to lock. Transition lock bits 0b01 => 0b00
9675   movptr(tmp, reg_rax);
9676   andptr(tmp, ~(int32_t)markWord::unlocked_value);
9677   orptr(reg_rax, markWord::unlocked_value);
9678   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
9679   jcc(Assembler::notEqual, slow);
9680 
9681   // Restore top, CAS clobbers register.
9682   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9683 
9684   bind(push);
9685   // After successful lock, push object on lock-stack.
9686   movptr(Address(thread, top), obj);
9687   incrementl(top, oopSize);
9688   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
9689 }
9690 
9691 // Implements lightweight-unlocking.
9692 //
9693 // obj: the object to be unlocked
9694 // reg_rax: rax
9695 // thread: the thread
9696 // tmp: a temporary register
9697 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register tmp, Label& slow) {
9698   Register thread = r15_thread;
9699 
9700   assert(reg_rax == rax, "");
9701   assert_different_registers(obj, reg_rax, thread, tmp);
9702 
9703   Label unlocked, push_and_slow;
9704   const Register top = tmp;
9705 
9706   // Check if obj is top of lock-stack.
9707   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9708   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
9709   jcc(Assembler::notEqual, slow);
9710 
9711   // Pop lock-stack.
9712   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
9713   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
9714 
9715   // Check if recursive.
9716   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
9717   jcc(Assembler::equal, unlocked);
9718 
9719   // Not recursive. Check header for monitor (0b10).
9720   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
9721   testptr(reg_rax, markWord::monitor_value);
9722   jcc(Assembler::notZero, push_and_slow);
9723 
9724 #ifdef ASSERT
9725   // Check header not unlocked (0b01).
9726   Label not_unlocked;
9727   testptr(reg_rax, markWord::unlocked_value);
9728   jcc(Assembler::zero, not_unlocked);
9729   stop("lightweight_unlock already unlocked");
9730   bind(not_unlocked);
9731 #endif
9732 
9733   // Try to unlock. Transition lock bits 0b00 => 0b01
9734   movptr(tmp, reg_rax);
9735   orptr(tmp, markWord::unlocked_value);
9736   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
9737   jcc(Assembler::equal, unlocked);
9738 
9739   bind(push_and_slow);
9740   // Restore lock-stack and handle the unlock in runtime.
9741 #ifdef ASSERT
9742   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9743   movptr(Address(thread, top), obj);
9744 #endif
9745   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
9746   jmp(slow);
9747 
9748   bind(unlocked);
9749 }
9750 
9751 // Saves legacy GPRs state on stack.
9752 void MacroAssembler::save_legacy_gprs() {
9753   subq(rsp, 16 * wordSize);
9754   movq(Address(rsp, 15 * wordSize), rax);
9755   movq(Address(rsp, 14 * wordSize), rcx);
9756   movq(Address(rsp, 13 * wordSize), rdx);
9757   movq(Address(rsp, 12 * wordSize), rbx);
9758   movq(Address(rsp, 10 * wordSize), rbp);
9759   movq(Address(rsp, 9 * wordSize), rsi);
9760   movq(Address(rsp, 8 * wordSize), rdi);
9761   movq(Address(rsp, 7 * wordSize), r8);
9762   movq(Address(rsp, 6 * wordSize), r9);
9763   movq(Address(rsp, 5 * wordSize), r10);
9764   movq(Address(rsp, 4 * wordSize), r11);
9765   movq(Address(rsp, 3 * wordSize), r12);
9766   movq(Address(rsp, 2 * wordSize), r13);
9767   movq(Address(rsp, wordSize), r14);
9768   movq(Address(rsp, 0), r15);
9769 }
9770 
9771 // Resotres back legacy GPRs state from stack.
9772 void MacroAssembler::restore_legacy_gprs() {
9773   movq(r15, Address(rsp, 0));
9774   movq(r14, Address(rsp, wordSize));
9775   movq(r13, Address(rsp, 2 * wordSize));
9776   movq(r12, Address(rsp, 3 * wordSize));
9777   movq(r11, Address(rsp, 4 * wordSize));
9778   movq(r10, Address(rsp, 5 * wordSize));
9779   movq(r9,  Address(rsp, 6 * wordSize));
9780   movq(r8,  Address(rsp, 7 * wordSize));
9781   movq(rdi, Address(rsp, 8 * wordSize));
9782   movq(rsi, Address(rsp, 9 * wordSize));
9783   movq(rbp, Address(rsp, 10 * wordSize));
9784   movq(rbx, Address(rsp, 12 * wordSize));
9785   movq(rdx, Address(rsp, 13 * wordSize));
9786   movq(rcx, Address(rsp, 14 * wordSize));
9787   movq(rax, Address(rsp, 15 * wordSize));
9788   addq(rsp, 16 * wordSize);
9789 }
9790 
9791 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
9792   if (VM_Version::supports_apx_f()) {
9793     esetzucc(comparison, dst);
9794   } else {
9795     setb(comparison, dst);
9796     movzbl(dst, dst);
9797   }
9798 }