1 /*
   2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/assembler.hpp"
  26 #include "asm/assembler.inline.hpp"
  27 #include "code/aotCodeCache.hpp"
  28 #include "code/compiledIC.hpp"
  29 #include "compiler/compiler_globals.hpp"
  30 #include "compiler/disassembler.hpp"

  31 #include "crc32c.h"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/barrierSetAssembler.hpp"
  34 #include "gc/shared/collectedHeap.inline.hpp"
  35 #include "gc/shared/tlab_globals.hpp"
  36 #include "interpreter/bytecodeHistogram.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interpreterRuntime.hpp"
  39 #include "jvm.h"
  40 #include "memory/resourceArea.hpp"
  41 #include "memory/universe.hpp"
  42 #include "oops/accessDecorators.hpp"
  43 #include "oops/compressedKlass.inline.hpp"
  44 #include "oops/compressedOops.inline.hpp"
  45 #include "oops/klass.inline.hpp"

  46 #include "prims/methodHandles.hpp"

  47 #include "runtime/continuation.hpp"
  48 #include "runtime/interfaceSupport.inline.hpp"
  49 #include "runtime/javaThread.hpp"
  50 #include "runtime/jniHandles.hpp"
  51 #include "runtime/objectMonitor.hpp"
  52 #include "runtime/os.hpp"
  53 #include "runtime/safepoint.hpp"
  54 #include "runtime/safepointMechanism.hpp"
  55 #include "runtime/sharedRuntime.hpp"

  56 #include "runtime/stubRoutines.hpp"
  57 #include "utilities/checkedCast.hpp"
  58 #include "utilities/macros.hpp"




  59 
  60 #ifdef PRODUCT
  61 #define BLOCK_COMMENT(str) /* nothing */
  62 #define STOP(error) stop(error)
  63 #else
  64 #define BLOCK_COMMENT(str) block_comment(str)
  65 #define STOP(error) block_comment(error); stop(error)
  66 #endif
  67 
  68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  69 
  70 #ifdef ASSERT
  71 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  72 #endif
  73 
  74 static const Assembler::Condition reverse[] = {
  75     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  76     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  77     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  78     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  79     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  80     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  81     Assembler::above          /* belowEqual    = 0x6 */ ,
  82     Assembler::belowEqual     /* above         = 0x7 */ ,
  83     Assembler::positive       /* negative      = 0x8 */ ,
  84     Assembler::negative       /* positive      = 0x9 */ ,
  85     Assembler::noParity       /* parity        = 0xa */ ,
  86     Assembler::parity         /* noParity      = 0xb */ ,
  87     Assembler::greaterEqual   /* less          = 0xc */ ,
  88     Assembler::less           /* greaterEqual  = 0xd */ ,
  89     Assembler::greater        /* lessEqual     = 0xe */ ,
  90     Assembler::lessEqual      /* greater       = 0xf, */
  91 
  92 };
  93 
  94 
  95 // Implementation of MacroAssembler
  96 
  97 Address MacroAssembler::as_Address(AddressLiteral adr) {
  98   // amd64 always does this as a pc-rel
  99   // we can be absolute or disp based on the instruction type
 100   // jmp/call are displacements others are absolute
 101   assert(!adr.is_lval(), "must be rval");
 102   assert(reachable(adr), "must be");
 103   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
 104 
 105 }
 106 
 107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
 108   AddressLiteral base = adr.base();
 109   lea(rscratch, base);
 110   Address index = adr.index();
 111   assert(index._disp == 0, "must not have disp"); // maybe it can?
 112   Address array(rscratch, index._index, index._scale, index._disp);
 113   return array;
 114 }
 115 
 116 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 117   Label L, E;
 118 
 119 #ifdef _WIN64
 120   // Windows always allocates space for it's register args
 121   assert(num_args <= 4, "only register arguments supported");
 122   subq(rsp,  frame::arg_reg_save_area_bytes);
 123 #endif
 124 
 125   // Align stack if necessary
 126   testl(rsp, 15);
 127   jcc(Assembler::zero, L);
 128 
 129   subq(rsp, 8);
 130   call(RuntimeAddress(entry_point));
 131   addq(rsp, 8);
 132   jmp(E);
 133 
 134   bind(L);
 135   call(RuntimeAddress(entry_point));
 136 
 137   bind(E);
 138 
 139 #ifdef _WIN64
 140   // restore stack pointer
 141   addq(rsp, frame::arg_reg_save_area_bytes);
 142 #endif
 143 }
 144 
 145 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
 146   assert(!src2.is_lval(), "should use cmpptr");
 147   assert(rscratch != noreg || always_reachable(src2), "missing");
 148 
 149   if (reachable(src2)) {
 150     cmpq(src1, as_Address(src2));
 151   } else {
 152     lea(rscratch, src2);
 153     Assembler::cmpq(src1, Address(rscratch, 0));
 154   }
 155 }
 156 
 157 int MacroAssembler::corrected_idivq(Register reg) {
 158   // Full implementation of Java ldiv and lrem; checks for special
 159   // case as described in JVM spec., p.243 & p.271.  The function
 160   // returns the (pc) offset of the idivl instruction - may be needed
 161   // for implicit exceptions.
 162   //
 163   //         normal case                           special case
 164   //
 165   // input : rax: dividend                         min_long
 166   //         reg: divisor   (may not be eax/edx)   -1
 167   //
 168   // output: rax: quotient  (= rax idiv reg)       min_long
 169   //         rdx: remainder (= rax irem reg)       0
 170   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 171   static const int64_t min_long = 0x8000000000000000;
 172   Label normal_case, special_case;
 173 
 174   // check for special case
 175   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
 176   jcc(Assembler::notEqual, normal_case);
 177   xorl(rdx, rdx); // prepare rdx for possible special case (where
 178                   // remainder = 0)
 179   cmpq(reg, -1);
 180   jcc(Assembler::equal, special_case);
 181 
 182   // handle normal case
 183   bind(normal_case);
 184   cdqq();
 185   int idivq_offset = offset();
 186   idivq(reg);
 187 
 188   // normal and special case exit
 189   bind(special_case);
 190 
 191   return idivq_offset;
 192 }
 193 
 194 void MacroAssembler::decrementq(Register reg, int value) {
 195   if (value == min_jint) { subq(reg, value); return; }
 196   if (value <  0) { incrementq(reg, -value); return; }
 197   if (value == 0) {                        ; return; }
 198   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 199   /* else */      { subq(reg, value)       ; return; }
 200 }
 201 
 202 void MacroAssembler::decrementq(Address dst, int value) {
 203   if (value == min_jint) { subq(dst, value); return; }
 204   if (value <  0) { incrementq(dst, -value); return; }
 205   if (value == 0) {                        ; return; }
 206   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 207   /* else */      { subq(dst, value)       ; return; }
 208 }
 209 
 210 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
 211   assert(rscratch != noreg || always_reachable(dst), "missing");
 212 
 213   if (reachable(dst)) {
 214     incrementq(as_Address(dst));
 215   } else {
 216     lea(rscratch, dst);
 217     incrementq(Address(rscratch, 0));
 218   }
 219 }
 220 
 221 void MacroAssembler::incrementq(Register reg, int value) {
 222   if (value == min_jint) { addq(reg, value); return; }
 223   if (value <  0) { decrementq(reg, -value); return; }
 224   if (value == 0) {                        ; return; }
 225   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 226   /* else */      { addq(reg, value)       ; return; }
 227 }
 228 
 229 void MacroAssembler::incrementq(Address dst, int value) {
 230   if (value == min_jint) { addq(dst, value); return; }
 231   if (value <  0) { decrementq(dst, -value); return; }
 232   if (value == 0) {                        ; return; }
 233   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 234   /* else */      { addq(dst, value)       ; return; }
 235 }
 236 
 237 // 32bit can do a case table jump in one instruction but we no longer allow the base
 238 // to be installed in the Address class
 239 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
 240   lea(rscratch, entry.base());
 241   Address dispatch = entry.index();
 242   assert(dispatch._base == noreg, "must be");
 243   dispatch._base = rscratch;
 244   jmp(dispatch);
 245 }
 246 
 247 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 248   ShouldNotReachHere(); // 64bit doesn't use two regs
 249   cmpq(x_lo, y_lo);
 250 }
 251 
 252 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 253   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 254 }
 255 
 256 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
 257   lea(rscratch, adr);
 258   movptr(dst, rscratch);
 259 }
 260 
 261 void MacroAssembler::leave() {
 262   // %%% is this really better? Why not on 32bit too?
 263   emit_int8((unsigned char)0xC9); // LEAVE
 264 }
 265 
 266 void MacroAssembler::lneg(Register hi, Register lo) {
 267   ShouldNotReachHere(); // 64bit doesn't use two regs
 268   negq(lo);
 269 }
 270 
 271 void MacroAssembler::movoop(Register dst, jobject obj) {
 272   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 273 }
 274 
 275 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
 276   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 277   movq(dst, rscratch);
 278 }
 279 
 280 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 281   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 282 }
 283 
 284 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
 285   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 286   movq(dst, rscratch);
 287 }
 288 
 289 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 290   if (src.is_lval()) {
 291     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 292   } else {
 293     if (reachable(src)) {
 294       movq(dst, as_Address(src));
 295     } else {
 296       lea(dst, src);
 297       movq(dst, Address(dst, 0));
 298     }
 299   }
 300 }
 301 
 302 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
 303   movq(as_Address(dst, rscratch), src);
 304 }
 305 
 306 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 307   movq(dst, as_Address(src, dst /*rscratch*/));
 308 }
 309 
 310 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 311 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
 312   if (is_simm32(src)) {
 313     movptr(dst, checked_cast<int32_t>(src));
 314   } else {
 315     mov64(rscratch, src);
 316     movq(dst, rscratch);
 317   }
 318 }
 319 
 320 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
 321   movoop(rscratch, obj);
 322   push(rscratch);
 323 }
 324 
 325 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
 326   mov_metadata(rscratch, obj);
 327   push(rscratch);
 328 }
 329 
 330 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
 331   lea(rscratch, src);
 332   if (src.is_lval()) {
 333     push(rscratch);
 334   } else {
 335     pushq(Address(rscratch, 0));
 336   }
 337 }
 338 
 339 static void pass_arg0(MacroAssembler* masm, Register arg) {
 340   if (c_rarg0 != arg ) {
 341     masm->mov(c_rarg0, arg);
 342   }
 343 }
 344 
 345 static void pass_arg1(MacroAssembler* masm, Register arg) {
 346   if (c_rarg1 != arg ) {
 347     masm->mov(c_rarg1, arg);
 348   }
 349 }
 350 
 351 static void pass_arg2(MacroAssembler* masm, Register arg) {
 352   if (c_rarg2 != arg ) {
 353     masm->mov(c_rarg2, arg);
 354   }
 355 }
 356 
 357 static void pass_arg3(MacroAssembler* masm, Register arg) {
 358   if (c_rarg3 != arg ) {
 359     masm->mov(c_rarg3, arg);
 360   }
 361 }
 362 
 363 void MacroAssembler::stop(const char* msg) {
 364   if (ShowMessageBoxOnError) {
 365     address rip = pc();
 366     pusha(); // get regs on stack
 367     lea(c_rarg1, InternalAddress(rip));
 368     movq(c_rarg2, rsp); // pass pointer to regs array
 369   }
 370   // Skip AOT caching C strings in scratch buffer.
 371   const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
 372   lea(c_rarg0, ExternalAddress((address) str));
 373   andq(rsp, -16); // align stack as required by ABI
 374   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 375   hlt();
 376 }
 377 
 378 void MacroAssembler::warn(const char* msg) {
 379   push(rbp);
 380   movq(rbp, rsp);
 381   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 382   push_CPU_state();   // keeps alignment at 16 bytes
 383 
 384 #ifdef _WIN64
 385   // Windows always allocates space for its register args
 386   subq(rsp,  frame::arg_reg_save_area_bytes);
 387 #endif
 388   lea(c_rarg0, ExternalAddress((address) msg));
 389   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 390 
 391 #ifdef _WIN64
 392   // restore stack pointer
 393   addq(rsp, frame::arg_reg_save_area_bytes);
 394 #endif
 395   pop_CPU_state();
 396   mov(rsp, rbp);
 397   pop(rbp);
 398 }
 399 
 400 void MacroAssembler::print_state() {
 401   address rip = pc();
 402   pusha();            // get regs on stack
 403   push(rbp);
 404   movq(rbp, rsp);
 405   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 406   push_CPU_state();   // keeps alignment at 16 bytes
 407 
 408   lea(c_rarg0, InternalAddress(rip));
 409   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 410   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 411 
 412   pop_CPU_state();
 413   mov(rsp, rbp);
 414   pop(rbp);
 415   popa();
 416 }
 417 
 418 #ifndef PRODUCT
 419 extern "C" void findpc(intptr_t x);
 420 #endif
 421 
 422 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 423   // In order to get locks to work, we need to fake a in_VM state
 424   if (ShowMessageBoxOnError) {
 425     JavaThread* thread = JavaThread::current();
 426     JavaThreadState saved_state = thread->thread_state();
 427     thread->set_thread_state(_thread_in_vm);
 428 #ifndef PRODUCT
 429     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 430       ttyLocker ttyl;
 431       BytecodeCounter::print();
 432     }
 433 #endif
 434     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 435     // XXX correct this offset for amd64
 436     // This is the value of eip which points to where verify_oop will return.
 437     if (os::message_box(msg, "Execution stopped, print registers?")) {
 438       print_state64(pc, regs);
 439       BREAKPOINT;
 440     }
 441   }
 442   fatal("DEBUG MESSAGE: %s", msg);
 443 }
 444 
 445 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 446   ttyLocker ttyl;
 447   DebuggingContext debugging{};
 448   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 449 #ifndef PRODUCT
 450   tty->cr();
 451   findpc(pc);
 452   tty->cr();
 453 #endif
 454 #define PRINT_REG(rax, value) \
 455   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 456   PRINT_REG(rax, regs[15]);
 457   PRINT_REG(rbx, regs[12]);
 458   PRINT_REG(rcx, regs[14]);
 459   PRINT_REG(rdx, regs[13]);
 460   PRINT_REG(rdi, regs[8]);
 461   PRINT_REG(rsi, regs[9]);
 462   PRINT_REG(rbp, regs[10]);
 463   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
 464   PRINT_REG(rsp, (intptr_t)(&regs[16]));
 465   PRINT_REG(r8 , regs[7]);
 466   PRINT_REG(r9 , regs[6]);
 467   PRINT_REG(r10, regs[5]);
 468   PRINT_REG(r11, regs[4]);
 469   PRINT_REG(r12, regs[3]);
 470   PRINT_REG(r13, regs[2]);
 471   PRINT_REG(r14, regs[1]);
 472   PRINT_REG(r15, regs[0]);
 473 #undef PRINT_REG
 474   // Print some words near the top of the stack.
 475   int64_t* rsp = &regs[16];
 476   int64_t* dump_sp = rsp;
 477   for (int col1 = 0; col1 < 8; col1++) {
 478     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 479     os::print_location(tty, *dump_sp++);
 480   }
 481   for (int row = 0; row < 25; row++) {
 482     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 483     for (int col = 0; col < 4; col++) {
 484       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 485     }
 486     tty->cr();
 487   }
 488   // Print some instructions around pc:
 489   Disassembler::decode((address)pc-64, (address)pc);
 490   tty->print_cr("--------");
 491   Disassembler::decode((address)pc, (address)pc+32);
 492 }
 493 
 494 // The java_calling_convention describes stack locations as ideal slots on
 495 // a frame with no abi restrictions. Since we must observe abi restrictions
 496 // (like the placement of the register window) the slots must be biased by
 497 // the following value.
 498 static int reg2offset_in(VMReg r) {
 499   // Account for saved rbp and return address
 500   // This should really be in_preserve_stack_slots
 501   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
 502 }
 503 
 504 static int reg2offset_out(VMReg r) {
 505   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 506 }
 507 
 508 // A long move
 509 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 510 
 511   // The calling conventions assures us that each VMregpair is either
 512   // all really one physical register or adjacent stack slots.
 513 
 514   if (src.is_single_phys_reg() ) {
 515     if (dst.is_single_phys_reg()) {
 516       if (dst.first() != src.first()) {
 517         mov(dst.first()->as_Register(), src.first()->as_Register());
 518       }
 519     } else {
 520       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
 521              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
 522       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 523     }
 524   } else if (dst.is_single_phys_reg()) {
 525     assert(src.is_single_reg(),  "not a stack pair");
 526     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 527   } else {
 528     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 529     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 530     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 531   }
 532 }
 533 
 534 // A double move
 535 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 536 
 537   // The calling conventions assures us that each VMregpair is either
 538   // all really one physical register or adjacent stack slots.
 539 
 540   if (src.is_single_phys_reg() ) {
 541     if (dst.is_single_phys_reg()) {
 542       // In theory these overlap but the ordering is such that this is likely a nop
 543       if ( src.first() != dst.first()) {
 544         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
 545       }
 546     } else {
 547       assert(dst.is_single_reg(), "not a stack pair");
 548       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
 549     }
 550   } else if (dst.is_single_phys_reg()) {
 551     assert(src.is_single_reg(),  "not a stack pair");
 552     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 553   } else {
 554     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
 555     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 556     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 557   }
 558 }
 559 
 560 
 561 // A float arg may have to do float reg int reg conversion
 562 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 563   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
 564 
 565   // The calling conventions assures us that each VMregpair is either
 566   // all really one physical register or adjacent stack slots.
 567 
 568   if (src.first()->is_stack()) {
 569     if (dst.first()->is_stack()) {
 570       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 571       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 572     } else {
 573       // stack to reg
 574       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
 575       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 576     }
 577   } else if (dst.first()->is_stack()) {
 578     // reg to stack
 579     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
 580     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
 581   } else {
 582     // reg to reg
 583     // In theory these overlap but the ordering is such that this is likely a nop
 584     if ( src.first() != dst.first()) {
 585       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
 586     }
 587   }
 588 }
 589 
 590 // On 64 bit we will store integer like items to the stack as
 591 // 64 bits items (x86_32/64 abi) even though java would only store
 592 // 32bits for a parameter. On 32bit it will simply be 32 bits
 593 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
 594 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
 595   if (src.first()->is_stack()) {
 596     if (dst.first()->is_stack()) {
 597       // stack to stack
 598       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 599       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
 600     } else {
 601       // stack to reg
 602       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
 603     }
 604   } else if (dst.first()->is_stack()) {
 605     // reg to stack
 606     // Do we really have to sign extend???
 607     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 608     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 609   } else {
 610     // Do we really have to sign extend???
 611     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 612     if (dst.first() != src.first()) {
 613       movq(dst.first()->as_Register(), src.first()->as_Register());
 614     }
 615   }
 616 }
 617 
 618 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 619   if (src.first()->is_stack()) {
 620     if (dst.first()->is_stack()) {
 621       // stack to stack
 622       movq(rax, Address(rbp, reg2offset_in(src.first())));
 623       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 624     } else {
 625       // stack to reg
 626       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 627     }
 628   } else if (dst.first()->is_stack()) {
 629     // reg to stack
 630     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 631   } else {
 632     if (dst.first() != src.first()) {
 633       movq(dst.first()->as_Register(), src.first()->as_Register());
 634     }
 635   }
 636 }
 637 
 638 // An oop arg. Must pass a handle not the oop itself
 639 void MacroAssembler::object_move(OopMap* map,
 640                         int oop_handle_offset,
 641                         int framesize_in_slots,
 642                         VMRegPair src,
 643                         VMRegPair dst,
 644                         bool is_receiver,
 645                         int* receiver_offset) {
 646 
 647   // must pass a handle. First figure out the location we use as a handle
 648 
 649   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 650 
 651   // See if oop is null if it is we need no handle
 652 
 653   if (src.first()->is_stack()) {
 654 
 655     // Oop is already on the stack as an argument
 656     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 657     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 658     if (is_receiver) {
 659       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 660     }
 661 
 662     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 663     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 664     // conditionally move a null
 665     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 666   } else {
 667 
 668     // Oop is in a register we must store it to the space we reserve
 669     // on the stack for oop_handles and pass a handle if oop is non-null
 670 
 671     const Register rOop = src.first()->as_Register();
 672     int oop_slot;
 673     if (rOop == j_rarg0)
 674       oop_slot = 0;
 675     else if (rOop == j_rarg1)
 676       oop_slot = 1;
 677     else if (rOop == j_rarg2)
 678       oop_slot = 2;
 679     else if (rOop == j_rarg3)
 680       oop_slot = 3;
 681     else if (rOop == j_rarg4)
 682       oop_slot = 4;
 683     else {
 684       assert(rOop == j_rarg5, "wrong register");
 685       oop_slot = 5;
 686     }
 687 
 688     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 689     int offset = oop_slot*VMRegImpl::stack_slot_size;
 690 
 691     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 692     // Store oop in handle area, may be null
 693     movptr(Address(rsp, offset), rOop);
 694     if (is_receiver) {
 695       *receiver_offset = offset;
 696     }
 697 
 698     cmpptr(rOop, NULL_WORD);
 699     lea(rHandle, Address(rsp, offset));
 700     // conditionally move a null from the handle area where it was just stored
 701     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 702   }
 703 
 704   // If arg is on the stack then place it otherwise it is already in correct reg.
 705   if (dst.first()->is_stack()) {
 706     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 707   }
 708 }
 709 
 710 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 711   addq(dst, imm32);
 712 }
 713 
 714 void MacroAssembler::addptr(Register dst, Register src) {
 715   addq(dst, src);
 716 }
 717 
 718 void MacroAssembler::addptr(Address dst, Register src) {
 719   addq(dst, src);
 720 }
 721 
 722 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 723   assert(rscratch != noreg || always_reachable(src), "missing");
 724 
 725   if (reachable(src)) {
 726     Assembler::addsd(dst, as_Address(src));
 727   } else {
 728     lea(rscratch, src);
 729     Assembler::addsd(dst, Address(rscratch, 0));
 730   }
 731 }
 732 
 733 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 734   assert(rscratch != noreg || always_reachable(src), "missing");
 735 
 736   if (reachable(src)) {
 737     addss(dst, as_Address(src));
 738   } else {
 739     lea(rscratch, src);
 740     addss(dst, Address(rscratch, 0));
 741   }
 742 }
 743 
 744 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 745   assert(rscratch != noreg || always_reachable(src), "missing");
 746 
 747   if (reachable(src)) {
 748     Assembler::addpd(dst, as_Address(src));
 749   } else {
 750     lea(rscratch, src);
 751     Assembler::addpd(dst, Address(rscratch, 0));
 752   }
 753 }
 754 
 755 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 756 // Stub code is generated once and never copied.
 757 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 758 void MacroAssembler::align64() {
 759   align(64, (uint)(uintptr_t)pc());
 760 }
 761 
 762 void MacroAssembler::align32() {
 763   align(32, (uint)(uintptr_t)pc());
 764 }
 765 
 766 void MacroAssembler::align(uint modulus) {
 767   // 8273459: Ensure alignment is possible with current segment alignment
 768   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 769   align(modulus, offset());
 770 }
 771 
 772 void MacroAssembler::align(uint modulus, uint target) {
 773   if (target % modulus != 0) {
 774     nop(modulus - (target % modulus));
 775   }
 776 }
 777 
 778 void MacroAssembler::push_f(XMMRegister r) {
 779   subptr(rsp, wordSize);
 780   movflt(Address(rsp, 0), r);
 781 }
 782 
 783 void MacroAssembler::pop_f(XMMRegister r) {
 784   movflt(r, Address(rsp, 0));
 785   addptr(rsp, wordSize);
 786 }
 787 
 788 void MacroAssembler::push_d(XMMRegister r) {
 789   subptr(rsp, 2 * wordSize);
 790   movdbl(Address(rsp, 0), r);
 791 }
 792 
 793 void MacroAssembler::pop_d(XMMRegister r) {
 794   movdbl(r, Address(rsp, 0));
 795   addptr(rsp, 2 * Interpreter::stackElementSize);
 796 }
 797 
 798 void MacroAssembler::push_ppx(Register src) {
 799   if (VM_Version::supports_apx_f()) {
 800     pushp(src);
 801   } else {
 802     Assembler::push(src);
 803   }
 804 }
 805 
 806 void MacroAssembler::pop_ppx(Register dst) {
 807   if (VM_Version::supports_apx_f()) {
 808     popp(dst);
 809   } else {
 810     Assembler::pop(dst);
 811   }
 812 }
 813 
 814 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 815   // Used in sign-masking with aligned address.
 816   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 817   assert(rscratch != noreg || always_reachable(src), "missing");
 818 
 819   if (UseAVX > 2 &&
 820       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 821       (dst->encoding() >= 16)) {
 822     vpand(dst, dst, src, AVX_512bit, rscratch);
 823   } else if (reachable(src)) {
 824     Assembler::andpd(dst, as_Address(src));
 825   } else {
 826     lea(rscratch, src);
 827     Assembler::andpd(dst, Address(rscratch, 0));
 828   }
 829 }
 830 
 831 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 832   // Used in sign-masking with aligned address.
 833   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 834   assert(rscratch != noreg || always_reachable(src), "missing");
 835 
 836   if (reachable(src)) {
 837     Assembler::andps(dst, as_Address(src));
 838   } else {
 839     lea(rscratch, src);
 840     Assembler::andps(dst, Address(rscratch, 0));
 841   }
 842 }
 843 
 844 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 845   andq(dst, imm32);
 846 }
 847 
 848 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 849   assert(rscratch != noreg || always_reachable(src), "missing");
 850 
 851   if (reachable(src)) {
 852     andq(dst, as_Address(src));
 853   } else {
 854     lea(rscratch, src);
 855     andq(dst, Address(rscratch, 0));
 856   }
 857 }
 858 
 859 void MacroAssembler::atomic_incl(Address counter_addr) {
 860   lock();
 861   incrementl(counter_addr);
 862 }
 863 
 864 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 865   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 866 
 867   if (reachable(counter_addr)) {
 868     atomic_incl(as_Address(counter_addr));
 869   } else {
 870     lea(rscratch, counter_addr);
 871     atomic_incl(Address(rscratch, 0));
 872   }
 873 }
 874 
 875 void MacroAssembler::atomic_incq(Address counter_addr) {
 876   lock();
 877   incrementq(counter_addr);
 878 }
 879 
 880 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 881   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 882 
 883   if (reachable(counter_addr)) {
 884     atomic_incq(as_Address(counter_addr));
 885   } else {
 886     lea(rscratch, counter_addr);
 887     atomic_incq(Address(rscratch, 0));
 888   }
 889 }
 890 
 891 // Writes to stack successive pages until offset reached to check for
 892 // stack overflow + shadow pages.  This clobbers tmp.
 893 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 894   movptr(tmp, rsp);
 895   // Bang stack for total size given plus shadow page size.
 896   // Bang one page at a time because large size can bang beyond yellow and
 897   // red zones.
 898   Label loop;
 899   bind(loop);
 900   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 901   subptr(tmp, (int)os::vm_page_size());
 902   subl(size, (int)os::vm_page_size());
 903   jcc(Assembler::greater, loop);
 904 
 905   // Bang down shadow pages too.
 906   // At this point, (tmp-0) is the last address touched, so don't
 907   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 908   // was post-decremented.)  Skip this address by starting at i=1, and
 909   // touch a few more pages below.  N.B.  It is important to touch all
 910   // the way down including all pages in the shadow zone.
 911   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 912     // this could be any sized move but this is can be a debugging crumb
 913     // so the bigger the better.
 914     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 915   }
 916 }
 917 
 918 void MacroAssembler::reserved_stack_check() {
 919   // testing if reserved zone needs to be enabled
 920   Label no_reserved_zone_enabling;
 921 
 922   cmpptr(rsp, Address(r15_thread, JavaThread::reserved_stack_activation_offset()));
 923   jcc(Assembler::below, no_reserved_zone_enabling);
 924 
 925   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), r15_thread);
 926   jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
 927   should_not_reach_here();
 928 
 929   bind(no_reserved_zone_enabling);
 930 }
 931 
 932 void MacroAssembler::c2bool(Register x) {
 933   // implements x == 0 ? 0 : 1
 934   // note: must only look at least-significant byte of x
 935   //       since C-style booleans are stored in one byte
 936   //       only! (was bug)
 937   andl(x, 0xFF);
 938   setb(Assembler::notZero, x);
 939 }
 940 
 941 // Wouldn't need if AddressLiteral version had new name
 942 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 943   Assembler::call(L, rtype);
 944 }
 945 
 946 void MacroAssembler::call(Register entry) {
 947   Assembler::call(entry);
 948 }
 949 
 950 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 951   assert(rscratch != noreg || always_reachable(entry), "missing");
 952 
 953   if (reachable(entry)) {
 954     Assembler::call_literal(entry.target(), entry.rspec());
 955   } else {
 956     lea(rscratch, entry);
 957     Assembler::call(rscratch);
 958   }
 959 }
 960 
 961 void MacroAssembler::ic_call(address entry, jint method_index) {
 962   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 963   // Needs full 64-bit immediate for later patching.
 964   mov64(rax, (int64_t)Universe::non_oop_word());
 965   call(AddressLiteral(entry, rh));
 966 }
 967 
 968 int MacroAssembler::ic_check_size() {
 969   return UseCompactObjectHeaders ? 17 : 14;
 970 }
 971 
 972 int MacroAssembler::ic_check(int end_alignment) {
 973   Register receiver = j_rarg0;
 974   Register data = rax;
 975   Register temp = rscratch1;
 976 
 977   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 978   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 979   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 980   // before the inline cache check here, and not after
 981   align(end_alignment, offset() + ic_check_size());
 982 
 983   int uep_offset = offset();
 984 
 985   if (UseCompactObjectHeaders) {
 986     load_narrow_klass_compact(temp, receiver);
 987     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 988   } else if (UseCompressedClassPointers) {
 989     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 990     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 991   } else {
 992     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 993     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 994   }
 995 
 996   // if inline cache check fails, then jump to runtime routine
 997   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 998   assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
 999 
1000   return uep_offset;
1001 }
1002 
1003 void MacroAssembler::emit_static_call_stub() {
1004   // Static stub relocation also tags the Method* in the code-stream.
1005   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
1006   // This is recognized as unresolved by relocs/nativeinst/ic code.
1007   jump(RuntimeAddress(pc()));
1008 }
1009 
1010 // Implementation of call_VM versions
1011 
1012 void MacroAssembler::call_VM(Register oop_result,
1013                              address entry_point,
1014                              bool check_exceptions) {
1015   Label C, E;
1016   call(C, relocInfo::none);
1017   jmp(E);
1018 
1019   bind(C);
1020   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
1021   ret(0);
1022 
1023   bind(E);
1024 }
1025 
1026 void MacroAssembler::call_VM(Register oop_result,
1027                              address entry_point,
1028                              Register arg_1,
1029                              bool check_exceptions) {
1030   Label C, E;
1031   call(C, relocInfo::none);
1032   jmp(E);
1033 
1034   bind(C);
1035   pass_arg1(this, arg_1);
1036   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
1037   ret(0);
1038 
1039   bind(E);
1040 }
1041 
1042 void MacroAssembler::call_VM(Register oop_result,
1043                              address entry_point,
1044                              Register arg_1,
1045                              Register arg_2,
1046                              bool check_exceptions) {
1047   Label C, E;
1048   call(C, relocInfo::none);
1049   jmp(E);
1050 
1051   bind(C);
1052 
1053   assert_different_registers(arg_1, c_rarg2);
1054 
1055   pass_arg2(this, arg_2);
1056   pass_arg1(this, arg_1);
1057   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
1058   ret(0);
1059 
1060   bind(E);
1061 }
1062 
1063 void MacroAssembler::call_VM(Register oop_result,
1064                              address entry_point,
1065                              Register arg_1,
1066                              Register arg_2,
1067                              Register arg_3,
1068                              bool check_exceptions) {
1069   Label C, E;
1070   call(C, relocInfo::none);
1071   jmp(E);
1072 
1073   bind(C);
1074 
1075   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1076   assert_different_registers(arg_2, c_rarg3);
1077   pass_arg3(this, arg_3);
1078   pass_arg2(this, arg_2);
1079   pass_arg1(this, arg_1);
1080   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
1081   ret(0);
1082 
1083   bind(E);
1084 }
1085 
1086 void MacroAssembler::call_VM(Register oop_result,
1087                              Register last_java_sp,
1088                              address entry_point,
1089                              int number_of_arguments,
1090                              bool check_exceptions) {
1091   call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1092 }
1093 
1094 void MacroAssembler::call_VM(Register oop_result,
1095                              Register last_java_sp,
1096                              address entry_point,
1097                              Register arg_1,
1098                              bool check_exceptions) {
1099   pass_arg1(this, arg_1);
1100   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1101 }
1102 
1103 void MacroAssembler::call_VM(Register oop_result,
1104                              Register last_java_sp,
1105                              address entry_point,
1106                              Register arg_1,
1107                              Register arg_2,
1108                              bool check_exceptions) {
1109 
1110   assert_different_registers(arg_1, c_rarg2);
1111   pass_arg2(this, arg_2);
1112   pass_arg1(this, arg_1);
1113   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1114 }
1115 
1116 void MacroAssembler::call_VM(Register oop_result,
1117                              Register last_java_sp,
1118                              address entry_point,
1119                              Register arg_1,
1120                              Register arg_2,
1121                              Register arg_3,
1122                              bool check_exceptions) {
1123   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1124   assert_different_registers(arg_2, c_rarg3);
1125   pass_arg3(this, arg_3);
1126   pass_arg2(this, arg_2);
1127   pass_arg1(this, arg_1);
1128   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1129 }
1130 
1131 void MacroAssembler::super_call_VM(Register oop_result,
1132                                    Register last_java_sp,
1133                                    address entry_point,
1134                                    int number_of_arguments,
1135                                    bool check_exceptions) {
1136   MacroAssembler::call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1137 }
1138 
1139 void MacroAssembler::super_call_VM(Register oop_result,
1140                                    Register last_java_sp,
1141                                    address entry_point,
1142                                    Register arg_1,
1143                                    bool check_exceptions) {
1144   pass_arg1(this, arg_1);
1145   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1146 }
1147 
1148 void MacroAssembler::super_call_VM(Register oop_result,
1149                                    Register last_java_sp,
1150                                    address entry_point,
1151                                    Register arg_1,
1152                                    Register arg_2,
1153                                    bool check_exceptions) {
1154 
1155   assert_different_registers(arg_1, c_rarg2);
1156   pass_arg2(this, arg_2);
1157   pass_arg1(this, arg_1);
1158   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1159 }
1160 
1161 void MacroAssembler::super_call_VM(Register oop_result,
1162                                    Register last_java_sp,
1163                                    address entry_point,
1164                                    Register arg_1,
1165                                    Register arg_2,
1166                                    Register arg_3,
1167                                    bool check_exceptions) {
1168   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1169   assert_different_registers(arg_2, c_rarg3);
1170   pass_arg3(this, arg_3);
1171   pass_arg2(this, arg_2);
1172   pass_arg1(this, arg_1);
1173   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1174 }
1175 
1176 void MacroAssembler::call_VM_base(Register oop_result,
1177                                   Register last_java_sp,
1178                                   address  entry_point,
1179                                   int      number_of_arguments,
1180                                   bool     check_exceptions) {
1181   Register java_thread = r15_thread;
1182 
1183   // determine last_java_sp register
1184   if (!last_java_sp->is_valid()) {
1185     last_java_sp = rsp;
1186   }
1187   // debugging support
1188   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
1189 #ifdef ASSERT
1190   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
1191   // r12 is the heapbase.
1192   if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
1193 #endif // ASSERT
1194 
1195   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
1196   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
1197 
1198   // push java thread (becomes first argument of C function)
1199 
1200   mov(c_rarg0, r15_thread);
1201 
1202   // set last Java frame before call
1203   assert(last_java_sp != rbp, "can't use ebp/rbp");
1204 
1205   // Only interpreter should have to set fp
1206   set_last_Java_frame(last_java_sp, rbp, nullptr, rscratch1);
1207 
1208   // do the call, remove parameters
1209   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1210 
1211 #ifdef ASSERT
1212   // Check that thread register is not clobbered.
1213   guarantee(java_thread != rax, "change this code");
1214   push(rax);
1215   { Label L;
1216     get_thread_slow(rax);
1217     cmpptr(java_thread, rax);
1218     jcc(Assembler::equal, L);
1219     STOP("MacroAssembler::call_VM_base: java_thread not callee saved?");
1220     bind(L);
1221   }
1222   pop(rax);
1223 #endif
1224 
1225   // reset last Java frame
1226   // Only interpreter should have to clear fp
1227   reset_last_Java_frame(true);
1228 
1229    // C++ interp handles this in the interpreter
1230   check_and_handle_popframe();
1231   check_and_handle_earlyret();
1232 
1233   if (check_exceptions) {
1234     // check for pending exceptions (java_thread is set upon return)
1235     cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
1236     // This used to conditionally jump to forward_exception however it is
1237     // possible if we relocate that the branch will not reach. So we must jump
1238     // around so we can always reach
1239 
1240     Label ok;
1241     jcc(Assembler::equal, ok);
1242     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1243     bind(ok);
1244   }
1245 
1246   // get oop result if there is one and reset the value in the thread
1247   if (oop_result->is_valid()) {
1248     get_vm_result_oop(oop_result);
1249   }
1250 }
1251 
1252 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
1253   // Calculate the value for last_Java_sp somewhat subtle.
1254   // call_VM does an intermediate call which places a return address on
1255   // the stack just under the stack pointer as the user finished with it.
1256   // This allows use to retrieve last_Java_pc from last_Java_sp[-1].
1257 
1258   // We've pushed one address, correct last_Java_sp
1259   lea(rax, Address(rsp, wordSize));
1260 
1261   call_VM_base(oop_result, rax, entry_point, number_of_arguments, check_exceptions);
1262 }
1263 
1264 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
1265 void MacroAssembler::call_VM_leaf0(address entry_point) {
1266   MacroAssembler::call_VM_leaf_base(entry_point, 0);
1267 }
1268 
1269 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1270   call_VM_leaf_base(entry_point, number_of_arguments);
1271 }
1272 
1273 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1274   pass_arg0(this, arg_0);
1275   call_VM_leaf(entry_point, 1);
1276 }
1277 
1278 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1279 
1280   assert_different_registers(arg_0, c_rarg1);
1281   pass_arg1(this, arg_1);
1282   pass_arg0(this, arg_0);
1283   call_VM_leaf(entry_point, 2);
1284 }
1285 
1286 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1287   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1288   assert_different_registers(arg_1, c_rarg2);
1289   pass_arg2(this, arg_2);
1290   pass_arg1(this, arg_1);
1291   pass_arg0(this, arg_0);
1292   call_VM_leaf(entry_point, 3);
1293 }
1294 
1295 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1296   assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3);
1297   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1298   assert_different_registers(arg_2, c_rarg3);
1299   pass_arg3(this, arg_3);
1300   pass_arg2(this, arg_2);
1301   pass_arg1(this, arg_1);
1302   pass_arg0(this, arg_0);
1303   call_VM_leaf(entry_point, 3);
1304 }
1305 




1306 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1307   pass_arg0(this, arg_0);
1308   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1309 }
1310 
1311 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1312   assert_different_registers(arg_0, c_rarg1);
1313   pass_arg1(this, arg_1);
1314   pass_arg0(this, arg_0);
1315   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1316 }
1317 
1318 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1319   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1320   assert_different_registers(arg_1, c_rarg2);
1321   pass_arg2(this, arg_2);
1322   pass_arg1(this, arg_1);
1323   pass_arg0(this, arg_0);
1324   MacroAssembler::call_VM_leaf_base(entry_point, 3);
1325 }
1326 
1327 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1328   assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3);
1329   assert_different_registers(arg_1, c_rarg2, c_rarg3);
1330   assert_different_registers(arg_2, c_rarg3);
1331   pass_arg3(this, arg_3);
1332   pass_arg2(this, arg_2);
1333   pass_arg1(this, arg_1);
1334   pass_arg0(this, arg_0);
1335   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1336 }
1337 
1338 void MacroAssembler::get_vm_result_oop(Register oop_result) {
1339   movptr(oop_result, Address(r15_thread, JavaThread::vm_result_oop_offset()));
1340   movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
1341   verify_oop_msg(oop_result, "broken oop in call_VM_base");
1342 }
1343 
1344 void MacroAssembler::get_vm_result_metadata(Register metadata_result) {
1345   movptr(metadata_result, Address(r15_thread, JavaThread::vm_result_metadata_offset()));
1346   movptr(Address(r15_thread, JavaThread::vm_result_metadata_offset()), NULL_WORD);
1347 }
1348 
1349 void MacroAssembler::check_and_handle_earlyret() {
1350 }
1351 
1352 void MacroAssembler::check_and_handle_popframe() {
1353 }
1354 
1355 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
1356   assert(rscratch != noreg || always_reachable(src1), "missing");
1357 
1358   if (reachable(src1)) {
1359     cmpl(as_Address(src1), imm);
1360   } else {
1361     lea(rscratch, src1);
1362     cmpl(Address(rscratch, 0), imm);
1363   }
1364 }
1365 
1366 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
1367   assert(!src2.is_lval(), "use cmpptr");
1368   assert(rscratch != noreg || always_reachable(src2), "missing");
1369 
1370   if (reachable(src2)) {
1371     cmpl(src1, as_Address(src2));
1372   } else {
1373     lea(rscratch, src2);
1374     cmpl(src1, Address(rscratch, 0));
1375   }
1376 }
1377 
1378 void MacroAssembler::cmp32(Register src1, int32_t imm) {
1379   Assembler::cmpl(src1, imm);
1380 }
1381 
1382 void MacroAssembler::cmp32(Register src1, Address src2) {
1383   Assembler::cmpl(src1, src2);
1384 }
1385 
1386 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1387   ucomisd(opr1, opr2);
1388 
1389   Label L;
1390   if (unordered_is_less) {
1391     movl(dst, -1);
1392     jcc(Assembler::parity, L);
1393     jcc(Assembler::below , L);
1394     movl(dst, 0);
1395     jcc(Assembler::equal , L);
1396     increment(dst);
1397   } else { // unordered is greater
1398     movl(dst, 1);
1399     jcc(Assembler::parity, L);
1400     jcc(Assembler::above , L);
1401     movl(dst, 0);
1402     jcc(Assembler::equal , L);
1403     decrementl(dst);
1404   }
1405   bind(L);
1406 }
1407 
1408 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
1409   ucomiss(opr1, opr2);
1410 
1411   Label L;
1412   if (unordered_is_less) {
1413     movl(dst, -1);
1414     jcc(Assembler::parity, L);
1415     jcc(Assembler::below , L);
1416     movl(dst, 0);
1417     jcc(Assembler::equal , L);
1418     increment(dst);
1419   } else { // unordered is greater
1420     movl(dst, 1);
1421     jcc(Assembler::parity, L);
1422     jcc(Assembler::above , L);
1423     movl(dst, 0);
1424     jcc(Assembler::equal , L);
1425     decrementl(dst);
1426   }
1427   bind(L);
1428 }
1429 
1430 
1431 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
1432   assert(rscratch != noreg || always_reachable(src1), "missing");
1433 
1434   if (reachable(src1)) {
1435     cmpb(as_Address(src1), imm);
1436   } else {
1437     lea(rscratch, src1);
1438     cmpb(Address(rscratch, 0), imm);
1439   }
1440 }
1441 
1442 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
1443   assert(rscratch != noreg || always_reachable(src2), "missing");
1444 
1445   if (src2.is_lval()) {
1446     movptr(rscratch, src2);
1447     Assembler::cmpq(src1, rscratch);
1448   } else if (reachable(src2)) {
1449     cmpq(src1, as_Address(src2));
1450   } else {
1451     lea(rscratch, src2);
1452     Assembler::cmpq(src1, Address(rscratch, 0));
1453   }
1454 }
1455 
1456 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
1457   assert(src2.is_lval(), "not a mem-mem compare");
1458   // moves src2's literal address
1459   movptr(rscratch, src2);
1460   Assembler::cmpq(src1, rscratch);
1461 }
1462 
1463 void MacroAssembler::cmpoop(Register src1, Register src2) {
1464   cmpptr(src1, src2);
1465 }
1466 
1467 void MacroAssembler::cmpoop(Register src1, Address src2) {
1468   cmpptr(src1, src2);
1469 }
1470 
1471 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
1472   movoop(rscratch, src2);
1473   cmpptr(src1, rscratch);
1474 }
1475 
1476 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
1477   assert(rscratch != noreg || always_reachable(adr), "missing");
1478 
1479   if (reachable(adr)) {
1480     lock();
1481     cmpxchgptr(reg, as_Address(adr));
1482   } else {
1483     lea(rscratch, adr);
1484     lock();
1485     cmpxchgptr(reg, Address(rscratch, 0));
1486   }
1487 }
1488 
1489 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
1490   cmpxchgq(reg, adr);
1491 }
1492 
1493 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1494   assert(rscratch != noreg || always_reachable(src), "missing");
1495 
1496   if (reachable(src)) {
1497     Assembler::comisd(dst, as_Address(src));
1498   } else {
1499     lea(rscratch, src);
1500     Assembler::comisd(dst, Address(rscratch, 0));
1501   }
1502 }
1503 
1504 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
1505   assert(rscratch != noreg || always_reachable(src), "missing");
1506 
1507   if (reachable(src)) {
1508     Assembler::comiss(dst, as_Address(src));
1509   } else {
1510     lea(rscratch, src);
1511     Assembler::comiss(dst, Address(rscratch, 0));
1512   }
1513 }
1514 
1515 
1516 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
1517   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
1518 
1519   Condition negated_cond = negate_condition(cond);
1520   Label L;
1521   jcc(negated_cond, L);
1522   pushf(); // Preserve flags
1523   atomic_incl(counter_addr, rscratch);
1524   popf();
1525   bind(L);
1526 }
1527 
1528 int MacroAssembler::corrected_idivl(Register reg) {
1529   // Full implementation of Java idiv and irem; checks for
1530   // special case as described in JVM spec., p.243 & p.271.
1531   // The function returns the (pc) offset of the idivl
1532   // instruction - may be needed for implicit exceptions.
1533   //
1534   //         normal case                           special case
1535   //
1536   // input : rax,: dividend                         min_int
1537   //         reg: divisor   (may not be rax,/rdx)   -1
1538   //
1539   // output: rax,: quotient  (= rax, idiv reg)       min_int
1540   //         rdx: remainder (= rax, irem reg)       0
1541   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
1542   const int min_int = 0x80000000;
1543   Label normal_case, special_case;
1544 
1545   // check for special case
1546   cmpl(rax, min_int);
1547   jcc(Assembler::notEqual, normal_case);
1548   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
1549   cmpl(reg, -1);
1550   jcc(Assembler::equal, special_case);
1551 
1552   // handle normal case
1553   bind(normal_case);
1554   cdql();
1555   int idivl_offset = offset();
1556   idivl(reg);
1557 
1558   // normal and special case exit
1559   bind(special_case);
1560 
1561   return idivl_offset;
1562 }
1563 
1564 
1565 
1566 void MacroAssembler::decrementl(Register reg, int value) {
1567   if (value == min_jint) {subl(reg, value) ; return; }
1568   if (value <  0) { incrementl(reg, -value); return; }
1569   if (value == 0) {                        ; return; }
1570   if (value == 1 && UseIncDec) { decl(reg) ; return; }
1571   /* else */      { subl(reg, value)       ; return; }
1572 }
1573 
1574 void MacroAssembler::decrementl(Address dst, int value) {
1575   if (value == min_jint) {subl(dst, value) ; return; }
1576   if (value <  0) { incrementl(dst, -value); return; }
1577   if (value == 0) {                        ; return; }
1578   if (value == 1 && UseIncDec) { decl(dst) ; return; }
1579   /* else */      { subl(dst, value)       ; return; }
1580 }
1581 
1582 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
1583   assert(shift_value > 0, "illegal shift value");
1584   Label _is_positive;
1585   testl (reg, reg);
1586   jcc (Assembler::positive, _is_positive);
1587   int offset = (1 << shift_value) - 1 ;
1588 
1589   if (offset == 1) {
1590     incrementl(reg);
1591   } else {
1592     addl(reg, offset);
1593   }
1594 
1595   bind (_is_positive);
1596   sarl(reg, shift_value);
1597 }
1598 
1599 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1600   assert(rscratch != noreg || always_reachable(src), "missing");
1601 
1602   if (reachable(src)) {
1603     Assembler::divsd(dst, as_Address(src));
1604   } else {
1605     lea(rscratch, src);
1606     Assembler::divsd(dst, Address(rscratch, 0));
1607   }
1608 }
1609 
1610 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
1611   assert(rscratch != noreg || always_reachable(src), "missing");
1612 
1613   if (reachable(src)) {
1614     Assembler::divss(dst, as_Address(src));
1615   } else {
1616     lea(rscratch, src);
1617     Assembler::divss(dst, Address(rscratch, 0));
1618   }
1619 }
1620 
1621 void MacroAssembler::enter() {
1622   push(rbp);
1623   mov(rbp, rsp);
1624 }
1625 
1626 void MacroAssembler::post_call_nop() {
1627   if (!Continuations::enabled()) {
1628     return;
1629   }
1630   InstructionMark im(this);
1631   relocate(post_call_nop_Relocation::spec());
1632   InlineSkippedInstructionsCounter skipCounter(this);
1633   emit_int8((uint8_t)0x0f);
1634   emit_int8((uint8_t)0x1f);
1635   emit_int8((uint8_t)0x84);
1636   emit_int8((uint8_t)0x00);
1637   emit_int32(0x00);
1638 }
1639 
1640 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
1641   assert(rscratch != noreg || always_reachable(src), "missing");
1642   if (reachable(src)) {
1643     Assembler::mulpd(dst, as_Address(src));
1644   } else {
1645     lea(rscratch, src);
1646     Assembler::mulpd(dst, Address(rscratch, 0));
1647   }
1648 }
1649 
1650 // dst = c = a * b + c
1651 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
1652   Assembler::vfmadd231sd(c, a, b);
1653   if (dst != c) {
1654     movdbl(dst, c);
1655   }
1656 }
1657 
1658 // dst = c = a * b + c
1659 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
1660   Assembler::vfmadd231ss(c, a, b);
1661   if (dst != c) {
1662     movflt(dst, c);
1663   }
1664 }
1665 
1666 // dst = c = a * b + c
1667 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
1668   Assembler::vfmadd231pd(c, a, b, vector_len);
1669   if (dst != c) {
1670     vmovdqu(dst, c);
1671   }
1672 }
1673 
1674 // dst = c = a * b + c
1675 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
1676   Assembler::vfmadd231ps(c, a, b, vector_len);
1677   if (dst != c) {
1678     vmovdqu(dst, c);
1679   }
1680 }
1681 
1682 // dst = c = a * b + c
1683 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
1684   Assembler::vfmadd231pd(c, a, b, vector_len);
1685   if (dst != c) {
1686     vmovdqu(dst, c);
1687   }
1688 }
1689 
1690 // dst = c = a * b + c
1691 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
1692   Assembler::vfmadd231ps(c, a, b, vector_len);
1693   if (dst != c) {
1694     vmovdqu(dst, c);
1695   }
1696 }
1697 
1698 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
1699   assert(rscratch != noreg || always_reachable(dst), "missing");
1700 
1701   if (reachable(dst)) {
1702     incrementl(as_Address(dst));
1703   } else {
1704     lea(rscratch, dst);
1705     incrementl(Address(rscratch, 0));
1706   }
1707 }
1708 
1709 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
1710   incrementl(as_Address(dst, rscratch));
1711 }
1712 
1713 void MacroAssembler::incrementl(Register reg, int value) {
1714   if (value == min_jint) {addl(reg, value) ; return; }
1715   if (value <  0) { decrementl(reg, -value); return; }
1716   if (value == 0) {                        ; return; }
1717   if (value == 1 && UseIncDec) { incl(reg) ; return; }
1718   /* else */      { addl(reg, value)       ; return; }
1719 }
1720 
1721 void MacroAssembler::incrementl(Address dst, int value) {
1722   if (value == min_jint) {addl(dst, value) ; return; }
1723   if (value <  0) { decrementl(dst, -value); return; }
1724   if (value == 0) {                        ; return; }
1725   if (value == 1 && UseIncDec) { incl(dst) ; return; }
1726   /* else */      { addl(dst, value)       ; return; }
1727 }
1728 
1729 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
1730   assert(rscratch != noreg || always_reachable(dst), "missing");
1731   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump");
1732   if (reachable(dst)) {
1733     jmp_literal(dst.target(), dst.rspec());
1734   } else {
1735     lea(rscratch, dst);
1736     jmp(rscratch);
1737   }
1738 }
1739 
1740 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
1741   assert(rscratch != noreg || always_reachable(dst), "missing");
1742   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc");
1743   if (reachable(dst)) {
1744     InstructionMark im(this);
1745     relocate(dst.reloc());
1746     const int short_size = 2;
1747     const int long_size = 6;
1748     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
1749     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
1750       // 0111 tttn #8-bit disp
1751       emit_int8(0x70 | cc);
1752       emit_int8((offs - short_size) & 0xFF);
1753     } else {
1754       // 0000 1111 1000 tttn #32-bit disp
1755       emit_int8(0x0F);
1756       emit_int8((unsigned char)(0x80 | cc));
1757       emit_int32(offs - long_size);
1758     }
1759   } else {
1760 #ifdef ASSERT
1761     warning("reversing conditional branch");
1762 #endif /* ASSERT */
1763     Label skip;
1764     jccb(reverse[cc], skip);
1765     lea(rscratch, dst);
1766     Assembler::jmp(rscratch);
1767     bind(skip);
1768   }
1769 }
1770 
1771 void MacroAssembler::cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch) {
1772   ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
1773   assert(rscratch != noreg || always_reachable(mxcsr_std), "missing");
1774 
1775   stmxcsr(mxcsr_save);
1776   movl(tmp, mxcsr_save);
1777   if (EnableX86ECoreOpts) {
1778     // The mxcsr_std has status bits set for performance on ECore
1779     orl(tmp, 0x003f);
1780   } else {
1781     // Mask out status bits (only check control and mask bits)
1782     andl(tmp, 0xFFC0);
1783   }
1784   cmp32(tmp, mxcsr_std, rscratch);
1785 }
1786 
1787 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
1788   assert(rscratch != noreg || always_reachable(src), "missing");
1789 
1790   if (reachable(src)) {
1791     Assembler::ldmxcsr(as_Address(src));
1792   } else {
1793     lea(rscratch, src);
1794     Assembler::ldmxcsr(Address(rscratch, 0));
1795   }
1796 }
1797 
1798 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1799   int off = offset();
1800   movsbl(dst, src); // movsxb
1801   return off;
1802 }
1803 
1804 // Note: load_signed_short used to be called load_signed_word.
1805 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
1806 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
1807 // The term "word" in HotSpot means a 32- or 64-bit machine word.
1808 int MacroAssembler::load_signed_short(Register dst, Address src) {
1809   // This is dubious to me since it seems safe to do a signed 16 => 64 bit
1810   // version but this is what 64bit has always done. This seems to imply
1811   // that users are only using 32bits worth.
1812   int off = offset();
1813   movswl(dst, src); // movsxw
1814   return off;
1815 }
1816 
1817 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1818   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
1819   // and "3.9 Partial Register Penalties", p. 22).
1820   int off = offset();
1821   movzbl(dst, src); // movzxb
1822   return off;
1823 }
1824 
1825 // Note: load_unsigned_short used to be called load_unsigned_word.
1826 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1827   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
1828   // and "3.9 Partial Register Penalties", p. 22).
1829   int off = offset();
1830   movzwl(dst, src); // movzxw
1831   return off;
1832 }
1833 
1834 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
1835   switch (size_in_bytes) {
1836   case  8:  movq(dst, src); break;
1837   case  4:  movl(dst, src); break;
1838   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1839   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1840   default:  ShouldNotReachHere();
1841   }
1842 }
1843 
1844 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
1845   switch (size_in_bytes) {
1846   case  8:  movq(dst, src); break;
1847   case  4:  movl(dst, src); break;
1848   case  2:  movw(dst, src); break;
1849   case  1:  movb(dst, src); break;
1850   default:  ShouldNotReachHere();
1851   }
1852 }
1853 
1854 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
1855   assert(rscratch != noreg || always_reachable(dst), "missing");
1856 
1857   if (reachable(dst)) {
1858     movl(as_Address(dst), src);
1859   } else {
1860     lea(rscratch, dst);
1861     movl(Address(rscratch, 0), src);
1862   }
1863 }
1864 
1865 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
1866   if (reachable(src)) {
1867     movl(dst, as_Address(src));
1868   } else {
1869     lea(dst, src);
1870     movl(dst, Address(dst, 0));
1871   }
1872 }
1873 
1874 // C++ bool manipulation
1875 
1876 void MacroAssembler::movbool(Register dst, Address src) {
1877   if(sizeof(bool) == 1)
1878     movb(dst, src);
1879   else if(sizeof(bool) == 2)
1880     movw(dst, src);
1881   else if(sizeof(bool) == 4)
1882     movl(dst, src);
1883   else
1884     // unsupported
1885     ShouldNotReachHere();
1886 }
1887 
1888 void MacroAssembler::movbool(Address dst, bool boolconst) {
1889   if(sizeof(bool) == 1)
1890     movb(dst, (int) boolconst);
1891   else if(sizeof(bool) == 2)
1892     movw(dst, (int) boolconst);
1893   else if(sizeof(bool) == 4)
1894     movl(dst, (int) boolconst);
1895   else
1896     // unsupported
1897     ShouldNotReachHere();
1898 }
1899 
1900 void MacroAssembler::movbool(Address dst, Register src) {
1901   if(sizeof(bool) == 1)
1902     movb(dst, src);
1903   else if(sizeof(bool) == 2)
1904     movw(dst, src);
1905   else if(sizeof(bool) == 4)
1906     movl(dst, src);
1907   else
1908     // unsupported
1909     ShouldNotReachHere();
1910 }
1911 
1912 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
1913   assert(rscratch != noreg || always_reachable(src), "missing");
1914 
1915   if (reachable(src)) {
1916     movdl(dst, as_Address(src));
1917   } else {
1918     lea(rscratch, src);
1919     movdl(dst, Address(rscratch, 0));
1920   }
1921 }
1922 
1923 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
1924   assert(rscratch != noreg || always_reachable(src), "missing");
1925 
1926   if (reachable(src)) {
1927     movq(dst, as_Address(src));
1928   } else {
1929     lea(rscratch, src);
1930     movq(dst, Address(rscratch, 0));
1931   }
1932 }
1933 
1934 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
1935   assert(rscratch != noreg || always_reachable(src), "missing");
1936 
1937   if (reachable(src)) {
1938     if (UseXmmLoadAndClearUpper) {
1939       movsd (dst, as_Address(src));
1940     } else {
1941       movlpd(dst, as_Address(src));
1942     }
1943   } else {
1944     lea(rscratch, src);
1945     if (UseXmmLoadAndClearUpper) {
1946       movsd (dst, Address(rscratch, 0));
1947     } else {
1948       movlpd(dst, Address(rscratch, 0));
1949     }
1950   }
1951 }
1952 
1953 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
1954   assert(rscratch != noreg || always_reachable(src), "missing");
1955 
1956   if (reachable(src)) {
1957     movss(dst, as_Address(src));
1958   } else {
1959     lea(rscratch, src);
1960     movss(dst, Address(rscratch, 0));
1961   }
1962 }
1963 
1964 void MacroAssembler::movptr(Register dst, Register src) {
1965   movq(dst, src);
1966 }
1967 
1968 void MacroAssembler::movptr(Register dst, Address src) {
1969   movq(dst, src);
1970 }
1971 
1972 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
1973 void MacroAssembler::movptr(Register dst, intptr_t src) {
1974   if (is_uimm32(src)) {
1975     movl(dst, checked_cast<uint32_t>(src));
1976   } else if (is_simm32(src)) {
1977     movq(dst, checked_cast<int32_t>(src));
1978   } else {
1979     mov64(dst, src);
1980   }
1981 }
1982 
1983 void MacroAssembler::movptr(Address dst, Register src) {
1984   movq(dst, src);
1985 }
1986 
1987 void MacroAssembler::movptr(Address dst, int32_t src) {
1988   movslq(dst, src);
1989 }
1990 
1991 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
1992   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
1993   Assembler::movdqu(dst, src);
1994 }
1995 
1996 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
1997   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
1998   Assembler::movdqu(dst, src);
1999 }
2000 
2001 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
2002   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2003   Assembler::movdqu(dst, src);
2004 }
2005 
2006 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
2007   assert(rscratch != noreg || always_reachable(src), "missing");
2008 
2009   if (reachable(src)) {
2010     movdqu(dst, as_Address(src));
2011   } else {
2012     lea(rscratch, src);
2013     movdqu(dst, Address(rscratch, 0));
2014   }
2015 }
2016 
2017 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
2018   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2019   Assembler::vmovdqu(dst, src);
2020 }
2021 
2022 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
2023   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2024   Assembler::vmovdqu(dst, src);
2025 }
2026 
2027 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2028   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
2029   Assembler::vmovdqu(dst, src);
2030 }
2031 
2032 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
2033   assert(rscratch != noreg || always_reachable(src), "missing");
2034 
2035   if (reachable(src)) {
2036     vmovdqu(dst, as_Address(src));
2037   }
2038   else {
2039     lea(rscratch, src);
2040     vmovdqu(dst, Address(rscratch, 0));
2041   }
2042 }
2043 
2044 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2045   assert(rscratch != noreg || always_reachable(src), "missing");
2046 
2047   if (vector_len == AVX_512bit) {
2048     evmovdquq(dst, src, AVX_512bit, rscratch);
2049   } else if (vector_len == AVX_256bit) {
2050     vmovdqu(dst, src, rscratch);
2051   } else {
2052     movdqu(dst, src, rscratch);
2053   }
2054 }
2055 
2056 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src, int vector_len) {
2057   if (vector_len == AVX_512bit) {
2058     evmovdquq(dst, src, AVX_512bit);
2059   } else if (vector_len == AVX_256bit) {
2060     vmovdqu(dst, src);
2061   } else {
2062     movdqu(dst, src);
2063   }
2064 }
2065 
2066 void MacroAssembler::vmovdqu(Address dst, XMMRegister src, int vector_len) {
2067   if (vector_len == AVX_512bit) {
2068     evmovdquq(dst, src, AVX_512bit);
2069   } else if (vector_len == AVX_256bit) {
2070     vmovdqu(dst, src);
2071   } else {
2072     movdqu(dst, src);
2073   }
2074 }
2075 
2076 void MacroAssembler::vmovdqu(XMMRegister dst, Address src, int vector_len) {
2077   if (vector_len == AVX_512bit) {
2078     evmovdquq(dst, src, AVX_512bit);
2079   } else if (vector_len == AVX_256bit) {
2080     vmovdqu(dst, src);
2081   } else {
2082     movdqu(dst, src);
2083   }
2084 }
2085 
2086 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
2087   assert(rscratch != noreg || always_reachable(src), "missing");
2088 
2089   if (reachable(src)) {
2090     vmovdqa(dst, as_Address(src));
2091   }
2092   else {
2093     lea(rscratch, src);
2094     vmovdqa(dst, Address(rscratch, 0));
2095   }
2096 }
2097 
2098 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2099   assert(rscratch != noreg || always_reachable(src), "missing");
2100 
2101   if (vector_len == AVX_512bit) {
2102     evmovdqaq(dst, src, AVX_512bit, rscratch);
2103   } else if (vector_len == AVX_256bit) {
2104     vmovdqa(dst, src, rscratch);
2105   } else {
2106     movdqa(dst, src, rscratch);
2107   }
2108 }
2109 
2110 void MacroAssembler::kmov(KRegister dst, Address src) {
2111   if (VM_Version::supports_avx512bw()) {
2112     kmovql(dst, src);
2113   } else {
2114     assert(VM_Version::supports_evex(), "");
2115     kmovwl(dst, src);
2116   }
2117 }
2118 
2119 void MacroAssembler::kmov(Address dst, KRegister src) {
2120   if (VM_Version::supports_avx512bw()) {
2121     kmovql(dst, src);
2122   } else {
2123     assert(VM_Version::supports_evex(), "");
2124     kmovwl(dst, src);
2125   }
2126 }
2127 
2128 void MacroAssembler::kmov(KRegister dst, KRegister src) {
2129   if (VM_Version::supports_avx512bw()) {
2130     kmovql(dst, src);
2131   } else {
2132     assert(VM_Version::supports_evex(), "");
2133     kmovwl(dst, src);
2134   }
2135 }
2136 
2137 void MacroAssembler::kmov(Register dst, KRegister src) {
2138   if (VM_Version::supports_avx512bw()) {
2139     kmovql(dst, src);
2140   } else {
2141     assert(VM_Version::supports_evex(), "");
2142     kmovwl(dst, src);
2143   }
2144 }
2145 
2146 void MacroAssembler::kmov(KRegister dst, Register src) {
2147   if (VM_Version::supports_avx512bw()) {
2148     kmovql(dst, src);
2149   } else {
2150     assert(VM_Version::supports_evex(), "");
2151     kmovwl(dst, src);
2152   }
2153 }
2154 
2155 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
2156   assert(rscratch != noreg || always_reachable(src), "missing");
2157 
2158   if (reachable(src)) {
2159     kmovql(dst, as_Address(src));
2160   } else {
2161     lea(rscratch, src);
2162     kmovql(dst, Address(rscratch, 0));
2163   }
2164 }
2165 
2166 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
2167   assert(rscratch != noreg || always_reachable(src), "missing");
2168 
2169   if (reachable(src)) {
2170     kmovwl(dst, as_Address(src));
2171   } else {
2172     lea(rscratch, src);
2173     kmovwl(dst, Address(rscratch, 0));
2174   }
2175 }
2176 
2177 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2178                                int vector_len, Register rscratch) {
2179   assert(rscratch != noreg || always_reachable(src), "missing");
2180 
2181   if (reachable(src)) {
2182     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
2183   } else {
2184     lea(rscratch, src);
2185     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
2186   }
2187 }
2188 
2189 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
2190                                int vector_len, Register rscratch) {
2191   assert(rscratch != noreg || always_reachable(src), "missing");
2192 
2193   if (reachable(src)) {
2194     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
2195   } else {
2196     lea(rscratch, src);
2197     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
2198   }
2199 }
2200 
2201 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2202   assert(rscratch != noreg || always_reachable(src), "missing");
2203 
2204   if (reachable(src)) {
2205     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
2206   } else {
2207     lea(rscratch, src);
2208     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
2209   }
2210 }
2211 
2212 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2213   assert(rscratch != noreg || always_reachable(src), "missing");
2214 
2215   if (reachable(src)) {
2216     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
2217   } else {
2218     lea(rscratch, src);
2219     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
2220   }
2221 }
2222 
2223 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2224   assert(rscratch != noreg || always_reachable(src), "missing");
2225 
2226   if (reachable(src)) {
2227     Assembler::evmovdquq(dst, as_Address(src), vector_len);
2228   } else {
2229     lea(rscratch, src);
2230     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
2231   }
2232 }
2233 
2234 void MacroAssembler::evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
2235   assert(rscratch != noreg || always_reachable(src), "missing");
2236 
2237   if (reachable(src)) {
2238     Assembler::evmovdqaq(dst, mask, as_Address(src), merge, vector_len);
2239   } else {
2240     lea(rscratch, src);
2241     Assembler::evmovdqaq(dst, mask, Address(rscratch, 0), merge, vector_len);
2242   }
2243 }
2244 
2245 void MacroAssembler::evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2246   assert(rscratch != noreg || always_reachable(src), "missing");
2247 
2248   if (reachable(src)) {
2249     Assembler::evmovdqaq(dst, as_Address(src), vector_len);
2250   } else {
2251     lea(rscratch, src);
2252     Assembler::evmovdqaq(dst, Address(rscratch, 0), vector_len);
2253   }
2254 }
2255 
2256 void MacroAssembler::movapd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2257   assert(rscratch != noreg || always_reachable(src), "missing");
2258 
2259   if (reachable(src)) {
2260     Assembler::movapd(dst, as_Address(src));
2261   } else {
2262     lea(rscratch, src);
2263     Assembler::movapd(dst, Address(rscratch, 0));
2264   }
2265 }
2266 
2267 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
2268   assert(rscratch != noreg || always_reachable(src), "missing");
2269 
2270   if (reachable(src)) {
2271     Assembler::movdqa(dst, as_Address(src));
2272   } else {
2273     lea(rscratch, src);
2274     Assembler::movdqa(dst, Address(rscratch, 0));
2275   }
2276 }
2277 
2278 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2279   assert(rscratch != noreg || always_reachable(src), "missing");
2280 
2281   if (reachable(src)) {
2282     Assembler::movsd(dst, as_Address(src));
2283   } else {
2284     lea(rscratch, src);
2285     Assembler::movsd(dst, Address(rscratch, 0));
2286   }
2287 }
2288 
2289 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2290   assert(rscratch != noreg || always_reachable(src), "missing");
2291 
2292   if (reachable(src)) {
2293     Assembler::movss(dst, as_Address(src));
2294   } else {
2295     lea(rscratch, src);
2296     Assembler::movss(dst, Address(rscratch, 0));
2297   }
2298 }
2299 
2300 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
2301   assert(rscratch != noreg || always_reachable(src), "missing");
2302 
2303   if (reachable(src)) {
2304     Assembler::movddup(dst, as_Address(src));
2305   } else {
2306     lea(rscratch, src);
2307     Assembler::movddup(dst, Address(rscratch, 0));
2308   }
2309 }
2310 
2311 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2312   assert(rscratch != noreg || always_reachable(src), "missing");
2313 
2314   if (reachable(src)) {
2315     Assembler::vmovddup(dst, as_Address(src), vector_len);
2316   } else {
2317     lea(rscratch, src);
2318     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
2319   }
2320 }
2321 
2322 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2323   assert(rscratch != noreg || always_reachable(src), "missing");
2324 
2325   if (reachable(src)) {
2326     Assembler::mulsd(dst, as_Address(src));
2327   } else {
2328     lea(rscratch, src);
2329     Assembler::mulsd(dst, Address(rscratch, 0));
2330   }
2331 }
2332 
2333 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2334   assert(rscratch != noreg || always_reachable(src), "missing");
2335 
2336   if (reachable(src)) {
2337     Assembler::mulss(dst, as_Address(src));
2338   } else {
2339     lea(rscratch, src);
2340     Assembler::mulss(dst, Address(rscratch, 0));
2341   }
2342 }
2343 
2344 void MacroAssembler::null_check(Register reg, int offset) {
2345   if (needs_explicit_null_check(offset)) {
2346     // provoke OS null exception if reg is null by
2347     // accessing M[reg] w/o changing any (non-CC) registers
2348     // NOTE: cmpl is plenty here to provoke a segv
2349     cmpptr(rax, Address(reg, 0));
2350     // Note: should probably use testl(rax, Address(reg, 0));
2351     //       may be shorter code (however, this version of
2352     //       testl needs to be implemented first)
2353   } else {
2354     // nothing to do, (later) access of M[reg + offset]
2355     // will provoke OS null exception if reg is null
2356   }
2357 }
2358 











































































































2359 void MacroAssembler::os_breakpoint() {
2360   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
2361   // (e.g., MSVC can't call ps() otherwise)
2362   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
2363 }
2364 
2365 void MacroAssembler::unimplemented(const char* what) {
2366   const char* buf = nullptr;
2367   {
2368     ResourceMark rm;
2369     stringStream ss;
2370     ss.print("unimplemented: %s", what);
2371     buf = code_string(ss.as_string());
2372   }
2373   stop(buf);
2374 }
2375 
2376 #define XSTATE_BV 0x200
2377 
2378 void MacroAssembler::pop_CPU_state() {
2379   pop_FPU_state();
2380   pop_IU_state();
2381 }
2382 
2383 void MacroAssembler::pop_FPU_state() {
2384   fxrstor(Address(rsp, 0));
2385   addptr(rsp, FPUStateSizeInWords * wordSize);
2386 }
2387 
2388 void MacroAssembler::pop_IU_state() {
2389   popa();
2390   addq(rsp, 8);
2391   popf();
2392 }
2393 
2394 // Save Integer and Float state
2395 // Warning: Stack must be 16 byte aligned (64bit)
2396 void MacroAssembler::push_CPU_state() {
2397   push_IU_state();
2398   push_FPU_state();
2399 }
2400 
2401 void MacroAssembler::push_FPU_state() {
2402   subptr(rsp, FPUStateSizeInWords * wordSize);
2403   fxsave(Address(rsp, 0));
2404 }
2405 
2406 void MacroAssembler::push_IU_state() {
2407   // Push flags first because pusha kills them
2408   pushf();
2409   // Make sure rsp stays 16-byte aligned
2410   subq(rsp, 8);
2411   pusha();
2412 }
2413 
2414 void MacroAssembler::push_cont_fastpath() {
2415   if (!Continuations::enabled()) return;
2416 
2417   Label L_done;
2418   cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset()));
2419   jccb(Assembler::belowEqual, L_done);
2420   movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rsp);
2421   bind(L_done);
2422 }
2423 
2424 void MacroAssembler::pop_cont_fastpath() {
2425   if (!Continuations::enabled()) return;
2426 
2427   Label L_done;
2428   cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset()));
2429   jccb(Assembler::below, L_done);
2430   movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
2431   bind(L_done);
2432 }
2433 
2434 #ifdef ASSERT
2435 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
2436   Label no_cont;
2437   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
2438   testl(cont, cont);
2439   jcc(Assembler::zero, no_cont);
2440   stop(name);
2441   bind(no_cont);
2442 }
2443 #endif
2444 
2445 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { // determine java_thread register
2446   // we must set sp to zero to clear frame
2447   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
2448   // must clear fp, so that compiled frames are not confused; it is
2449   // possible that we need it only for debugging
2450   if (clear_fp) {
2451     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
2452   }
2453   // Always clear the pc because it could have been set by make_walkable()
2454   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
2455   vzeroupper();
2456 }
2457 
2458 void MacroAssembler::round_to(Register reg, int modulus) {
2459   addptr(reg, modulus - 1);
2460   andptr(reg, -modulus);
2461 }
2462 
2463 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod) {
2464   if (at_return) {
2465     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
2466     // we may safely use rsp instead to perform the stack watermark check.
2467     cmpptr(in_nmethod ? rsp : rbp, Address(r15_thread, JavaThread::polling_word_offset()));
2468     jcc(Assembler::above, slow_path);
2469     return;
2470   }
2471   testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2472   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
2473 }
2474 
2475 // Calls to C land
2476 //
2477 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
2478 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
2479 // has to be reset to 0. This is required to allow proper stack traversal.
2480 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
2481                                          Register last_java_fp,
2482                                          address  last_java_pc,
2483                                          Register rscratch) {
2484   vzeroupper();
2485   // determine last_java_sp register
2486   if (!last_java_sp->is_valid()) {
2487     last_java_sp = rsp;
2488   }
2489   // last_java_fp is optional
2490   if (last_java_fp->is_valid()) {
2491     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
2492   }
2493   // last_java_pc is optional
2494   if (last_java_pc != nullptr) {
2495     Address java_pc(r15_thread,
2496                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
2497     lea(java_pc, InternalAddress(last_java_pc), rscratch);
2498   }
2499   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
2500 }
2501 
2502 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
2503                                          Register last_java_fp,
2504                                          Label &L,
2505                                          Register scratch) {
2506   lea(scratch, L);
2507   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch);
2508   set_last_Java_frame(last_java_sp, last_java_fp, nullptr, scratch);
2509 }
2510 
2511 void MacroAssembler::shlptr(Register dst, int imm8) {
2512   shlq(dst, imm8);
2513 }
2514 
2515 void MacroAssembler::shrptr(Register dst, int imm8) {
2516   shrq(dst, imm8);
2517 }
2518 
2519 void MacroAssembler::sign_extend_byte(Register reg) {
2520   movsbl(reg, reg); // movsxb
2521 }
2522 
2523 void MacroAssembler::sign_extend_short(Register reg) {
2524   movswl(reg, reg); // movsxw
2525 }
2526 
2527 void MacroAssembler::testl(Address dst, int32_t imm32) {
2528   if (imm32 >= 0 && is8bit(imm32)) {
2529     testb(dst, imm32);
2530   } else {
2531     Assembler::testl(dst, imm32);
2532   }
2533 }
2534 
2535 void MacroAssembler::testl(Register dst, int32_t imm32) {
2536   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
2537     testb(dst, imm32);
2538   } else {
2539     Assembler::testl(dst, imm32);
2540   }
2541 }
2542 
2543 void MacroAssembler::testl(Register dst, AddressLiteral src) {
2544   assert(always_reachable(src), "Address should be reachable");
2545   testl(dst, as_Address(src));
2546 }
2547 
2548 void MacroAssembler::testq(Address dst, int32_t imm32) {
2549   if (imm32 >= 0) {
2550     testl(dst, imm32);
2551   } else {
2552     Assembler::testq(dst, imm32);
2553   }
2554 }
2555 
2556 void MacroAssembler::testq(Register dst, int32_t imm32) {
2557   if (imm32 >= 0) {
2558     testl(dst, imm32);
2559   } else {
2560     Assembler::testq(dst, imm32);
2561   }
2562 }
2563 
2564 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
2565   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2566   Assembler::pcmpeqb(dst, src);
2567 }
2568 
2569 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
2570   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2571   Assembler::pcmpeqw(dst, src);
2572 }
2573 
2574 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2575   assert((dst->encoding() < 16),"XMM register should be 0-15");
2576   Assembler::pcmpestri(dst, src, imm8);
2577 }
2578 
2579 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2580   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2581   Assembler::pcmpestri(dst, src, imm8);
2582 }
2583 
2584 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2585   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2586   Assembler::pmovzxbw(dst, src);
2587 }
2588 
2589 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
2590   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2591   Assembler::pmovzxbw(dst, src);
2592 }
2593 
2594 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
2595   assert((src->encoding() < 16),"XMM register should be 0-15");
2596   Assembler::pmovmskb(dst, src);
2597 }
2598 
2599 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
2600   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
2601   Assembler::ptest(dst, src);
2602 }
2603 
2604 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2605   assert(rscratch != noreg || always_reachable(src), "missing");
2606 
2607   if (reachable(src)) {
2608     Assembler::sqrtss(dst, as_Address(src));
2609   } else {
2610     lea(rscratch, src);
2611     Assembler::sqrtss(dst, Address(rscratch, 0));
2612   }
2613 }
2614 
2615 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2616   assert(rscratch != noreg || always_reachable(src), "missing");
2617 
2618   if (reachable(src)) {
2619     Assembler::subsd(dst, as_Address(src));
2620   } else {
2621     lea(rscratch, src);
2622     Assembler::subsd(dst, Address(rscratch, 0));
2623   }
2624 }
2625 
2626 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
2627   assert(rscratch != noreg || always_reachable(src), "missing");
2628 
2629   if (reachable(src)) {
2630     Assembler::roundsd(dst, as_Address(src), rmode);
2631   } else {
2632     lea(rscratch, src);
2633     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
2634   }
2635 }
2636 
2637 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2638   assert(rscratch != noreg || always_reachable(src), "missing");
2639 
2640   if (reachable(src)) {
2641     Assembler::subss(dst, as_Address(src));
2642   } else {
2643     lea(rscratch, src);
2644     Assembler::subss(dst, Address(rscratch, 0));
2645   }
2646 }
2647 
2648 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2649   assert(rscratch != noreg || always_reachable(src), "missing");
2650 
2651   if (reachable(src)) {
2652     Assembler::ucomisd(dst, as_Address(src));
2653   } else {
2654     lea(rscratch, src);
2655     Assembler::ucomisd(dst, Address(rscratch, 0));
2656   }
2657 }
2658 
2659 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
2660   assert(rscratch != noreg || always_reachable(src), "missing");
2661 
2662   if (reachable(src)) {
2663     Assembler::ucomiss(dst, as_Address(src));
2664   } else {
2665     lea(rscratch, src);
2666     Assembler::ucomiss(dst, Address(rscratch, 0));
2667   }
2668 }
2669 
2670 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
2671   assert(rscratch != noreg || always_reachable(src), "missing");
2672 
2673   // Used in sign-bit flipping with aligned address.
2674   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
2675 
2676   if (UseAVX > 2 &&
2677       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2678       (dst->encoding() >= 16)) {
2679     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
2680   } else if (reachable(src)) {
2681     Assembler::xorpd(dst, as_Address(src));
2682   } else {
2683     lea(rscratch, src);
2684     Assembler::xorpd(dst, Address(rscratch, 0));
2685   }
2686 }
2687 
2688 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
2689   if (UseAVX > 2 &&
2690       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2691       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
2692     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
2693   } else {
2694     Assembler::xorpd(dst, src);
2695   }
2696 }
2697 
2698 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
2699   if (UseAVX > 2 &&
2700       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2701       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
2702     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
2703   } else {
2704     Assembler::xorps(dst, src);
2705   }
2706 }
2707 
2708 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
2709   assert(rscratch != noreg || always_reachable(src), "missing");
2710 
2711   // Used in sign-bit flipping with aligned address.
2712   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
2713 
2714   if (UseAVX > 2 &&
2715       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
2716       (dst->encoding() >= 16)) {
2717     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
2718   } else if (reachable(src)) {
2719     Assembler::xorps(dst, as_Address(src));
2720   } else {
2721     lea(rscratch, src);
2722     Assembler::xorps(dst, Address(rscratch, 0));
2723   }
2724 }
2725 
2726 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
2727   assert(rscratch != noreg || always_reachable(src), "missing");
2728 
2729   // Used in sign-bit flipping with aligned address.
2730   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
2731   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
2732   if (reachable(src)) {
2733     Assembler::pshufb(dst, as_Address(src));
2734   } else {
2735     lea(rscratch, src);
2736     Assembler::pshufb(dst, Address(rscratch, 0));
2737   }
2738 }
2739 
2740 // AVX 3-operands instructions
2741 
2742 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
2743   assert(rscratch != noreg || always_reachable(src), "missing");
2744 
2745   if (reachable(src)) {
2746     vaddsd(dst, nds, as_Address(src));
2747   } else {
2748     lea(rscratch, src);
2749     vaddsd(dst, nds, Address(rscratch, 0));
2750   }
2751 }
2752 
2753 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
2754   assert(rscratch != noreg || always_reachable(src), "missing");
2755 
2756   if (reachable(src)) {
2757     vaddss(dst, nds, as_Address(src));
2758   } else {
2759     lea(rscratch, src);
2760     vaddss(dst, nds, Address(rscratch, 0));
2761   }
2762 }
2763 
2764 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2765   assert(UseAVX > 0, "requires some form of AVX");
2766   assert(rscratch != noreg || always_reachable(src), "missing");
2767 
2768   if (reachable(src)) {
2769     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
2770   } else {
2771     lea(rscratch, src);
2772     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
2773   }
2774 }
2775 
2776 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2777   assert(UseAVX > 0, "requires some form of AVX");
2778   assert(rscratch != noreg || always_reachable(src), "missing");
2779 
2780   if (reachable(src)) {
2781     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
2782   } else {
2783     lea(rscratch, src);
2784     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
2785   }
2786 }
2787 
2788 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
2789   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
2790   assert(rscratch != noreg || always_reachable(negate_field), "missing");
2791 
2792   vandps(dst, nds, negate_field, vector_len, rscratch);
2793 }
2794 
2795 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
2796   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
2797   assert(rscratch != noreg || always_reachable(negate_field), "missing");
2798 
2799   vandpd(dst, nds, negate_field, vector_len, rscratch);
2800 }
2801 
2802 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2803   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2804   Assembler::vpaddb(dst, nds, src, vector_len);
2805 }
2806 
2807 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2808   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2809   Assembler::vpaddb(dst, nds, src, vector_len);
2810 }
2811 
2812 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2813   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2814   Assembler::vpaddw(dst, nds, src, vector_len);
2815 }
2816 
2817 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2818   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2819   Assembler::vpaddw(dst, nds, src, vector_len);
2820 }
2821 
2822 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2823   assert(rscratch != noreg || always_reachable(src), "missing");
2824 
2825   if (reachable(src)) {
2826     Assembler::vpand(dst, nds, as_Address(src), vector_len);
2827   } else {
2828     lea(rscratch, src);
2829     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
2830   }
2831 }
2832 
2833 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2834   assert(rscratch != noreg || always_reachable(src), "missing");
2835 
2836   if (reachable(src)) {
2837     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
2838   } else {
2839     lea(rscratch, src);
2840     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
2841   }
2842 }
2843 
2844 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2845   assert(rscratch != noreg || always_reachable(src), "missing");
2846 
2847   if (reachable(src)) {
2848     Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
2849   } else {
2850     lea(rscratch, src);
2851     Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
2852   }
2853 }
2854 
2855 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2856   assert(rscratch != noreg || always_reachable(src), "missing");
2857 
2858   if (reachable(src)) {
2859     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
2860   } else {
2861     lea(rscratch, src);
2862     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
2863   }
2864 }
2865 
2866 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2867   assert(rscratch != noreg || always_reachable(src), "missing");
2868 
2869   if (reachable(src)) {
2870     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
2871   } else {
2872     lea(rscratch, src);
2873     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
2874   }
2875 }
2876 
2877 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
2878   assert(rscratch != noreg || always_reachable(src), "missing");
2879 
2880   if (reachable(src)) {
2881     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
2882   } else {
2883     lea(rscratch, src);
2884     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
2885   }
2886 }
2887 
2888 // Vector float blend
2889 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
2890 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
2891   // WARN: Allow dst == (src1|src2), mask == scratch
2892   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
2893                          !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
2894   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
2895   bool dst_available = dst != mask && (dst != src1 || dst != src2);
2896   if (blend_emulation && scratch_available && dst_available) {
2897     if (compute_mask) {
2898       vpsrad(scratch, mask, 32, vector_len);
2899       mask = scratch;
2900     }
2901     if (dst == src1) {
2902       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
2903       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
2904     } else {
2905       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
2906       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
2907     }
2908     vpor(dst, dst, scratch, vector_len);
2909   } else {
2910     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
2911   }
2912 }
2913 
2914 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
2915 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
2916   // WARN: Allow dst == (src1|src2), mask == scratch
2917   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
2918                          !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
2919   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
2920   bool dst_available = dst != mask && (dst != src1 || dst != src2);
2921   if (blend_emulation && scratch_available && dst_available) {
2922     if (compute_mask) {
2923       vpxor(scratch, scratch, scratch, vector_len);
2924       vpcmpgtq(scratch, scratch, mask, vector_len);
2925       mask = scratch;
2926     }
2927     if (dst == src1) {
2928       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
2929       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
2930     } else {
2931       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
2932       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
2933     }
2934     vpor(dst, dst, scratch, vector_len);
2935   } else {
2936     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
2937   }
2938 }
2939 
2940 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2941   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2942   Assembler::vpcmpeqb(dst, nds, src, vector_len);
2943 }
2944 
2945 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
2946   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2947   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
2948 }
2949 
2950 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2951   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2952   Assembler::vpcmpeqw(dst, nds, src, vector_len);
2953 }
2954 
2955 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
2956   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
2957   Assembler::vpcmpeqw(dst, nds, src, vector_len);
2958 }
2959 
2960 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
2961   assert(rscratch != noreg || always_reachable(src), "missing");
2962 
2963   if (reachable(src)) {
2964     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
2965   } else {
2966     lea(rscratch, src);
2967     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
2968   }
2969 }
2970 
2971 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
2972                              int comparison, bool is_signed, int vector_len, Register rscratch) {
2973   assert(rscratch != noreg || always_reachable(src), "missing");
2974 
2975   if (reachable(src)) {
2976     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
2977   } else {
2978     lea(rscratch, src);
2979     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
2980   }
2981 }
2982 
2983 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
2984                              int comparison, bool is_signed, int vector_len, Register rscratch) {
2985   assert(rscratch != noreg || always_reachable(src), "missing");
2986 
2987   if (reachable(src)) {
2988     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
2989   } else {
2990     lea(rscratch, src);
2991     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
2992   }
2993 }
2994 
2995 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
2996                              int comparison, bool is_signed, int vector_len, Register rscratch) {
2997   assert(rscratch != noreg || always_reachable(src), "missing");
2998 
2999   if (reachable(src)) {
3000     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3001   } else {
3002     lea(rscratch, src);
3003     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
3004   }
3005 }
3006 
3007 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
3008                              int comparison, bool is_signed, int vector_len, Register rscratch) {
3009   assert(rscratch != noreg || always_reachable(src), "missing");
3010 
3011   if (reachable(src)) {
3012     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
3013   } else {
3014     lea(rscratch, src);
3015     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
3016   }
3017 }
3018 
3019 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
3020   if (width == Assembler::Q) {
3021     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
3022   } else {
3023     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
3024   }
3025 }
3026 
3027 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
3028   int eq_cond_enc = 0x29;
3029   int gt_cond_enc = 0x37;
3030   if (width != Assembler::Q) {
3031     eq_cond_enc = 0x74 + width;
3032     gt_cond_enc = 0x64 + width;
3033   }
3034   switch (cond) {
3035   case eq:
3036     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3037     break;
3038   case neq:
3039     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
3040     vallones(xtmp, vector_len);
3041     vpxor(dst, xtmp, dst, vector_len);
3042     break;
3043   case le:
3044     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3045     vallones(xtmp, vector_len);
3046     vpxor(dst, xtmp, dst, vector_len);
3047     break;
3048   case nlt:
3049     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3050     vallones(xtmp, vector_len);
3051     vpxor(dst, xtmp, dst, vector_len);
3052     break;
3053   case lt:
3054     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
3055     break;
3056   case nle:
3057     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
3058     break;
3059   default:
3060     assert(false, "Should not reach here");
3061   }
3062 }
3063 
3064 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3065   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3066   Assembler::vpmovzxbw(dst, src, vector_len);
3067 }
3068 
3069 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
3070   assert((src->encoding() < 16),"XMM register should be 0-15");
3071   Assembler::vpmovmskb(dst, src, vector_len);
3072 }
3073 
3074 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3075   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3076   Assembler::vpmullw(dst, nds, src, vector_len);
3077 }
3078 
3079 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3080   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3081   Assembler::vpmullw(dst, nds, src, vector_len);
3082 }
3083 
3084 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3085   assert((UseAVX > 0), "AVX support is needed");
3086   assert(rscratch != noreg || always_reachable(src), "missing");
3087 
3088   if (reachable(src)) {
3089     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
3090   } else {
3091     lea(rscratch, src);
3092     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
3093   }
3094 }
3095 
3096 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3097   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3098   Assembler::vpsubb(dst, nds, src, vector_len);
3099 }
3100 
3101 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3102   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3103   Assembler::vpsubb(dst, nds, src, vector_len);
3104 }
3105 
3106 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3107   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3108   Assembler::vpsubw(dst, nds, src, vector_len);
3109 }
3110 
3111 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3112   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3113   Assembler::vpsubw(dst, nds, src, vector_len);
3114 }
3115 
3116 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3117   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3118   Assembler::vpsraw(dst, nds, shift, vector_len);
3119 }
3120 
3121 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3122   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3123   Assembler::vpsraw(dst, nds, shift, vector_len);
3124 }
3125 
3126 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3127   assert(UseAVX > 2,"");
3128   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3129      vector_len = 2;
3130   }
3131   Assembler::evpsraq(dst, nds, shift, vector_len);
3132 }
3133 
3134 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3135   assert(UseAVX > 2,"");
3136   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3137      vector_len = 2;
3138   }
3139   Assembler::evpsraq(dst, nds, shift, vector_len);
3140 }
3141 
3142 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3143   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3144   Assembler::vpsrlw(dst, nds, shift, vector_len);
3145 }
3146 
3147 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3148   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3149   Assembler::vpsrlw(dst, nds, shift, vector_len);
3150 }
3151 
3152 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3153   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3154   Assembler::vpsllw(dst, nds, shift, vector_len);
3155 }
3156 
3157 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3158   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3159   Assembler::vpsllw(dst, nds, shift, vector_len);
3160 }
3161 
3162 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
3163   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
3164   Assembler::vptest(dst, src);
3165 }
3166 
3167 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3168   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3169   Assembler::punpcklbw(dst, src);
3170 }
3171 
3172 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
3173   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3174   Assembler::pshufd(dst, src, mode);
3175 }
3176 
3177 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3178   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3179   Assembler::pshuflw(dst, src, mode);
3180 }
3181 
3182 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3183   assert(rscratch != noreg || always_reachable(src), "missing");
3184 
3185   if (reachable(src)) {
3186     vandpd(dst, nds, as_Address(src), vector_len);
3187   } else {
3188     lea(rscratch, src);
3189     vandpd(dst, nds, Address(rscratch, 0), vector_len);
3190   }
3191 }
3192 
3193 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3194   assert(rscratch != noreg || always_reachable(src), "missing");
3195 
3196   if (reachable(src)) {
3197     vandps(dst, nds, as_Address(src), vector_len);
3198   } else {
3199     lea(rscratch, src);
3200     vandps(dst, nds, Address(rscratch, 0), vector_len);
3201   }
3202 }
3203 
3204 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
3205                             bool merge, int vector_len, Register rscratch) {
3206   assert(rscratch != noreg || always_reachable(src), "missing");
3207 
3208   if (reachable(src)) {
3209     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
3210   } else {
3211     lea(rscratch, src);
3212     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
3213   }
3214 }
3215 
3216 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3217   assert(rscratch != noreg || always_reachable(src), "missing");
3218 
3219   if (reachable(src)) {
3220     vdivsd(dst, nds, as_Address(src));
3221   } else {
3222     lea(rscratch, src);
3223     vdivsd(dst, nds, Address(rscratch, 0));
3224   }
3225 }
3226 
3227 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3228   assert(rscratch != noreg || always_reachable(src), "missing");
3229 
3230   if (reachable(src)) {
3231     vdivss(dst, nds, as_Address(src));
3232   } else {
3233     lea(rscratch, src);
3234     vdivss(dst, nds, Address(rscratch, 0));
3235   }
3236 }
3237 
3238 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3239   assert(rscratch != noreg || always_reachable(src), "missing");
3240 
3241   if (reachable(src)) {
3242     vmulsd(dst, nds, as_Address(src));
3243   } else {
3244     lea(rscratch, src);
3245     vmulsd(dst, nds, Address(rscratch, 0));
3246   }
3247 }
3248 
3249 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3250   assert(rscratch != noreg || always_reachable(src), "missing");
3251 
3252   if (reachable(src)) {
3253     vmulss(dst, nds, as_Address(src));
3254   } else {
3255     lea(rscratch, src);
3256     vmulss(dst, nds, Address(rscratch, 0));
3257   }
3258 }
3259 
3260 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3261   assert(rscratch != noreg || always_reachable(src), "missing");
3262 
3263   if (reachable(src)) {
3264     vsubsd(dst, nds, as_Address(src));
3265   } else {
3266     lea(rscratch, src);
3267     vsubsd(dst, nds, Address(rscratch, 0));
3268   }
3269 }
3270 
3271 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3272   assert(rscratch != noreg || always_reachable(src), "missing");
3273 
3274   if (reachable(src)) {
3275     vsubss(dst, nds, as_Address(src));
3276   } else {
3277     lea(rscratch, src);
3278     vsubss(dst, nds, Address(rscratch, 0));
3279   }
3280 }
3281 
3282 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3283   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3284   assert(rscratch != noreg || always_reachable(src), "missing");
3285 
3286   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
3287 }
3288 
3289 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
3290   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3291   assert(rscratch != noreg || always_reachable(src), "missing");
3292 
3293   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
3294 }
3295 
3296 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3297   assert(rscratch != noreg || always_reachable(src), "missing");
3298 
3299   if (reachable(src)) {
3300     vxorpd(dst, nds, as_Address(src), vector_len);
3301   } else {
3302     lea(rscratch, src);
3303     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
3304   }
3305 }
3306 
3307 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3308   assert(rscratch != noreg || always_reachable(src), "missing");
3309 
3310   if (reachable(src)) {
3311     vxorps(dst, nds, as_Address(src), vector_len);
3312   } else {
3313     lea(rscratch, src);
3314     vxorps(dst, nds, Address(rscratch, 0), vector_len);
3315   }
3316 }
3317 
3318 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3319   assert(rscratch != noreg || always_reachable(src), "missing");
3320 
3321   if (UseAVX > 1 || (vector_len < 1)) {
3322     if (reachable(src)) {
3323       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
3324     } else {
3325       lea(rscratch, src);
3326       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
3327     }
3328   } else {
3329     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
3330   }
3331 }
3332 
3333 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
3334   assert(rscratch != noreg || always_reachable(src), "missing");
3335 
3336   if (reachable(src)) {
3337     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
3338   } else {
3339     lea(rscratch, src);
3340     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
3341   }
3342 }
3343 
3344 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
3345   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
3346   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
3347   // The inverted mask is sign-extended
3348   andptr(possibly_non_local, inverted_mask);
3349 }
3350 
3351 void MacroAssembler::resolve_jobject(Register value,
3352                                      Register tmp) {
3353   Register thread = r15_thread;
3354   assert_different_registers(value, thread, tmp);
3355   Label done, tagged, weak_tagged;
3356   testptr(value, value);
3357   jcc(Assembler::zero, done);           // Use null as-is.
3358   testptr(value, JNIHandles::tag_mask); // Test for tag.
3359   jcc(Assembler::notZero, tagged);
3360 
3361   // Resolve local handle
3362   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp);
3363   verify_oop(value);
3364   jmp(done);
3365 
3366   bind(tagged);
3367   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
3368   jcc(Assembler::notZero, weak_tagged);
3369 
3370   // Resolve global handle
3371   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp);
3372   verify_oop(value);
3373   jmp(done);
3374 
3375   bind(weak_tagged);
3376   // Resolve jweak.
3377   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3378                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp);
3379   verify_oop(value);
3380 
3381   bind(done);
3382 }
3383 
3384 void MacroAssembler::resolve_global_jobject(Register value,
3385                                             Register tmp) {
3386   Register thread = r15_thread;
3387   assert_different_registers(value, thread, tmp);
3388   Label done;
3389 
3390   testptr(value, value);
3391   jcc(Assembler::zero, done);           // Use null as-is.
3392 
3393 #ifdef ASSERT
3394   {
3395     Label valid_global_tag;
3396     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
3397     jcc(Assembler::notZero, valid_global_tag);
3398     stop("non global jobject using resolve_global_jobject");
3399     bind(valid_global_tag);
3400   }
3401 #endif
3402 
3403   // Resolve global handle
3404   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp);
3405   verify_oop(value);
3406 
3407   bind(done);
3408 }
3409 
3410 void MacroAssembler::subptr(Register dst, int32_t imm32) {
3411   subq(dst, imm32);
3412 }
3413 
3414 // Force generation of a 4 byte immediate value even if it fits into 8bit
3415 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
3416   subq_imm32(dst, imm32);
3417 }
3418 
3419 void MacroAssembler::subptr(Register dst, Register src) {
3420   subq(dst, src);
3421 }
3422 
3423 // C++ bool manipulation
3424 void MacroAssembler::testbool(Register dst) {
3425   if(sizeof(bool) == 1)
3426     testb(dst, 0xff);
3427   else if(sizeof(bool) == 2) {
3428     // testw implementation needed for two byte bools
3429     ShouldNotReachHere();
3430   } else if(sizeof(bool) == 4)
3431     testl(dst, dst);
3432   else
3433     // unsupported
3434     ShouldNotReachHere();
3435 }
3436 
3437 void MacroAssembler::testptr(Register dst, Register src) {
3438   testq(dst, src);
3439 }
3440 






















































































































3441 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3442 void MacroAssembler::tlab_allocate(Register obj,
3443                                    Register var_size_in_bytes,
3444                                    int con_size_in_bytes,
3445                                    Register t1,
3446                                    Register t2,
3447                                    Label& slow_case) {
3448   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3449   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3450 }
3451 
3452 RegSet MacroAssembler::call_clobbered_gp_registers() {
3453   RegSet regs;
3454   regs += RegSet::of(rax, rcx, rdx);
3455 #ifndef _WINDOWS
3456   regs += RegSet::of(rsi, rdi);
3457 #endif
3458   regs += RegSet::range(r8, r11);
3459   if (UseAPX) {
3460     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
3461   }
3462   return regs;
3463 }
3464 
3465 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
3466   int num_xmm_registers = XMMRegister::available_xmm_registers();
3467 #if defined(_WINDOWS)
3468   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
3469   if (num_xmm_registers > 16) {
3470      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
3471   }
3472   return result;
3473 #else
3474   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
3475 #endif
3476 }
3477 
3478 // C1 only ever uses the first double/float of the XMM register.
3479 static int xmm_save_size() { return sizeof(double); }
3480 
3481 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
3482   masm->movdbl(Address(rsp, offset), reg);
3483 }
3484 
3485 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
3486   masm->movdbl(reg, Address(rsp, offset));
3487 }
3488 
3489 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
3490                                   bool save_fpu, int& gp_area_size, int& xmm_area_size) {
3491 
3492   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
3493                          StackAlignmentInBytes);
3494   xmm_area_size = save_fpu ? xmm_registers.size() * xmm_save_size() : 0;
3495 
3496   return gp_area_size + xmm_area_size;
3497 }
3498 
3499 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
3500   block_comment("push_call_clobbered_registers start");
3501   // Regular registers
3502   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
3503 
3504   int gp_area_size;
3505   int xmm_area_size;
3506   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
3507                                                gp_area_size, xmm_area_size);
3508   subptr(rsp, total_save_size);
3509 
3510   push_set(gp_registers_to_push, 0);
3511 
3512   if (save_fpu) {
3513     push_set(call_clobbered_xmm_registers(), gp_area_size);
3514   }
3515 
3516   block_comment("push_call_clobbered_registers end");
3517 }
3518 
3519 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
3520   block_comment("pop_call_clobbered_registers start");
3521 
3522   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
3523 
3524   int gp_area_size;
3525   int xmm_area_size;
3526   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
3527                                                gp_area_size, xmm_area_size);
3528 
3529   if (restore_fpu) {
3530     pop_set(call_clobbered_xmm_registers(), gp_area_size);
3531   }
3532 
3533   pop_set(gp_registers_to_pop, 0);
3534 
3535   addptr(rsp, total_save_size);
3536 
3537   vzeroupper();
3538 
3539   block_comment("pop_call_clobbered_registers end");
3540 }
3541 
3542 void MacroAssembler::push_set(XMMRegSet set, int offset) {
3543   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
3544   int spill_offset = offset;
3545 
3546   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
3547     save_xmm_register(this, spill_offset, *it);
3548     spill_offset += xmm_save_size();
3549   }
3550 }
3551 
3552 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
3553   int restore_size = set.size() * xmm_save_size();
3554   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
3555 
3556   int restore_offset = offset + restore_size - xmm_save_size();
3557 
3558   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
3559     restore_xmm_register(this, restore_offset, *it);
3560     restore_offset -= xmm_save_size();
3561   }
3562 }
3563 
3564 void MacroAssembler::push_set(RegSet set, int offset) {
3565   int spill_offset;
3566   if (offset == -1) {
3567     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3568     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
3569     subptr(rsp, aligned_size);
3570     spill_offset = 0;
3571   } else {
3572     spill_offset = offset;
3573   }
3574 
3575   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
3576     movptr(Address(rsp, spill_offset), *it);
3577     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3578   }
3579 }
3580 
3581 void MacroAssembler::pop_set(RegSet set, int offset) {
3582 
3583   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
3584   int restore_size = set.size() * gp_reg_size;
3585   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
3586 
3587   int restore_offset;
3588   if (offset == -1) {
3589     restore_offset = restore_size - gp_reg_size;
3590   } else {
3591     restore_offset = offset + restore_size - gp_reg_size;
3592   }
3593   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
3594     movptr(*it, Address(rsp, restore_offset));
3595     restore_offset -= gp_reg_size;
3596   }
3597 
3598   if (offset == -1) {
3599     addptr(rsp, aligned_size);
3600   }
3601 }
3602 
3603 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3604 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3605   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3606   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3607   Label done;
3608 
3609   testptr(length_in_bytes, length_in_bytes);
3610   jcc(Assembler::zero, done);
3611 
3612   // initialize topmost word, divide index by 2, check if odd and test if zero
3613   // note: for the remaining code to work, index must be a multiple of BytesPerWord
3614 #ifdef ASSERT
3615   {
3616     Label L;
3617     testptr(length_in_bytes, BytesPerWord - 1);
3618     jcc(Assembler::zero, L);
3619     stop("length must be a multiple of BytesPerWord");
3620     bind(L);
3621   }
3622 #endif
3623   Register index = length_in_bytes;
3624   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
3625   if (UseIncDec) {
3626     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
3627   } else {
3628     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
3629     shrptr(index, 1);
3630   }
3631 
3632   // initialize remaining object fields: index is a multiple of 2 now
3633   {
3634     Label loop;
3635     bind(loop);
3636     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
3637     decrement(index);
3638     jcc(Assembler::notZero, loop);
3639   }
3640 
3641   bind(done);
3642 }
3643 



























3644 // Look up the method for a megamorphic invokeinterface call.
3645 // The target method is determined by <intf_klass, itable_index>.
3646 // The receiver klass is in recv_klass.
3647 // On success, the result will be in method_result, and execution falls through.
3648 // On failure, execution transfers to the given label.
3649 void MacroAssembler::lookup_interface_method(Register recv_klass,
3650                                              Register intf_klass,
3651                                              RegisterOrConstant itable_index,
3652                                              Register method_result,
3653                                              Register scan_temp,
3654                                              Label& L_no_such_interface,
3655                                              bool return_method) {
3656   assert_different_registers(recv_klass, intf_klass, scan_temp);
3657   assert_different_registers(method_result, intf_klass, scan_temp);
3658   assert(recv_klass != method_result || !return_method,
3659          "recv_klass can be destroyed when method isn't needed");
3660 
3661   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
3662          "caller must use same register for non-constant itable index as for method");
3663 
3664   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
3665   int vtable_base = in_bytes(Klass::vtable_start_offset());
3666   int itentry_off = in_bytes(itableMethodEntry::method_offset());
3667   int scan_step   = itableOffsetEntry::size() * wordSize;
3668   int vte_size    = vtableEntry::size_in_bytes();
3669   Address::ScaleFactor times_vte_scale = Address::times_ptr;
3670   assert(vte_size == wordSize, "else adjust times_vte_scale");
3671 
3672   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
3673 
3674   // Could store the aligned, prescaled offset in the klass.
3675   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
3676 
3677   if (return_method) {
3678     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
3679     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
3680     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
3681   }
3682 
3683   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
3684   //   if (scan->interface() == intf) {
3685   //     result = (klass + scan->offset() + itable_index);
3686   //   }
3687   // }
3688   Label search, found_method;
3689 
3690   for (int peel = 1; peel >= 0; peel--) {
3691     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
3692     cmpptr(intf_klass, method_result);
3693 
3694     if (peel) {
3695       jccb(Assembler::equal, found_method);
3696     } else {
3697       jccb(Assembler::notEqual, search);
3698       // (invert the test to fall through to found_method...)
3699     }
3700 
3701     if (!peel)  break;
3702 
3703     bind(search);
3704 
3705     // Check that the previous entry is non-null.  A null entry means that
3706     // the receiver class doesn't implement the interface, and wasn't the
3707     // same as when the caller was compiled.
3708     testptr(method_result, method_result);
3709     jcc(Assembler::zero, L_no_such_interface);
3710     addptr(scan_temp, scan_step);
3711   }
3712 
3713   bind(found_method);
3714 
3715   if (return_method) {
3716     // Got a hit.
3717     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
3718     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
3719   }
3720 }
3721 
3722 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
3723 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
3724 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
3725 // The target method is determined by <holder_klass, itable_index>.
3726 // The receiver klass is in recv_klass.
3727 // On success, the result will be in method_result, and execution falls through.
3728 // On failure, execution transfers to the given label.
3729 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
3730                                                   Register holder_klass,
3731                                                   Register resolved_klass,
3732                                                   Register method_result,
3733                                                   Register scan_temp,
3734                                                   Register temp_reg2,
3735                                                   Register receiver,
3736                                                   int itable_index,
3737                                                   Label& L_no_such_interface) {
3738   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
3739   Register temp_itbl_klass = method_result;
3740   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
3741 
3742   int vtable_base = in_bytes(Klass::vtable_start_offset());
3743   int itentry_off = in_bytes(itableMethodEntry::method_offset());
3744   int scan_step = itableOffsetEntry::size() * wordSize;
3745   int vte_size = vtableEntry::size_in_bytes();
3746   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
3747   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
3748   Address::ScaleFactor times_vte_scale = Address::times_ptr;
3749   assert(vte_size == wordSize, "adjust times_vte_scale");
3750 
3751   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
3752 
3753   // temp_itbl_klass = recv_klass.itable[0]
3754   // scan_temp = &recv_klass.itable[0] + step
3755   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
3756   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
3757   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
3758   xorptr(temp_reg, temp_reg);
3759 
3760   // Initial checks:
3761   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
3762   //   - if (itable[0] == 0), no such interface
3763   //   - if (itable[0] == holder_klass), shortcut to "holder found"
3764   cmpptr(holder_klass, resolved_klass);
3765   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
3766   testptr(temp_itbl_klass, temp_itbl_klass);
3767   jccb(Assembler::zero, L_no_such_interface);
3768   cmpptr(holder_klass, temp_itbl_klass);
3769   jccb(Assembler::equal, L_holder_found);
3770 
3771   // Loop: Look for holder_klass record in itable
3772   //   do {
3773   //     tmp = itable[index];
3774   //     index += step;
3775   //     if (tmp == holder_klass) {
3776   //       goto L_holder_found; // Found!
3777   //     }
3778   //   } while (tmp != 0);
3779   //   goto L_no_such_interface // Not found.
3780   Label L_scan_holder;
3781   bind(L_scan_holder);
3782     movptr(temp_itbl_klass, Address(scan_temp, 0));
3783     addptr(scan_temp, scan_step);
3784     cmpptr(holder_klass, temp_itbl_klass);
3785     jccb(Assembler::equal, L_holder_found);
3786     testptr(temp_itbl_klass, temp_itbl_klass);
3787     jccb(Assembler::notZero, L_scan_holder);
3788 
3789   jmpb(L_no_such_interface);
3790 
3791   // Loop: Look for resolved_class record in itable
3792   //   do {
3793   //     tmp = itable[index];
3794   //     index += step;
3795   //     if (tmp == holder_klass) {
3796   //        // Also check if we have met a holder klass
3797   //        holder_tmp = itable[index-step-ioffset];
3798   //     }
3799   //     if (tmp == resolved_klass) {
3800   //        goto L_resolved_found;  // Found!
3801   //     }
3802   //   } while (tmp != 0);
3803   //   goto L_no_such_interface // Not found.
3804   //
3805   Label L_loop_scan_resolved;
3806   bind(L_loop_scan_resolved);
3807     movptr(temp_itbl_klass, Address(scan_temp, 0));
3808     addptr(scan_temp, scan_step);
3809     bind(L_loop_scan_resolved_entry);
3810     cmpptr(holder_klass, temp_itbl_klass);
3811     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
3812     cmpptr(resolved_klass, temp_itbl_klass);
3813     jccb(Assembler::equal, L_resolved_found);
3814     testptr(temp_itbl_klass, temp_itbl_klass);
3815     jccb(Assembler::notZero, L_loop_scan_resolved);
3816 
3817   jmpb(L_no_such_interface);
3818 
3819   Label L_ready;
3820 
3821   // See if we already have a holder klass. If not, go and scan for it.
3822   bind(L_resolved_found);
3823   testptr(temp_reg, temp_reg);
3824   jccb(Assembler::zero, L_scan_holder);
3825   jmpb(L_ready);
3826 
3827   bind(L_holder_found);
3828   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
3829 
3830   // Finally, temp_reg contains holder_klass vtable offset
3831   bind(L_ready);
3832   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
3833   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
3834     load_klass(scan_temp, receiver, noreg);
3835     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
3836   } else {
3837     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
3838   }
3839 }
3840 
3841 
3842 // virtual method calling
3843 void MacroAssembler::lookup_virtual_method(Register recv_klass,
3844                                            RegisterOrConstant vtable_index,
3845                                            Register method_result) {
3846   const ByteSize base = Klass::vtable_start_offset();
3847   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
3848   Address vtable_entry_addr(recv_klass,
3849                             vtable_index, Address::times_ptr,
3850                             base + vtableEntry::method_offset());
3851   movptr(method_result, vtable_entry_addr);
3852 }
3853 
3854 
3855 void MacroAssembler::check_klass_subtype(Register sub_klass,
3856                            Register super_klass,
3857                            Register temp_reg,
3858                            Label& L_success) {
3859   Label L_failure;
3860   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
3861   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
3862   bind(L_failure);
3863 }
3864 
3865 
3866 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3867                                                    Register super_klass,
3868                                                    Register temp_reg,
3869                                                    Label* L_success,
3870                                                    Label* L_failure,
3871                                                    Label* L_slow_path,
3872                                         RegisterOrConstant super_check_offset) {
3873   assert_different_registers(sub_klass, super_klass, temp_reg);
3874   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
3875   if (super_check_offset.is_register()) {
3876     assert_different_registers(sub_klass, super_klass,
3877                                super_check_offset.as_register());
3878   } else if (must_load_sco) {
3879     assert(temp_reg != noreg, "supply either a temp or a register offset");
3880   }
3881 
3882   Label L_fallthrough;
3883   int label_nulls = 0;
3884   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
3885   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
3886   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
3887   assert(label_nulls <= 1, "at most one null in the batch");
3888 
3889   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3890   int sco_offset = in_bytes(Klass::super_check_offset_offset());
3891   Address super_check_offset_addr(super_klass, sco_offset);
3892 
3893   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
3894   // range of a jccb.  If this routine grows larger, reconsider at
3895   // least some of these.
3896 #define local_jcc(assembler_cond, label)                                \
3897   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
3898   else                             jcc( assembler_cond, label) /*omit semi*/
3899 
3900   // Hacked jmp, which may only be used just before L_fallthrough.
3901 #define final_jmp(label)                                                \
3902   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
3903   else                            jmp(label)                /*omit semi*/
3904 
3905   // If the pointers are equal, we are done (e.g., String[] elements).
3906   // This self-check enables sharing of secondary supertype arrays among
3907   // non-primary types such as array-of-interface.  Otherwise, each such
3908   // type would need its own customized SSA.
3909   // We move this check to the front of the fast path because many
3910   // type checks are in fact trivially successful in this manner,
3911   // so we get a nicely predicted branch right at the start of the check.
3912   cmpptr(sub_klass, super_klass);
3913   local_jcc(Assembler::equal, *L_success);
3914 
3915   // Check the supertype display:
3916   if (must_load_sco) {
3917     // Positive movl does right thing on LP64.
3918     movl(temp_reg, super_check_offset_addr);
3919     super_check_offset = RegisterOrConstant(temp_reg);
3920   }
3921   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
3922   cmpptr(super_klass, super_check_addr); // load displayed supertype
3923 
3924   // This check has worked decisively for primary supers.
3925   // Secondary supers are sought in the super_cache ('super_cache_addr').
3926   // (Secondary supers are interfaces and very deeply nested subtypes.)
3927   // This works in the same check above because of a tricky aliasing
3928   // between the super_cache and the primary super display elements.
3929   // (The 'super_check_addr' can address either, as the case requires.)
3930   // Note that the cache is updated below if it does not help us find
3931   // what we need immediately.
3932   // So if it was a primary super, we can just fail immediately.
3933   // Otherwise, it's the slow path for us (no success at this point).
3934 
3935   if (super_check_offset.is_register()) {
3936     local_jcc(Assembler::equal, *L_success);
3937     cmpl(super_check_offset.as_register(), sc_offset);
3938     if (L_failure == &L_fallthrough) {
3939       local_jcc(Assembler::equal, *L_slow_path);
3940     } else {
3941       local_jcc(Assembler::notEqual, *L_failure);
3942       final_jmp(*L_slow_path);
3943     }
3944   } else if (super_check_offset.as_constant() == sc_offset) {
3945     // Need a slow path; fast failure is impossible.
3946     if (L_slow_path == &L_fallthrough) {
3947       local_jcc(Assembler::equal, *L_success);
3948     } else {
3949       local_jcc(Assembler::notEqual, *L_slow_path);
3950       final_jmp(*L_success);
3951     }
3952   } else {
3953     // No slow path; it's a fast decision.
3954     if (L_failure == &L_fallthrough) {
3955       local_jcc(Assembler::equal, *L_success);
3956     } else {
3957       local_jcc(Assembler::notEqual, *L_failure);
3958       final_jmp(*L_success);
3959     }
3960   }
3961 
3962   bind(L_fallthrough);
3963 
3964 #undef local_jcc
3965 #undef final_jmp
3966 }
3967 
3968 
3969 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
3970                                                           Register super_klass,
3971                                                           Register temp_reg,
3972                                                           Register temp2_reg,
3973                                                           Label* L_success,
3974                                                           Label* L_failure,
3975                                                           bool set_cond_codes) {
3976   assert_different_registers(sub_klass, super_klass, temp_reg);
3977   if (temp2_reg != noreg)
3978     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
3979 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
3980 
3981   Label L_fallthrough;
3982   int label_nulls = 0;
3983   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
3984   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
3985   assert(label_nulls <= 1, "at most one null in the batch");
3986 
3987   // a couple of useful fields in sub_klass:
3988   int ss_offset = in_bytes(Klass::secondary_supers_offset());
3989   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3990   Address secondary_supers_addr(sub_klass, ss_offset);
3991   Address super_cache_addr(     sub_klass, sc_offset);
3992 
3993   // Do a linear scan of the secondary super-klass chain.
3994   // This code is rarely used, so simplicity is a virtue here.
3995   // The repne_scan instruction uses fixed registers, which we must spill.
3996   // Don't worry too much about pre-existing connections with the input regs.
3997 
3998   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
3999   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
4000 
4001   // Get super_klass value into rax (even if it was in rdi or rcx).
4002   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
4003   if (super_klass != rax) {
4004     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
4005     mov(rax, super_klass);
4006   }
4007   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
4008   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
4009 
4010 #ifndef PRODUCT
4011   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
4012   ExternalAddress pst_counter_addr((address) pst_counter);
4013   lea(rcx, pst_counter_addr);
4014   incrementl(Address(rcx, 0));
4015 #endif //PRODUCT
4016 
4017   // We will consult the secondary-super array.
4018   movptr(rdi, secondary_supers_addr);
4019   // Load the array length.  (Positive movl does right thing on LP64.)
4020   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
4021   // Skip to start of data.
4022   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
4023 
4024   // Scan RCX words at [RDI] for an occurrence of RAX.
4025   // Set NZ/Z based on last compare.
4026   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
4027   // not change flags (only scas instruction which is repeated sets flags).
4028   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
4029 
4030     testptr(rax,rax); // Set Z = 0
4031     repne_scan();
4032 
4033   // Unspill the temp. registers:
4034   if (pushed_rdi)  pop(rdi);
4035   if (pushed_rcx)  pop(rcx);
4036   if (pushed_rax)  pop(rax);
4037 
4038   if (set_cond_codes) {
4039     // Special hack for the AD files:  rdi is guaranteed non-zero.
4040     assert(!pushed_rdi, "rdi must be left non-null");
4041     // Also, the condition codes are properly set Z/NZ on succeed/failure.
4042   }
4043 
4044   if (L_failure == &L_fallthrough)
4045         jccb(Assembler::notEqual, *L_failure);
4046   else  jcc(Assembler::notEqual, *L_failure);
4047 
4048   // Success.  Cache the super we found and proceed in triumph.
4049   movptr(super_cache_addr, super_klass);
4050 
4051   if (L_success != &L_fallthrough) {
4052     jmp(*L_success);
4053   }
4054 
4055 #undef IS_A_TEMP
4056 
4057   bind(L_fallthrough);
4058 }
4059 
4060 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
4061                                                    Register super_klass,
4062                                                    Register temp_reg,
4063                                                    Register temp2_reg,
4064                                                    Label* L_success,
4065                                                    Label* L_failure,
4066                                                    bool set_cond_codes) {
4067   assert(set_cond_codes == false, "must be false on 64-bit x86");
4068   check_klass_subtype_slow_path
4069     (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg,
4070      L_success, L_failure);
4071 }
4072 
4073 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
4074                                                    Register super_klass,
4075                                                    Register temp_reg,
4076                                                    Register temp2_reg,
4077                                                    Register temp3_reg,
4078                                                    Register temp4_reg,
4079                                                    Label* L_success,
4080                                                    Label* L_failure) {
4081   if (UseSecondarySupersTable) {
4082     check_klass_subtype_slow_path_table
4083       (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg,
4084        L_success, L_failure);
4085   } else {
4086     check_klass_subtype_slow_path_linear
4087       (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false);
4088   }
4089 }
4090 
4091 Register MacroAssembler::allocate_if_noreg(Register r,
4092                                   RegSetIterator<Register> &available_regs,
4093                                   RegSet &regs_to_push) {
4094   if (!r->is_valid()) {
4095     r = *available_regs++;
4096     regs_to_push += r;
4097   }
4098   return r;
4099 }
4100 
4101 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
4102                                                          Register super_klass,
4103                                                          Register temp_reg,
4104                                                          Register temp2_reg,
4105                                                          Register temp3_reg,
4106                                                          Register result_reg,
4107                                                          Label* L_success,
4108                                                          Label* L_failure) {
4109   // NB! Callers may assume that, when temp2_reg is a valid register,
4110   // this code sets it to a nonzero value.
4111   bool temp2_reg_was_valid = temp2_reg->is_valid();
4112 
4113   RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg);
4114 
4115   Label L_fallthrough;
4116   int label_nulls = 0;
4117   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4118   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4119   assert(label_nulls <= 1, "at most one null in the batch");
4120 
4121   BLOCK_COMMENT("check_klass_subtype_slow_path_table");
4122 
4123   RegSetIterator<Register> available_regs
4124     = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin();
4125 
4126   RegSet pushed_regs;
4127 
4128   temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
4129   temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
4130   temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);
4131   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
4132   Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs);
4133 
4134   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg);
4135 
4136   {
4137 
4138     int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
4139     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
4140     subptr(rsp, aligned_size);
4141     push_set(pushed_regs, 0);
4142 
4143     lookup_secondary_supers_table_var(sub_klass,
4144                                       super_klass,
4145                                       temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
4146     cmpq(result_reg, 0);
4147 
4148     // Unspill the temp. registers:
4149     pop_set(pushed_regs, 0);
4150     // Increment SP but do not clobber flags.
4151     lea(rsp, Address(rsp, aligned_size));
4152   }
4153 
4154   if (temp2_reg_was_valid) {
4155     movq(temp2_reg, 1);
4156   }
4157 
4158   jcc(Assembler::notEqual, *L_failure);
4159 
4160   if (L_success != &L_fallthrough) {
4161     jmp(*L_success);
4162   }
4163 
4164   bind(L_fallthrough);
4165 }
4166 
4167 // population_count variant for running without the POPCNT
4168 // instruction, which was introduced with SSE4.2 in 2008.
4169 void MacroAssembler::population_count(Register dst, Register src,
4170                                       Register scratch1, Register scratch2) {
4171   assert_different_registers(src, scratch1, scratch2);
4172   if (UsePopCountInstruction) {
4173     Assembler::popcntq(dst, src);
4174   } else {
4175     assert_different_registers(src, scratch1, scratch2);
4176     assert_different_registers(dst, scratch1, scratch2);
4177     Label loop, done;
4178 
4179     mov(scratch1, src);
4180     // dst = 0;
4181     // while(scratch1 != 0) {
4182     //   dst++;
4183     //   scratch1 &= (scratch1 - 1);
4184     // }
4185     xorl(dst, dst);
4186     testq(scratch1, scratch1);
4187     jccb(Assembler::equal, done);
4188     {
4189       bind(loop);
4190       incq(dst);
4191       movq(scratch2, scratch1);
4192       decq(scratch2);
4193       andq(scratch1, scratch2);
4194       jccb(Assembler::notEqual, loop);
4195     }
4196     bind(done);
4197   }
4198 #ifdef ASSERT
4199   mov64(scratch1, 0xCafeBabeDeadBeef);
4200   movq(scratch2, scratch1);
4201 #endif
4202 }
4203 
4204 // Ensure that the inline code and the stub are using the same registers.
4205 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
4206 do {                                                                 \
4207   assert(r_super_klass  == rax, "mismatch");                         \
4208   assert(r_array_base   == rbx, "mismatch");                         \
4209   assert(r_array_length == rcx, "mismatch");                         \
4210   assert(r_array_index  == rdx, "mismatch");                         \
4211   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
4212   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
4213   assert(result         == rdi || result      == noreg, "mismatch"); \
4214 } while(0)
4215 
4216 // Versions of salq and rorq that don't need count to be in rcx
4217 
4218 void MacroAssembler::salq(Register dest, Register count) {
4219   if (count == rcx) {
4220     Assembler::salq(dest);
4221   } else {
4222     assert_different_registers(rcx, dest);
4223     xchgq(rcx, count);
4224     Assembler::salq(dest);
4225     xchgq(rcx, count);
4226   }
4227 }
4228 
4229 void MacroAssembler::rorq(Register dest, Register count) {
4230   if (count == rcx) {
4231     Assembler::rorq(dest);
4232   } else {
4233     assert_different_registers(rcx, dest);
4234     xchgq(rcx, count);
4235     Assembler::rorq(dest);
4236     xchgq(rcx, count);
4237   }
4238 }
4239 
4240 // Return true: we succeeded in generating this code
4241 //
4242 // At runtime, return 0 in result if r_super_klass is a superclass of
4243 // r_sub_klass, otherwise return nonzero. Use this if you know the
4244 // super_klass_slot of the class you're looking for. This is always
4245 // the case for instanceof and checkcast.
4246 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
4247                                                          Register r_super_klass,
4248                                                          Register temp1,
4249                                                          Register temp2,
4250                                                          Register temp3,
4251                                                          Register temp4,
4252                                                          Register result,
4253                                                          u1 super_klass_slot) {
4254   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
4255 
4256   Label L_fallthrough, L_success, L_failure;
4257 
4258   BLOCK_COMMENT("lookup_secondary_supers_table {");
4259 
4260   const Register
4261     r_array_index  = temp1,
4262     r_array_length = temp2,
4263     r_array_base   = temp3,
4264     r_bitmap       = temp4;
4265 
4266   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
4267 
4268   xorq(result, result); // = 0
4269 
4270   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
4271   movq(r_array_index, r_bitmap);
4272 
4273   // First check the bitmap to see if super_klass might be present. If
4274   // the bit is zero, we are certain that super_klass is not one of
4275   // the secondary supers.
4276   u1 bit = super_klass_slot;
4277   {
4278     // NB: If the count in a x86 shift instruction is 0, the flags are
4279     // not affected, so we do a testq instead.
4280     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
4281     if (shift_count != 0) {
4282       salq(r_array_index, shift_count);
4283     } else {
4284       testq(r_array_index, r_array_index);
4285     }
4286   }
4287   // We test the MSB of r_array_index, i.e. its sign bit
4288   jcc(Assembler::positive, L_failure);
4289 
4290   // Get the first array index that can contain super_klass into r_array_index.
4291   if (bit != 0) {
4292     population_count(r_array_index, r_array_index, temp2, temp3);
4293   } else {
4294     movl(r_array_index, 1);
4295   }
4296   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
4297 
4298   // We will consult the secondary-super array.
4299   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4300 
4301   // We're asserting that the first word in an Array<Klass*> is the
4302   // length, and the second word is the first word of the data. If
4303   // that ever changes, r_array_base will have to be adjusted here.
4304   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
4305   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
4306 
4307   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4308   jccb(Assembler::equal, L_success);
4309 
4310   // Is there another entry to check? Consult the bitmap.
4311   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
4312   jccb(Assembler::carryClear, L_failure);
4313 
4314   // Linear probe. Rotate the bitmap so that the next bit to test is
4315   // in Bit 1.
4316   if (bit != 0) {
4317     rorq(r_bitmap, bit);
4318   }
4319 
4320   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
4321   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
4322   // Kills: r_array_length.
4323   // Returns: result.
4324   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
4325   // Result (0/1) is in rdi
4326   jmpb(L_fallthrough);
4327 
4328   bind(L_failure);
4329   incq(result); // 0 => 1
4330 
4331   bind(L_success);
4332   // result = 0;
4333 
4334   bind(L_fallthrough);
4335   BLOCK_COMMENT("} lookup_secondary_supers_table");
4336 
4337   if (VerifySecondarySupers) {
4338     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
4339                                   temp1, temp2, temp3);
4340   }
4341 }
4342 
4343 // At runtime, return 0 in result if r_super_klass is a superclass of
4344 // r_sub_klass, otherwise return nonzero. Use this version of
4345 // lookup_secondary_supers_table() if you don't know ahead of time
4346 // which superclass will be searched for. Used by interpreter and
4347 // runtime stubs. It is larger and has somewhat greater latency than
4348 // the version above, which takes a constant super_klass_slot.
4349 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
4350                                                        Register r_super_klass,
4351                                                        Register temp1,
4352                                                        Register temp2,
4353                                                        Register temp3,
4354                                                        Register temp4,
4355                                                        Register result) {
4356   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
4357   assert_different_registers(r_sub_klass, r_super_klass, rcx);
4358   RegSet temps = RegSet::of(temp1, temp2, temp3, temp4);
4359 
4360   Label L_fallthrough, L_success, L_failure;
4361 
4362   BLOCK_COMMENT("lookup_secondary_supers_table {");
4363 
4364   RegSetIterator<Register> available_regs = (temps - rcx).begin();
4365 
4366   // FIXME. Once we are sure that all paths reaching this point really
4367   // do pass rcx as one of our temps we can get rid of the following
4368   // workaround.
4369   assert(temps.contains(rcx), "fix this code");
4370 
4371   // We prefer to have our shift count in rcx. If rcx is one of our
4372   // temps, use it for slot. If not, pick any of our temps.
4373   Register slot;
4374   if (!temps.contains(rcx)) {
4375     slot = *available_regs++;
4376   } else {
4377     slot = rcx;
4378   }
4379 
4380   const Register r_array_index = *available_regs++;
4381   const Register r_bitmap      = *available_regs++;
4382 
4383   // The logic above guarantees this property, but we state it here.
4384   assert_different_registers(r_array_index, r_bitmap, rcx);
4385 
4386   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
4387   movq(r_array_index, r_bitmap);
4388 
4389   // First check the bitmap to see if super_klass might be present. If
4390   // the bit is zero, we are certain that super_klass is not one of
4391   // the secondary supers.
4392   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
4393   xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
4394   salq(r_array_index, slot);
4395 
4396   testq(r_array_index, r_array_index);
4397   // We test the MSB of r_array_index, i.e. its sign bit
4398   jcc(Assembler::positive, L_failure);
4399 
4400   const Register r_array_base = *available_regs++;
4401 
4402   // Get the first array index that can contain super_klass into r_array_index.
4403   // Note: Clobbers r_array_base and slot.
4404   population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot);
4405 
4406   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
4407 
4408   // We will consult the secondary-super array.
4409   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4410 
4411   // We're asserting that the first word in an Array<Klass*> is the
4412   // length, and the second word is the first word of the data. If
4413   // that ever changes, r_array_base will have to be adjusted here.
4414   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
4415   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
4416 
4417   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4418   jccb(Assembler::equal, L_success);
4419 
4420   // Restore slot to its true value
4421   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
4422 
4423   // Linear probe. Rotate the bitmap so that the next bit to test is
4424   // in Bit 1.
4425   rorq(r_bitmap, slot);
4426 
4427   // Is there another entry to check? Consult the bitmap.
4428   btq(r_bitmap, 1);
4429   jccb(Assembler::carryClear, L_failure);
4430 
4431   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
4432   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
4433   // Kills: r_array_length.
4434   // Returns: result.
4435   lookup_secondary_supers_table_slow_path(r_super_klass,
4436                                           r_array_base,
4437                                           r_array_index,
4438                                           r_bitmap,
4439                                           /*temp1*/result,
4440                                           /*temp2*/slot,
4441                                           &L_success,
4442                                           nullptr);
4443 
4444   bind(L_failure);
4445   movq(result, 1);
4446   jmpb(L_fallthrough);
4447 
4448   bind(L_success);
4449   xorq(result, result); // = 0
4450 
4451   bind(L_fallthrough);
4452   BLOCK_COMMENT("} lookup_secondary_supers_table");
4453 
4454   if (VerifySecondarySupers) {
4455     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
4456                                   temp1, temp2, temp3);
4457   }
4458 }
4459 
4460 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
4461                                  Label* L_success, Label* L_failure) {
4462   Label L_loop, L_fallthrough;
4463   {
4464     int label_nulls = 0;
4465     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
4466     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
4467     assert(label_nulls <= 1, "at most one null in the batch");
4468   }
4469   bind(L_loop);
4470   cmpq(value, Address(addr, count, Address::times_8));
4471   jcc(Assembler::equal, *L_success);
4472   addl(count, 1);
4473   cmpl(count, limit);
4474   jcc(Assembler::less, L_loop);
4475 
4476   if (&L_fallthrough != L_failure) {
4477     jmp(*L_failure);
4478   }
4479   bind(L_fallthrough);
4480 }
4481 
4482 // Called by code generated by check_klass_subtype_slow_path
4483 // above. This is called when there is a collision in the hashed
4484 // lookup in the secondary supers array.
4485 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
4486                                                              Register r_array_base,
4487                                                              Register r_array_index,
4488                                                              Register r_bitmap,
4489                                                              Register temp1,
4490                                                              Register temp2,
4491                                                              Label* L_success,
4492                                                              Label* L_failure) {
4493   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
4494 
4495   const Register
4496     r_array_length = temp1,
4497     r_sub_klass    = noreg,
4498     result         = noreg;
4499 
4500   Label L_fallthrough;
4501   int label_nulls = 0;
4502   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4503   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4504   assert(label_nulls <= 1, "at most one null in the batch");
4505 
4506   // Load the array length.
4507   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
4508   // And adjust the array base to point to the data.
4509   // NB! Effectively increments current slot index by 1.
4510   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
4511   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
4512 
4513   // Linear probe
4514   Label L_huge;
4515 
4516   // The bitmap is full to bursting.
4517   // Implicit invariant: BITMAP_FULL implies (length > 0)
4518   cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
4519   jcc(Assembler::greater, L_huge);
4520 
4521   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
4522   // current slot (at secondary_supers[r_array_index]) has not yet
4523   // been inspected, and r_array_index may be out of bounds if we
4524   // wrapped around the end of the array.
4525 
4526   { // This is conventional linear probing, but instead of terminating
4527     // when a null entry is found in the table, we maintain a bitmap
4528     // in which a 0 indicates missing entries.
4529     // The check above guarantees there are 0s in the bitmap, so the loop
4530     // eventually terminates.
4531 
4532     xorl(temp2, temp2); // = 0;
4533 
4534     Label L_again;
4535     bind(L_again);
4536 
4537     // Check for array wraparound.
4538     cmpl(r_array_index, r_array_length);
4539     cmovl(Assembler::greaterEqual, r_array_index, temp2);
4540 
4541     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
4542     jcc(Assembler::equal, *L_success);
4543 
4544     // If the next bit in bitmap is zero, we're done.
4545     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
4546     jcc(Assembler::carryClear, *L_failure);
4547 
4548     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
4549     addl(r_array_index, 1);
4550 
4551     jmp(L_again);
4552   }
4553 
4554   { // Degenerate case: more than 64 secondary supers.
4555     // FIXME: We could do something smarter here, maybe a vectorized
4556     // comparison or a binary search, but is that worth any added
4557     // complexity?
4558     bind(L_huge);
4559     xorl(r_array_index, r_array_index); // = 0
4560     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
4561                 L_success,
4562                 (&L_fallthrough != L_failure ? L_failure : nullptr));
4563 
4564     bind(L_fallthrough);
4565   }
4566 }
4567 
4568 struct VerifyHelperArguments {
4569   Klass* _super;
4570   Klass* _sub;
4571   intptr_t _linear_result;
4572   intptr_t _table_result;
4573 };
4574 
4575 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
4576   Klass::on_secondary_supers_verification_failure(args->_super,
4577                                                   args->_sub,
4578                                                   args->_linear_result,
4579                                                   args->_table_result,
4580                                                   msg);
4581 }
4582 
4583 // Make sure that the hashed lookup and a linear scan agree.
4584 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
4585                                                    Register r_super_klass,
4586                                                    Register result,
4587                                                    Register temp1,
4588                                                    Register temp2,
4589                                                    Register temp3) {
4590   const Register
4591       r_array_index  = temp1,
4592       r_array_length = temp2,
4593       r_array_base   = temp3,
4594       r_bitmap       = noreg;
4595 
4596   BLOCK_COMMENT("verify_secondary_supers_table {");
4597 
4598   Label L_success, L_failure, L_check, L_done;
4599 
4600   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
4601   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
4602   // And adjust the array base to point to the data.
4603   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
4604 
4605   testl(r_array_length, r_array_length); // array_length == 0?
4606   jcc(Assembler::zero, L_failure);
4607 
4608   movl(r_array_index, 0);
4609   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
4610   // fall through to L_failure
4611 
4612   const Register linear_result = r_array_index; // reuse temp1
4613 
4614   bind(L_failure); // not present
4615   movl(linear_result, 1);
4616   jmp(L_check);
4617 
4618   bind(L_success); // present
4619   movl(linear_result, 0);
4620 
4621   bind(L_check);
4622   cmpl(linear_result, result);
4623   jcc(Assembler::equal, L_done);
4624 
4625   { // To avoid calling convention issues, build a record on the stack
4626     // and pass the pointer to that instead.
4627     push(result);
4628     push(linear_result);
4629     push(r_sub_klass);
4630     push(r_super_klass);
4631     movptr(c_rarg1, rsp);
4632     movptr(c_rarg0, (uintptr_t) "mismatch");
4633     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
4634     should_not_reach_here();
4635   }
4636   bind(L_done);
4637 
4638   BLOCK_COMMENT("} verify_secondary_supers_table");
4639 }
4640 
4641 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
4642 
4643 void MacroAssembler::clinit_barrier(Register klass, Label* L_fast_path, Label* L_slow_path) {
4644   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
4645 
4646   Label L_fallthrough;
4647   if (L_fast_path == nullptr) {
4648     L_fast_path = &L_fallthrough;
4649   } else if (L_slow_path == nullptr) {
4650     L_slow_path = &L_fallthrough;
4651   }
4652 
4653   // Fast path check: class is fully initialized.
4654   // init_state needs acquire, but x86 is TSO, and so we are already good.
4655   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
4656   jcc(Assembler::equal, *L_fast_path);
4657 
4658   // Fast path check: current thread is initializer thread
4659   cmpptr(r15_thread, Address(klass, InstanceKlass::init_thread_offset()));
4660   if (L_slow_path == &L_fallthrough) {
4661     jcc(Assembler::equal, *L_fast_path);
4662     bind(*L_slow_path);
4663   } else if (L_fast_path == &L_fallthrough) {
4664     jcc(Assembler::notEqual, *L_slow_path);
4665     bind(*L_fast_path);
4666   } else {
4667     Unimplemented();
4668   }
4669 }
4670 
4671 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
4672   if (VM_Version::supports_cmov()) {
4673     cmovl(cc, dst, src);
4674   } else {
4675     Label L;
4676     jccb(negate_condition(cc), L);
4677     movl(dst, src);
4678     bind(L);
4679   }
4680 }
4681 
4682 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
4683   if (VM_Version::supports_cmov()) {
4684     cmovl(cc, dst, src);
4685   } else {
4686     Label L;
4687     jccb(negate_condition(cc), L);
4688     movl(dst, src);
4689     bind(L);
4690   }
4691 }
4692 
4693 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
4694   if (!VerifyOops) return;




4695 
4696   BLOCK_COMMENT("verify_oop {");
4697   push(rscratch1);
4698   push(rax);                          // save rax
4699   push(reg);                          // pass register argument
4700 
4701   // Pass register number to verify_oop_subroutine
4702   const char* b = nullptr;
4703   {
4704     ResourceMark rm;
4705     stringStream ss;
4706     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
4707     b = code_string(ss.as_string());
4708   }
4709   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
4710   pushptr(buffer.addr(), rscratch1);
4711 
4712   // call indirectly to solve generation ordering problem
4713   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4714   call(rax);
4715   // Caller pops the arguments (oop, message) and restores rax, r10
4716   BLOCK_COMMENT("} verify_oop");
4717 }
4718 
4719 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
4720   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
4721     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
4722     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
4723     vpternlogd(dst, 0xFF, dst, dst, vector_len);
4724   } else if (VM_Version::supports_avx()) {
4725     vpcmpeqd(dst, dst, dst, vector_len);
4726   } else {
4727     pcmpeqd(dst, dst);
4728   }
4729 }
4730 
4731 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
4732                                          int extra_slot_offset) {
4733   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4734   int stackElementSize = Interpreter::stackElementSize;
4735   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4736 #ifdef ASSERT
4737   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4738   assert(offset1 - offset == stackElementSize, "correct arithmetic");
4739 #endif
4740   Register             scale_reg    = noreg;
4741   Address::ScaleFactor scale_factor = Address::no_scale;
4742   if (arg_slot.is_constant()) {
4743     offset += arg_slot.as_constant() * stackElementSize;
4744   } else {
4745     scale_reg    = arg_slot.as_register();
4746     scale_factor = Address::times(stackElementSize);
4747   }
4748   offset += wordSize;           // return PC is on stack
4749   return Address(rsp, scale_reg, scale_factor, offset);
4750 }
4751 
4752 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
4753   if (!VerifyOops) return;




4754 
4755   push(rscratch1);
4756   push(rax); // save rax,
4757   // addr may contain rsp so we will have to adjust it based on the push
4758   // we just did (and on 64 bit we do two pushes)
4759   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4760   // stores rax into addr which is backwards of what was intended.
4761   if (addr.uses(rsp)) {
4762     lea(rax, addr);
4763     pushptr(Address(rax, 2 * BytesPerWord));
4764   } else {
4765     pushptr(addr);
4766   }
4767 
4768   // Pass register number to verify_oop_subroutine
4769   const char* b = nullptr;
4770   {
4771     ResourceMark rm;
4772     stringStream ss;
4773     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
4774     b = code_string(ss.as_string());
4775   }
4776   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
4777   pushptr(buffer.addr(), rscratch1);
4778 
4779   // call indirectly to solve generation ordering problem
4780   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4781   call(rax);
4782   // Caller pops the arguments (addr, message) and restores rax, r10.
4783 }
4784 
4785 void MacroAssembler::verify_tlab() {
4786 #ifdef ASSERT
4787   if (UseTLAB && VerifyOops) {
4788     Label next, ok;
4789     Register t1 = rsi;
4790 
4791     push(t1);
4792 
4793     movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
4794     cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset())));
4795     jcc(Assembler::aboveEqual, next);
4796     STOP("assert(top >= start)");
4797     should_not_reach_here();
4798 
4799     bind(next);
4800     movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
4801     cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
4802     jcc(Assembler::aboveEqual, ok);
4803     STOP("assert(top <= end)");
4804     should_not_reach_here();
4805 
4806     bind(ok);
4807     pop(t1);
4808   }
4809 #endif
4810 }
4811 
4812 class ControlWord {
4813  public:
4814   int32_t _value;
4815 
4816   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
4817   int  precision_control() const       { return  (_value >>  8) & 3      ; }
4818   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4819   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4820   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4821   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4822   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4823   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4824 
4825   void print() const {
4826     // rounding control
4827     const char* rc;
4828     switch (rounding_control()) {
4829       case 0: rc = "round near"; break;
4830       case 1: rc = "round down"; break;
4831       case 2: rc = "round up  "; break;
4832       case 3: rc = "chop      "; break;
4833       default:
4834         rc = nullptr; // silence compiler warnings
4835         fatal("Unknown rounding control: %d", rounding_control());
4836     };
4837     // precision control
4838     const char* pc;
4839     switch (precision_control()) {
4840       case 0: pc = "24 bits "; break;
4841       case 1: pc = "reserved"; break;
4842       case 2: pc = "53 bits "; break;
4843       case 3: pc = "64 bits "; break;
4844       default:
4845         pc = nullptr; // silence compiler warnings
4846         fatal("Unknown precision control: %d", precision_control());
4847     };
4848     // flags
4849     char f[9];
4850     f[0] = ' ';
4851     f[1] = ' ';
4852     f[2] = (precision   ()) ? 'P' : 'p';
4853     f[3] = (underflow   ()) ? 'U' : 'u';
4854     f[4] = (overflow    ()) ? 'O' : 'o';
4855     f[5] = (zero_divide ()) ? 'Z' : 'z';
4856     f[6] = (denormalized()) ? 'D' : 'd';
4857     f[7] = (invalid     ()) ? 'I' : 'i';
4858     f[8] = '\x0';
4859     // output
4860     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
4861   }
4862 
4863 };
4864 
4865 class StatusWord {
4866  public:
4867   int32_t _value;
4868 
4869   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
4870   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
4871   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
4872   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
4873   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
4874   int  top() const                     { return  (_value >> 11) & 7      ; }
4875   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
4876   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
4877   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4878   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4879   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4880   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4881   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4882   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4883 
4884   void print() const {
4885     // condition codes
4886     char c[5];
4887     c[0] = (C3()) ? '3' : '-';
4888     c[1] = (C2()) ? '2' : '-';
4889     c[2] = (C1()) ? '1' : '-';
4890     c[3] = (C0()) ? '0' : '-';
4891     c[4] = '\x0';
4892     // flags
4893     char f[9];
4894     f[0] = (error_status()) ? 'E' : '-';
4895     f[1] = (stack_fault ()) ? 'S' : '-';
4896     f[2] = (precision   ()) ? 'P' : '-';
4897     f[3] = (underflow   ()) ? 'U' : '-';
4898     f[4] = (overflow    ()) ? 'O' : '-';
4899     f[5] = (zero_divide ()) ? 'Z' : '-';
4900     f[6] = (denormalized()) ? 'D' : '-';
4901     f[7] = (invalid     ()) ? 'I' : '-';
4902     f[8] = '\x0';
4903     // output
4904     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
4905   }
4906 
4907 };
4908 
4909 class TagWord {
4910  public:
4911   int32_t _value;
4912 
4913   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
4914 
4915   void print() const {
4916     printf("%04x", _value & 0xFFFF);
4917   }
4918 
4919 };
4920 
4921 class FPU_Register {
4922  public:
4923   int32_t _m0;
4924   int32_t _m1;
4925   int16_t _ex;
4926 
4927   bool is_indefinite() const           {
4928     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
4929   }
4930 
4931   void print() const {
4932     char  sign = (_ex < 0) ? '-' : '+';
4933     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
4934     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
4935   };
4936 
4937 };
4938 
4939 class FPU_State {
4940  public:
4941   enum {
4942     register_size       = 10,
4943     number_of_registers =  8,
4944     register_mask       =  7
4945   };
4946 
4947   ControlWord  _control_word;
4948   StatusWord   _status_word;
4949   TagWord      _tag_word;
4950   int32_t      _error_offset;
4951   int32_t      _error_selector;
4952   int32_t      _data_offset;
4953   int32_t      _data_selector;
4954   int8_t       _register[register_size * number_of_registers];
4955 
4956   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
4957   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
4958 
4959   const char* tag_as_string(int tag) const {
4960     switch (tag) {
4961       case 0: return "valid";
4962       case 1: return "zero";
4963       case 2: return "special";
4964       case 3: return "empty";
4965     }
4966     ShouldNotReachHere();
4967     return nullptr;
4968   }
4969 
4970   void print() const {
4971     // print computation registers
4972     { int t = _status_word.top();
4973       for (int i = 0; i < number_of_registers; i++) {
4974         int j = (i - t) & register_mask;
4975         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
4976         st(j)->print();
4977         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
4978       }
4979     }
4980     printf("\n");
4981     // print control registers
4982     printf("ctrl = "); _control_word.print(); printf("\n");
4983     printf("stat = "); _status_word .print(); printf("\n");
4984     printf("tags = "); _tag_word    .print(); printf("\n");
4985   }
4986 
4987 };
4988 
4989 class Flag_Register {
4990  public:
4991   int32_t _value;
4992 
4993   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
4994   bool direction() const               { return ((_value >> 10) & 1) != 0; }
4995   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
4996   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
4997   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
4998   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
4999   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5000 
5001   void print() const {
5002     // flags
5003     char f[8];
5004     f[0] = (overflow       ()) ? 'O' : '-';
5005     f[1] = (direction      ()) ? 'D' : '-';
5006     f[2] = (sign           ()) ? 'S' : '-';
5007     f[3] = (zero           ()) ? 'Z' : '-';
5008     f[4] = (auxiliary_carry()) ? 'A' : '-';
5009     f[5] = (parity         ()) ? 'P' : '-';
5010     f[6] = (carry          ()) ? 'C' : '-';
5011     f[7] = '\x0';
5012     // output
5013     printf("%08x  flags = %s", _value, f);
5014   }
5015 
5016 };
5017 
5018 class IU_Register {
5019  public:
5020   int32_t _value;
5021 
5022   void print() const {
5023     printf("%08x  %11d", _value, _value);
5024   }
5025 
5026 };
5027 
5028 class IU_State {
5029  public:
5030   Flag_Register _eflags;
5031   IU_Register   _rdi;
5032   IU_Register   _rsi;
5033   IU_Register   _rbp;
5034   IU_Register   _rsp;
5035   IU_Register   _rbx;
5036   IU_Register   _rdx;
5037   IU_Register   _rcx;
5038   IU_Register   _rax;
5039 
5040   void print() const {
5041     // computation registers
5042     printf("rax,  = "); _rax.print(); printf("\n");
5043     printf("rbx,  = "); _rbx.print(); printf("\n");
5044     printf("rcx  = "); _rcx.print(); printf("\n");
5045     printf("rdx  = "); _rdx.print(); printf("\n");
5046     printf("rdi  = "); _rdi.print(); printf("\n");
5047     printf("rsi  = "); _rsi.print(); printf("\n");
5048     printf("rbp,  = "); _rbp.print(); printf("\n");
5049     printf("rsp  = "); _rsp.print(); printf("\n");
5050     printf("\n");
5051     // control registers
5052     printf("flgs = "); _eflags.print(); printf("\n");
5053   }
5054 };
5055 
5056 
5057 class CPU_State {
5058  public:
5059   FPU_State _fpu_state;
5060   IU_State  _iu_state;
5061 
5062   void print() const {
5063     printf("--------------------------------------------------\n");
5064     _iu_state .print();
5065     printf("\n");
5066     _fpu_state.print();
5067     printf("--------------------------------------------------\n");
5068   }
5069 
5070 };
5071 
5072 
5073 static void _print_CPU_state(CPU_State* state) {
5074   state->print();
5075 };
5076 
5077 
5078 void MacroAssembler::print_CPU_state() {
5079   push_CPU_state();
5080   push(rsp);                // pass CPU state
5081   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5082   addptr(rsp, wordSize);       // discard argument
5083   pop_CPU_state();
5084 }
5085 
5086 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
5087   // Either restore the MXCSR register after returning from the JNI Call
5088   // or verify that it wasn't changed (with -Xcheck:jni flag).
5089   if (VM_Version::supports_sse()) {
5090     if (RestoreMXCSROnJNICalls) {
5091       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
5092     } else if (CheckJNICalls) {
5093       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5094     }
5095   }
5096   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5097   vzeroupper();
5098 }
5099 
5100 // ((OopHandle)result).resolve();
5101 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
5102   assert_different_registers(result, tmp);
5103 
5104   // Only 64 bit platforms support GCs that require a tmp register
5105   // Only IN_HEAP loads require a thread_tmp register
5106   // OopHandle::resolve is an indirection like jobject.
5107   access_load_at(T_OBJECT, IN_NATIVE,
5108                  result, Address(result, 0), tmp);
5109 }
5110 
5111 // ((WeakHandle)result).resolve();
5112 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
5113   assert_different_registers(rresult, rtmp);
5114   Label resolved;
5115 
5116   // A null weak handle resolves to null.
5117   cmpptr(rresult, 0);
5118   jcc(Assembler::equal, resolved);
5119 
5120   // Only 64 bit platforms support GCs that require a tmp register
5121   // Only IN_HEAP loads require a thread_tmp register
5122   // WeakHandle::resolve is an indirection like jweak.
5123   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
5124                  rresult, Address(rresult, 0), rtmp);
5125   bind(resolved);
5126 }
5127 
5128 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
5129   // get mirror
5130   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
5131   load_method_holder(mirror, method);
5132   movptr(mirror, Address(mirror, mirror_offset));
5133   resolve_oop_handle(mirror, tmp);
5134 }
5135 
5136 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
5137   load_method_holder(rresult, rmethod);
5138   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
5139 }
5140 
5141 void MacroAssembler::load_method_holder(Register holder, Register method) {
5142   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
5143   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
5144   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
5145 }
5146 










5147 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
5148   assert(UseCompactObjectHeaders, "expect compact object headers");
5149   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
5150   shrq(dst, markWord::klass_shift);
5151 }
5152 
5153 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
5154   assert_different_registers(src, tmp);
5155   assert_different_registers(dst, tmp);
5156 
5157   if (UseCompactObjectHeaders) {
5158     load_narrow_klass_compact(dst, src);
5159     decode_klass_not_null(dst, tmp);
5160   } else if (UseCompressedClassPointers) {
5161     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5162     decode_klass_not_null(dst, tmp);
5163   } else {
5164     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5165   }
5166 }
5167 





5168 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
5169   assert(!UseCompactObjectHeaders, "not with compact headers");
5170   assert_different_registers(src, tmp);
5171   assert_different_registers(dst, tmp);
5172   if (UseCompressedClassPointers) {
5173     encode_klass_not_null(src, tmp);
5174     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5175   } else {
5176     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5177   }
5178 }
5179 
5180 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
5181   if (UseCompactObjectHeaders) {
5182     assert(tmp != noreg, "need tmp");
5183     assert_different_registers(klass, obj, tmp);
5184     load_narrow_klass_compact(tmp, obj);
5185     cmpl(klass, tmp);
5186   } else if (UseCompressedClassPointers) {
5187     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
5188   } else {
5189     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
5190   }
5191 }
5192 
5193 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
5194   if (UseCompactObjectHeaders) {
5195     assert(tmp2 != noreg, "need tmp2");
5196     assert_different_registers(obj1, obj2, tmp1, tmp2);
5197     load_narrow_klass_compact(tmp1, obj1);
5198     load_narrow_klass_compact(tmp2, obj2);
5199     cmpl(tmp1, tmp2);
5200   } else if (UseCompressedClassPointers) {
5201     movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
5202     cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
5203   } else {
5204     movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
5205     cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
5206   }
5207 }
5208 
5209 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
5210                                     Register tmp1) {
5211   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5212   decorators = AccessInternal::decorator_fixup(decorators, type);
5213   bool as_raw = (decorators & AS_RAW) != 0;
5214   if (as_raw) {
5215     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1);
5216   } else {
5217     bs->load_at(this, decorators, type, dst, src, tmp1);
5218   }
5219 }
5220 
5221 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
5222                                      Register tmp1, Register tmp2, Register tmp3) {
5223   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5224   decorators = AccessInternal::decorator_fixup(decorators, type);
5225   bool as_raw = (decorators & AS_RAW) != 0;
5226   if (as_raw) {
5227     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
5228   } else {
5229     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
5230   }
5231 }
5232 








































5233 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, DecoratorSet decorators) {
5234   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1);
5235 }
5236 
5237 // Doesn't do verification, generates fixed size code
5238 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, DecoratorSet decorators) {
5239   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1);
5240 }
5241 
5242 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
5243                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
5244   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
5245 }
5246 
5247 // Used for storing nulls.
5248 void MacroAssembler::store_heap_oop_null(Address dst) {
5249   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
5250 }
5251 
5252 void MacroAssembler::store_klass_gap(Register dst, Register src) {
5253   assert(!UseCompactObjectHeaders, "Don't use with compact headers");
5254   if (UseCompressedClassPointers) {
5255     // Store to klass gap in destination
5256     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
5257   }
5258 }
5259 
5260 #ifdef ASSERT
5261 void MacroAssembler::verify_heapbase(const char* msg) {
5262   assert (UseCompressedOops, "should be compressed");
5263   assert (Universe::heap() != nullptr, "java heap should be initialized");
5264   if (CheckCompressedOops) {
5265     Label ok;
5266     ExternalAddress src2(CompressedOops::base_addr());
5267     const bool is_src2_reachable = reachable(src2);
5268     if (!is_src2_reachable) {
5269       push(rscratch1);  // cmpptr trashes rscratch1
5270     }
5271     cmpptr(r12_heapbase, src2, rscratch1);
5272     jcc(Assembler::equal, ok);
5273     STOP(msg);
5274     bind(ok);
5275     if (!is_src2_reachable) {
5276       pop(rscratch1);
5277     }
5278   }
5279 }
5280 #endif
5281 
5282 // Algorithm must match oop.inline.hpp encode_heap_oop.
5283 void MacroAssembler::encode_heap_oop(Register r) {
5284 #ifdef ASSERT
5285   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
5286 #endif
5287   verify_oop_msg(r, "broken oop in encode_heap_oop");
5288   if (CompressedOops::base() == nullptr) {
5289     if (CompressedOops::shift() != 0) {
5290       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5291       shrq(r, LogMinObjAlignmentInBytes);
5292     }
5293     return;
5294   }
5295   testq(r, r);
5296   cmovq(Assembler::equal, r, r12_heapbase);
5297   subq(r, r12_heapbase);
5298   shrq(r, LogMinObjAlignmentInBytes);
5299 }
5300 
5301 void MacroAssembler::encode_heap_oop_not_null(Register r) {
5302 #ifdef ASSERT
5303   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
5304   if (CheckCompressedOops) {
5305     Label ok;
5306     testq(r, r);
5307     jcc(Assembler::notEqual, ok);
5308     STOP("null oop passed to encode_heap_oop_not_null");
5309     bind(ok);
5310   }
5311 #endif
5312   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
5313   if (CompressedOops::base() != nullptr) {
5314     subq(r, r12_heapbase);
5315   }
5316   if (CompressedOops::shift() != 0) {
5317     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5318     shrq(r, LogMinObjAlignmentInBytes);
5319   }
5320 }
5321 
5322 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
5323 #ifdef ASSERT
5324   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
5325   if (CheckCompressedOops) {
5326     Label ok;
5327     testq(src, src);
5328     jcc(Assembler::notEqual, ok);
5329     STOP("null oop passed to encode_heap_oop_not_null2");
5330     bind(ok);
5331   }
5332 #endif
5333   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
5334   if (dst != src) {
5335     movq(dst, src);
5336   }
5337   if (CompressedOops::base() != nullptr) {
5338     subq(dst, r12_heapbase);
5339   }
5340   if (CompressedOops::shift() != 0) {
5341     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5342     shrq(dst, LogMinObjAlignmentInBytes);
5343   }
5344 }
5345 
5346 void  MacroAssembler::decode_heap_oop(Register r) {
5347 #ifdef ASSERT
5348   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
5349 #endif
5350   if (CompressedOops::base() == nullptr) {
5351     if (CompressedOops::shift() != 0) {
5352       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5353       shlq(r, LogMinObjAlignmentInBytes);
5354     }
5355   } else {
5356     Label done;
5357     shlq(r, LogMinObjAlignmentInBytes);
5358     jccb(Assembler::equal, done);
5359     addq(r, r12_heapbase);
5360     bind(done);
5361   }
5362   verify_oop_msg(r, "broken oop in decode_heap_oop");
5363 }
5364 
5365 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
5366   // Note: it will change flags
5367   assert (UseCompressedOops, "should only be used for compressed headers");
5368   assert (Universe::heap() != nullptr, "java heap should be initialized");
5369   // Cannot assert, unverified entry point counts instructions (see .ad file)
5370   // vtableStubs also counts instructions in pd_code_size_limit.
5371   // Also do not verify_oop as this is called by verify_oop.
5372   if (CompressedOops::shift() != 0) {
5373     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5374     shlq(r, LogMinObjAlignmentInBytes);
5375     if (CompressedOops::base() != nullptr) {
5376       addq(r, r12_heapbase);
5377     }
5378   } else {
5379     assert (CompressedOops::base() == nullptr, "sanity");
5380   }
5381 }
5382 
5383 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
5384   // Note: it will change flags
5385   assert (UseCompressedOops, "should only be used for compressed headers");
5386   assert (Universe::heap() != nullptr, "java heap should be initialized");
5387   // Cannot assert, unverified entry point counts instructions (see .ad file)
5388   // vtableStubs also counts instructions in pd_code_size_limit.
5389   // Also do not verify_oop as this is called by verify_oop.
5390   if (CompressedOops::shift() != 0) {
5391     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
5392     if (LogMinObjAlignmentInBytes == Address::times_8) {
5393       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
5394     } else {
5395       if (dst != src) {
5396         movq(dst, src);
5397       }
5398       shlq(dst, LogMinObjAlignmentInBytes);
5399       if (CompressedOops::base() != nullptr) {
5400         addq(dst, r12_heapbase);
5401       }
5402     }
5403   } else {
5404     assert (CompressedOops::base() == nullptr, "sanity");
5405     if (dst != src) {
5406       movq(dst, src);
5407     }
5408   }
5409 }
5410 
5411 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
5412   BLOCK_COMMENT("encode_klass_not_null {");
5413   assert_different_registers(r, tmp);
5414   if (CompressedKlassPointers::base() != nullptr) {
5415     if (AOTCodeCache::is_on_for_dump()) {
5416       movptr(tmp, ExternalAddress(CompressedKlassPointers::base_addr()));
5417     } else {
5418       movptr(tmp, (intptr_t)CompressedKlassPointers::base());
5419     }
5420     subq(r, tmp);
5421   }
5422   if (CompressedKlassPointers::shift() != 0) {
5423     shrq(r, CompressedKlassPointers::shift());
5424   }
5425   BLOCK_COMMENT("} encode_klass_not_null");
5426 }
5427 
5428 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
5429   BLOCK_COMMENT("encode_and_move_klass_not_null {");
5430   assert_different_registers(src, dst);
5431   if (CompressedKlassPointers::base() != nullptr) {
5432     movptr(dst, -(intptr_t)CompressedKlassPointers::base());
5433     addq(dst, src);
5434   } else {
5435     movptr(dst, src);
5436   }
5437   if (CompressedKlassPointers::shift() != 0) {
5438     shrq(dst, CompressedKlassPointers::shift());
5439   }
5440   BLOCK_COMMENT("} encode_and_move_klass_not_null");
5441 }
5442 
5443 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
5444   BLOCK_COMMENT("decode_klass_not_null {");
5445   assert_different_registers(r, tmp);
5446   // Note: it will change flags
5447   assert(UseCompressedClassPointers, "should only be used for compressed headers");
5448   // Cannot assert, unverified entry point counts instructions (see .ad file)
5449   // vtableStubs also counts instructions in pd_code_size_limit.
5450   // Also do not verify_oop as this is called by verify_oop.
5451   if (CompressedKlassPointers::shift() != 0) {
5452     shlq(r, CompressedKlassPointers::shift());
5453   }
5454   if (CompressedKlassPointers::base() != nullptr) {
5455     if (AOTCodeCache::is_on_for_dump()) {
5456       movptr(tmp, ExternalAddress(CompressedKlassPointers::base_addr()));
5457     } else {
5458       movptr(tmp, (intptr_t)CompressedKlassPointers::base());
5459     }
5460     addq(r, tmp);
5461   }
5462   BLOCK_COMMENT("} decode_klass_not_null");
5463 }
5464 
5465 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
5466   BLOCK_COMMENT("decode_and_move_klass_not_null {");
5467   assert_different_registers(src, dst);
5468   // Note: it will change flags
5469   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5470   // Cannot assert, unverified entry point counts instructions (see .ad file)
5471   // vtableStubs also counts instructions in pd_code_size_limit.
5472   // Also do not verify_oop as this is called by verify_oop.
5473 
5474   if (CompressedKlassPointers::base() == nullptr &&
5475       CompressedKlassPointers::shift() == 0) {
5476     // The best case scenario is that there is no base or shift. Then it is already
5477     // a pointer that needs nothing but a register rename.
5478     movl(dst, src);
5479   } else {
5480     if (CompressedKlassPointers::shift() <= Address::times_8) {
5481       if (CompressedKlassPointers::base() != nullptr) {
5482         movptr(dst, (intptr_t)CompressedKlassPointers::base());
5483       } else {
5484         xorq(dst, dst);
5485       }
5486       if (CompressedKlassPointers::shift() != 0) {
5487         assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
5488         leaq(dst, Address(dst, src, Address::times_8, 0));
5489       } else {
5490         addq(dst, src);
5491       }
5492     } else {
5493       if (CompressedKlassPointers::base() != nullptr) {
5494         const intptr_t base_right_shifted =
5495             (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
5496         movptr(dst, base_right_shifted);
5497       } else {
5498         xorq(dst, dst);
5499       }
5500       addq(dst, src);
5501       shlq(dst, CompressedKlassPointers::shift());
5502     }
5503   }
5504   BLOCK_COMMENT("} decode_and_move_klass_not_null");
5505 }
5506 
5507 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5508   assert (UseCompressedOops, "should only be used for compressed headers");
5509   assert (Universe::heap() != nullptr, "java heap should be initialized");
5510   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5511   int oop_index = oop_recorder()->find_index(obj);
5512   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5513   mov_narrow_oop(dst, oop_index, rspec);
5514 }
5515 
5516 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
5517   assert (UseCompressedOops, "should only be used for compressed headers");
5518   assert (Universe::heap() != nullptr, "java heap should be initialized");
5519   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5520   int oop_index = oop_recorder()->find_index(obj);
5521   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5522   mov_narrow_oop(dst, oop_index, rspec);
5523 }
5524 
5525 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
5526   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5527   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5528   int klass_index = oop_recorder()->find_index(k);
5529   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5530   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5531 }
5532 
5533 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
5534   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5535   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5536   int klass_index = oop_recorder()->find_index(k);
5537   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5538   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5539 }
5540 
5541 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
5542   assert (UseCompressedOops, "should only be used for compressed headers");
5543   assert (Universe::heap() != nullptr, "java heap should be initialized");
5544   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5545   int oop_index = oop_recorder()->find_index(obj);
5546   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5547   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5548 }
5549 
5550 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
5551   assert (UseCompressedOops, "should only be used for compressed headers");
5552   assert (Universe::heap() != nullptr, "java heap should be initialized");
5553   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5554   int oop_index = oop_recorder()->find_index(obj);
5555   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5556   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5557 }
5558 
5559 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
5560   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5561   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5562   int klass_index = oop_recorder()->find_index(k);
5563   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5564   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5565 }
5566 
5567 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
5568   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5569   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5570   int klass_index = oop_recorder()->find_index(k);
5571   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5572   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5573 }
5574 
5575 void MacroAssembler::reinit_heapbase() {
5576   if (UseCompressedOops) {
5577     if (Universe::heap() != nullptr) {
5578       if (CompressedOops::base() == nullptr) {
5579         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5580       } else {
5581         mov64(r12_heapbase, (int64_t)CompressedOops::base());
5582       }
5583     } else {
5584       movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr()));
5585     }
5586   }
5587 }
5588 




























































































































































































































































































































































































































































































5589 #if COMPILER2_OR_JVMCI
5590 
5591 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5592 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5593   // cnt - number of qwords (8-byte words).
5594   // base - start address, qword aligned.
5595   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5596   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
5597   if (use64byteVector) {
5598     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5599   } else if (MaxVectorSize >= 32) {
5600     vpxor(xtmp, xtmp, xtmp, AVX_256bit);


5601   } else {
5602     pxor(xtmp, xtmp);

5603   }
5604   jmp(L_zero_64_bytes);
5605 
5606   BIND(L_loop);
5607   if (MaxVectorSize >= 32) {
5608     fill64(base, 0, xtmp, use64byteVector);
5609   } else {
5610     movdqu(Address(base,  0), xtmp);
5611     movdqu(Address(base, 16), xtmp);
5612     movdqu(Address(base, 32), xtmp);
5613     movdqu(Address(base, 48), xtmp);
5614   }
5615   addptr(base, 64);
5616 
5617   BIND(L_zero_64_bytes);
5618   subptr(cnt, 8);
5619   jccb(Assembler::greaterEqual, L_loop);
5620 
5621   // Copy trailing 64 bytes
5622   if (use64byteVector) {
5623     addptr(cnt, 8);
5624     jccb(Assembler::equal, L_end);
5625     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
5626     jmp(L_end);
5627   } else {
5628     addptr(cnt, 4);
5629     jccb(Assembler::less, L_tail);
5630     if (MaxVectorSize >= 32) {
5631       vmovdqu(Address(base, 0), xtmp);
5632     } else {
5633       movdqu(Address(base,  0), xtmp);
5634       movdqu(Address(base, 16), xtmp);
5635     }
5636   }
5637   addptr(base, 32);
5638   subptr(cnt, 4);
5639 
5640   BIND(L_tail);
5641   addptr(cnt, 4);
5642   jccb(Assembler::lessEqual, L_end);
5643   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
5644     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
5645   } else {
5646     decrement(cnt);
5647 
5648     BIND(L_sloop);
5649     movq(Address(base, 0), xtmp);
5650     addptr(base, 8);
5651     decrement(cnt);
5652     jccb(Assembler::greaterEqual, L_sloop);
5653   }
5654   BIND(L_end);
5655 }
5656 
5657 // Clearing constant sized memory using YMM/ZMM registers.
5658 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5659   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
5660   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
5661 
5662   int vector64_count = (cnt & (~0x7)) >> 3;
5663   cnt = cnt & 0x7;
5664   const int fill64_per_loop = 4;
5665   const int max_unrolled_fill64 = 8;
5666 
5667   // 64 byte initialization loop.
5668   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
5669   int start64 = 0;
5670   if (vector64_count > max_unrolled_fill64) {
5671     Label LOOP;
5672     Register index = rtmp;
5673 
5674     start64 = vector64_count - (vector64_count % fill64_per_loop);
5675 
5676     movl(index, 0);
5677     BIND(LOOP);
5678     for (int i = 0; i < fill64_per_loop; i++) {
5679       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
5680     }
5681     addl(index, fill64_per_loop * 64);
5682     cmpl(index, start64 * 64);
5683     jccb(Assembler::less, LOOP);
5684   }
5685   for (int i = start64; i < vector64_count; i++) {
5686     fill64(base, i * 64, xtmp, use64byteVector);
5687   }
5688 
5689   // Clear remaining 64 byte tail.
5690   int disp = vector64_count * 64;
5691   if (cnt) {
5692     switch (cnt) {
5693       case 1:
5694         movq(Address(base, disp), xtmp);
5695         break;
5696       case 2:
5697         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
5698         break;
5699       case 3:
5700         movl(rtmp, 0x7);
5701         kmovwl(mask, rtmp);
5702         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
5703         break;
5704       case 4:
5705         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5706         break;
5707       case 5:
5708         if (use64byteVector) {
5709           movl(rtmp, 0x1F);
5710           kmovwl(mask, rtmp);
5711           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5712         } else {
5713           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5714           movq(Address(base, disp + 32), xtmp);
5715         }
5716         break;
5717       case 6:
5718         if (use64byteVector) {
5719           movl(rtmp, 0x3F);
5720           kmovwl(mask, rtmp);
5721           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5722         } else {
5723           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5724           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
5725         }
5726         break;
5727       case 7:
5728         if (use64byteVector) {
5729           movl(rtmp, 0x7F);
5730           kmovwl(mask, rtmp);
5731           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
5732         } else {
5733           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
5734           movl(rtmp, 0x7);
5735           kmovwl(mask, rtmp);
5736           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
5737         }
5738         break;
5739       default:
5740         fatal("Unexpected length : %d\n",cnt);
5741         break;
5742     }
5743   }
5744 }
5745 
5746 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
5747                                bool is_large, KRegister mask) {
5748   // cnt      - number of qwords (8-byte words).
5749   // base     - start address, qword aligned.
5750   // is_large - if optimizers know cnt is larger than InitArrayShortSize
5751   assert(base==rdi, "base register must be edi for rep stos");
5752   assert(tmp==rax,   "tmp register must be eax for rep stos");
5753   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
5754   assert(InitArrayShortSize % BytesPerLong == 0,
5755     "InitArrayShortSize should be the multiple of BytesPerLong");
5756 
5757   Label DONE;
5758   if (!is_large || !UseXMMForObjInit) {
5759     xorptr(tmp, tmp);
5760   }
5761 
5762   if (!is_large) {
5763     Label LOOP, LONG;
5764     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
5765     jccb(Assembler::greater, LONG);
5766 
5767     decrement(cnt);
5768     jccb(Assembler::negative, DONE); // Zero length
5769 
5770     // Use individual pointer-sized stores for small counts:
5771     BIND(LOOP);
5772     movptr(Address(base, cnt, Address::times_ptr), tmp);
5773     decrement(cnt);
5774     jccb(Assembler::greaterEqual, LOOP);
5775     jmpb(DONE);
5776 
5777     BIND(LONG);
5778   }
5779 
5780   // Use longer rep-prefixed ops for non-small counts:
5781   if (UseFastStosb) {
5782     shlptr(cnt, 3); // convert to number of bytes
5783     rep_stosb();
5784   } else if (UseXMMForObjInit) {
5785     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
5786   } else {
5787     rep_stos();
5788   }
5789 
5790   BIND(DONE);
5791 }
5792 
5793 #endif //COMPILER2_OR_JVMCI
5794 
5795 
5796 void MacroAssembler::generate_fill(BasicType t, bool aligned,
5797                                    Register to, Register value, Register count,
5798                                    Register rtmp, XMMRegister xtmp) {
5799   ShortBranchVerifier sbv(this);
5800   assert_different_registers(to, value, count, rtmp);
5801   Label L_exit;
5802   Label L_fill_2_bytes, L_fill_4_bytes;
5803 
5804 #if defined(COMPILER2)
5805   if(MaxVectorSize >=32 &&
5806      VM_Version::supports_avx512vlbw() &&
5807      VM_Version::supports_bmi2()) {
5808     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
5809     return;
5810   }
5811 #endif
5812 
5813   int shift = -1;
5814   switch (t) {
5815     case T_BYTE:
5816       shift = 2;
5817       break;
5818     case T_SHORT:
5819       shift = 1;
5820       break;
5821     case T_INT:
5822       shift = 0;
5823       break;
5824     default: ShouldNotReachHere();
5825   }
5826 
5827   if (t == T_BYTE) {
5828     andl(value, 0xff);
5829     movl(rtmp, value);
5830     shll(rtmp, 8);
5831     orl(value, rtmp);
5832   }
5833   if (t == T_SHORT) {
5834     andl(value, 0xffff);
5835   }
5836   if (t == T_BYTE || t == T_SHORT) {
5837     movl(rtmp, value);
5838     shll(rtmp, 16);
5839     orl(value, rtmp);
5840   }
5841 
5842   cmpptr(count, 8 << shift); // Short arrays (< 32 bytes) fill by element
5843   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
5844   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
5845     Label L_skip_align2;
5846     // align source address at 4 bytes address boundary
5847     if (t == T_BYTE) {
5848       Label L_skip_align1;
5849       // One byte misalignment happens only for byte arrays
5850       testptr(to, 1);
5851       jccb(Assembler::zero, L_skip_align1);
5852       movb(Address(to, 0), value);
5853       increment(to);
5854       decrement(count);
5855       BIND(L_skip_align1);
5856     }
5857     // Two bytes misalignment happens only for byte and short (char) arrays
5858     testptr(to, 2);
5859     jccb(Assembler::zero, L_skip_align2);
5860     movw(Address(to, 0), value);
5861     addptr(to, 2);
5862     subptr(count, 1<<(shift-1));
5863     BIND(L_skip_align2);
5864   }
5865   {
5866     Label L_fill_32_bytes;
5867     if (!UseUnalignedLoadStores) {
5868       // align to 8 bytes, we know we are 4 byte aligned to start
5869       testptr(to, 4);
5870       jccb(Assembler::zero, L_fill_32_bytes);
5871       movl(Address(to, 0), value);
5872       addptr(to, 4);
5873       subptr(count, 1<<shift);
5874     }
5875     BIND(L_fill_32_bytes);
5876     {
5877       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
5878       movdl(xtmp, value);
5879       if (UseAVX >= 2 && UseUnalignedLoadStores) {
5880         Label L_check_fill_32_bytes;
5881         if (UseAVX > 2) {
5882           // Fill 64-byte chunks
5883           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
5884 
5885           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
5886           cmpptr(count, VM_Version::avx3_threshold());
5887           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
5888 
5889           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
5890 
5891           subptr(count, 16 << shift);
5892           jccb(Assembler::less, L_check_fill_32_bytes);
5893           align(16);
5894 
5895           BIND(L_fill_64_bytes_loop_avx3);
5896           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
5897           addptr(to, 64);
5898           subptr(count, 16 << shift);
5899           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
5900           jmpb(L_check_fill_32_bytes);
5901 
5902           BIND(L_check_fill_64_bytes_avx2);
5903         }
5904         // Fill 64-byte chunks
5905         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
5906 
5907         subptr(count, 16 << shift);
5908         jcc(Assembler::less, L_check_fill_32_bytes);
5909 
5910         // align data for 64-byte chunks
5911         Label L_fill_64_bytes_loop, L_align_64_bytes_loop;
5912         if (EnableX86ECoreOpts) {
5913             // align 'big' arrays to cache lines to minimize split_stores
5914             cmpptr(count, 96 << shift);
5915             jcc(Assembler::below, L_fill_64_bytes_loop);
5916 
5917             // Find the bytes needed for alignment
5918             movptr(rtmp, to);
5919             andptr(rtmp, 0x1c);
5920             jcc(Assembler::zero, L_fill_64_bytes_loop);
5921             negptr(rtmp);           // number of bytes to fill 32-rtmp. it filled by 2 mov by 32
5922             addptr(rtmp, 32);
5923             shrptr(rtmp, 2 - shift);// get number of elements from bytes
5924             subptr(count, rtmp);    // adjust count by number of elements
5925 
5926             align(16);
5927             BIND(L_align_64_bytes_loop);
5928             movdl(Address(to, 0), xtmp);
5929             addptr(to, 4);
5930             subptr(rtmp, 1 << shift);
5931             jcc(Assembler::greater, L_align_64_bytes_loop);
5932         }
5933 
5934         align(16);
5935         BIND(L_fill_64_bytes_loop);
5936         vmovdqu(Address(to, 0), xtmp);
5937         vmovdqu(Address(to, 32), xtmp);
5938         addptr(to, 64);
5939         subptr(count, 16 << shift);
5940         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
5941 
5942         align(16);
5943         BIND(L_check_fill_32_bytes);
5944         addptr(count, 8 << shift);
5945         jccb(Assembler::less, L_check_fill_8_bytes);
5946         vmovdqu(Address(to, 0), xtmp);
5947         addptr(to, 32);
5948         subptr(count, 8 << shift);
5949 
5950         BIND(L_check_fill_8_bytes);
5951         // clean upper bits of YMM registers
5952         movdl(xtmp, value);
5953         pshufd(xtmp, xtmp, 0);
5954       } else {
5955         // Fill 32-byte chunks
5956         pshufd(xtmp, xtmp, 0);
5957 
5958         subptr(count, 8 << shift);
5959         jcc(Assembler::less, L_check_fill_8_bytes);
5960         align(16);
5961 
5962         BIND(L_fill_32_bytes_loop);
5963 
5964         if (UseUnalignedLoadStores) {
5965           movdqu(Address(to, 0), xtmp);
5966           movdqu(Address(to, 16), xtmp);
5967         } else {
5968           movq(Address(to, 0), xtmp);
5969           movq(Address(to, 8), xtmp);
5970           movq(Address(to, 16), xtmp);
5971           movq(Address(to, 24), xtmp);
5972         }
5973 
5974         addptr(to, 32);
5975         subptr(count, 8 << shift);
5976         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
5977 
5978         BIND(L_check_fill_8_bytes);
5979       }
5980       addptr(count, 8 << shift);
5981       jccb(Assembler::zero, L_exit);
5982       jmpb(L_fill_8_bytes);
5983 
5984       //
5985       // length is too short, just fill qwords
5986       //
5987       align(16);
5988       BIND(L_fill_8_bytes_loop);
5989       movq(Address(to, 0), xtmp);
5990       addptr(to, 8);
5991       BIND(L_fill_8_bytes);
5992       subptr(count, 1 << (shift + 1));
5993       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
5994     }
5995   }
5996 
5997   Label L_fill_4_bytes_loop;
5998   testl(count, 1 << shift);
5999   jccb(Assembler::zero, L_fill_2_bytes);
6000 
6001   align(16);
6002   BIND(L_fill_4_bytes_loop);
6003   movl(Address(to, 0), value);
6004   addptr(to, 4);
6005 
6006   BIND(L_fill_4_bytes);
6007   subptr(count, 1 << shift);
6008   jccb(Assembler::greaterEqual, L_fill_4_bytes_loop);
6009 
6010   if (t == T_BYTE || t == T_SHORT) {
6011     Label L_fill_byte;
6012     BIND(L_fill_2_bytes);
6013     // fill trailing 2 bytes
6014     testl(count, 1<<(shift-1));
6015     jccb(Assembler::zero, L_fill_byte);
6016     movw(Address(to, 0), value);
6017     if (t == T_BYTE) {
6018       addptr(to, 2);
6019       BIND(L_fill_byte);
6020       // fill trailing byte
6021       testl(count, 1);
6022       jccb(Assembler::zero, L_exit);
6023       movb(Address(to, 0), value);
6024     } else {
6025       BIND(L_fill_byte);
6026     }
6027   } else {
6028     BIND(L_fill_2_bytes);
6029   }
6030   BIND(L_exit);
6031 }
6032 
6033 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
6034   switch(type) {
6035     case T_BYTE:
6036     case T_BOOLEAN:
6037       evpbroadcastb(dst, src, vector_len);
6038       break;
6039     case T_SHORT:
6040     case T_CHAR:
6041       evpbroadcastw(dst, src, vector_len);
6042       break;
6043     case T_INT:
6044     case T_FLOAT:
6045       evpbroadcastd(dst, src, vector_len);
6046       break;
6047     case T_LONG:
6048     case T_DOUBLE:
6049       evpbroadcastq(dst, src, vector_len);
6050       break;
6051     default:
6052       fatal("Unhandled type : %s", type2name(type));
6053       break;
6054   }
6055 }
6056 
6057 // Encode given char[]/byte[] to byte[] in ISO_8859_1 or ASCII
6058 //
6059 // @IntrinsicCandidate
6060 // int sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(
6061 //         char[] sa, int sp, byte[] da, int dp, int len) {
6062 //     int i = 0;
6063 //     for (; i < len; i++) {
6064 //         char c = sa[sp++];
6065 //         if (c > '\u00FF')
6066 //             break;
6067 //         da[dp++] = (byte) c;
6068 //     }
6069 //     return i;
6070 // }
6071 //
6072 // @IntrinsicCandidate
6073 // int java.lang.StringCoding.encodeISOArray0(
6074 //         byte[] sa, int sp, byte[] da, int dp, int len) {
6075 //   int i = 0;
6076 //   for (; i < len; i++) {
6077 //     char c = StringUTF16.getChar(sa, sp++);
6078 //     if (c > '\u00FF')
6079 //       break;
6080 //     da[dp++] = (byte) c;
6081 //   }
6082 //   return i;
6083 // }
6084 //
6085 // @IntrinsicCandidate
6086 // int java.lang.StringCoding.encodeAsciiArray0(
6087 //         char[] sa, int sp, byte[] da, int dp, int len) {
6088 //   int i = 0;
6089 //   for (; i < len; i++) {
6090 //     char c = sa[sp++];
6091 //     if (c >= '\u0080')
6092 //       break;
6093 //     da[dp++] = (byte) c;
6094 //   }
6095 //   return i;
6096 // }
6097 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
6098   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
6099   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
6100   Register tmp5, Register result, bool ascii) {
6101 
6102   // rsi: src
6103   // rdi: dst
6104   // rdx: len
6105   // rcx: tmp5
6106   // rax: result
6107   ShortBranchVerifier sbv(this);
6108   assert_different_registers(src, dst, len, tmp5, result);
6109   Label L_done, L_copy_1_char, L_copy_1_char_exit;
6110 
6111   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
6112   int short_mask = ascii ? 0xff80 : 0xff00;
6113 
6114   // set result
6115   xorl(result, result);
6116   // check for zero length
6117   testl(len, len);
6118   jcc(Assembler::zero, L_done);
6119 
6120   movl(result, len);
6121 
6122   // Setup pointers
6123   lea(src, Address(src, len, Address::times_2)); // char[]
6124   lea(dst, Address(dst, len, Address::times_1)); // byte[]
6125   negptr(len);
6126 
6127   if (UseSSE42Intrinsics || UseAVX >= 2) {
6128     Label L_copy_8_chars, L_copy_8_chars_exit;
6129     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
6130 
6131     if (UseAVX >= 2) {
6132       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
6133       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
6134       movdl(tmp1Reg, tmp5);
6135       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
6136       jmp(L_chars_32_check);
6137 
6138       bind(L_copy_32_chars);
6139       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
6140       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
6141       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
6142       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
6143       jccb(Assembler::notZero, L_copy_32_chars_exit);
6144       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
6145       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
6146       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
6147 
6148       bind(L_chars_32_check);
6149       addptr(len, 32);
6150       jcc(Assembler::lessEqual, L_copy_32_chars);
6151 
6152       bind(L_copy_32_chars_exit);
6153       subptr(len, 16);
6154       jccb(Assembler::greater, L_copy_16_chars_exit);
6155 
6156     } else if (UseSSE42Intrinsics) {
6157       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
6158       movdl(tmp1Reg, tmp5);
6159       pshufd(tmp1Reg, tmp1Reg, 0);
6160       jmpb(L_chars_16_check);
6161     }
6162 
6163     bind(L_copy_16_chars);
6164     if (UseAVX >= 2) {
6165       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
6166       vptest(tmp2Reg, tmp1Reg);
6167       jcc(Assembler::notZero, L_copy_16_chars_exit);
6168       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
6169       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
6170     } else {
6171       if (UseAVX > 0) {
6172         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6173         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6174         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
6175       } else {
6176         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6177         por(tmp2Reg, tmp3Reg);
6178         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6179         por(tmp2Reg, tmp4Reg);
6180       }
6181       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
6182       jccb(Assembler::notZero, L_copy_16_chars_exit);
6183       packuswb(tmp3Reg, tmp4Reg);
6184     }
6185     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
6186 
6187     bind(L_chars_16_check);
6188     addptr(len, 16);
6189     jcc(Assembler::lessEqual, L_copy_16_chars);
6190 
6191     bind(L_copy_16_chars_exit);
6192     if (UseAVX >= 2) {
6193       // clean upper bits of YMM registers
6194       vpxor(tmp2Reg, tmp2Reg);
6195       vpxor(tmp3Reg, tmp3Reg);
6196       vpxor(tmp4Reg, tmp4Reg);
6197       movdl(tmp1Reg, tmp5);
6198       pshufd(tmp1Reg, tmp1Reg, 0);
6199     }
6200     subptr(len, 8);
6201     jccb(Assembler::greater, L_copy_8_chars_exit);
6202 
6203     bind(L_copy_8_chars);
6204     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
6205     ptest(tmp3Reg, tmp1Reg);
6206     jccb(Assembler::notZero, L_copy_8_chars_exit);
6207     packuswb(tmp3Reg, tmp1Reg);
6208     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
6209     addptr(len, 8);
6210     jccb(Assembler::lessEqual, L_copy_8_chars);
6211 
6212     bind(L_copy_8_chars_exit);
6213     subptr(len, 8);
6214     jccb(Assembler::zero, L_done);
6215   }
6216 
6217   bind(L_copy_1_char);
6218   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
6219   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
6220   jccb(Assembler::notZero, L_copy_1_char_exit);
6221   movb(Address(dst, len, Address::times_1, 0), tmp5);
6222   addptr(len, 1);
6223   jccb(Assembler::less, L_copy_1_char);
6224 
6225   bind(L_copy_1_char_exit);
6226   addptr(result, len); // len is negative count of not processed elements
6227 
6228   bind(L_done);
6229 }
6230 
6231 /**
6232  * Helper for multiply_to_len().
6233  */
6234 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
6235   addq(dest_lo, src1);
6236   adcq(dest_hi, 0);
6237   addq(dest_lo, src2);
6238   adcq(dest_hi, 0);
6239 }
6240 
6241 /**
6242  * Multiply 64 bit by 64 bit first loop.
6243  */
6244 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
6245                                            Register y, Register y_idx, Register z,
6246                                            Register carry, Register product,
6247                                            Register idx, Register kdx) {
6248   //
6249   //  jlong carry, x[], y[], z[];
6250   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
6251   //    huge_128 product = y[idx] * x[xstart] + carry;
6252   //    z[kdx] = (jlong)product;
6253   //    carry  = (jlong)(product >>> 64);
6254   //  }
6255   //  z[xstart] = carry;
6256   //
6257 
6258   Label L_first_loop, L_first_loop_exit;
6259   Label L_one_x, L_one_y, L_multiply;
6260 
6261   decrementl(xstart);
6262   jcc(Assembler::negative, L_one_x);
6263 
6264   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
6265   rorq(x_xstart, 32); // convert big-endian to little-endian
6266 
6267   bind(L_first_loop);
6268   decrementl(idx);
6269   jcc(Assembler::negative, L_first_loop_exit);
6270   decrementl(idx);
6271   jcc(Assembler::negative, L_one_y);
6272   movq(y_idx, Address(y, idx, Address::times_4,  0));
6273   rorq(y_idx, 32); // convert big-endian to little-endian
6274   bind(L_multiply);
6275   movq(product, x_xstart);
6276   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
6277   addq(product, carry);
6278   adcq(rdx, 0);
6279   subl(kdx, 2);
6280   movl(Address(z, kdx, Address::times_4,  4), product);
6281   shrq(product, 32);
6282   movl(Address(z, kdx, Address::times_4,  0), product);
6283   movq(carry, rdx);
6284   jmp(L_first_loop);
6285 
6286   bind(L_one_y);
6287   movl(y_idx, Address(y,  0));
6288   jmp(L_multiply);
6289 
6290   bind(L_one_x);
6291   movl(x_xstart, Address(x,  0));
6292   jmp(L_first_loop);
6293 
6294   bind(L_first_loop_exit);
6295 }
6296 
6297 /**
6298  * Multiply 64 bit by 64 bit and add 128 bit.
6299  */
6300 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
6301                                             Register yz_idx, Register idx,
6302                                             Register carry, Register product, int offset) {
6303   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
6304   //     z[kdx] = (jlong)product;
6305 
6306   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
6307   rorq(yz_idx, 32); // convert big-endian to little-endian
6308   movq(product, x_xstart);
6309   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
6310   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
6311   rorq(yz_idx, 32); // convert big-endian to little-endian
6312 
6313   add2_with_carry(rdx, product, carry, yz_idx);
6314 
6315   movl(Address(z, idx, Address::times_4,  offset+4), product);
6316   shrq(product, 32);
6317   movl(Address(z, idx, Address::times_4,  offset), product);
6318 
6319 }
6320 
6321 /**
6322  * Multiply 128 bit by 128 bit. Unrolled inner loop.
6323  */
6324 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
6325                                              Register yz_idx, Register idx, Register jdx,
6326                                              Register carry, Register product,
6327                                              Register carry2) {
6328   //   jlong carry, x[], y[], z[];
6329   //   int kdx = ystart+1;
6330   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
6331   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
6332   //     z[kdx+idx+1] = (jlong)product;
6333   //     jlong carry2  = (jlong)(product >>> 64);
6334   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
6335   //     z[kdx+idx] = (jlong)product;
6336   //     carry  = (jlong)(product >>> 64);
6337   //   }
6338   //   idx += 2;
6339   //   if (idx > 0) {
6340   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
6341   //     z[kdx+idx] = (jlong)product;
6342   //     carry  = (jlong)(product >>> 64);
6343   //   }
6344   //
6345 
6346   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
6347 
6348   movl(jdx, idx);
6349   andl(jdx, 0xFFFFFFFC);
6350   shrl(jdx, 2);
6351 
6352   bind(L_third_loop);
6353   subl(jdx, 1);
6354   jcc(Assembler::negative, L_third_loop_exit);
6355   subl(idx, 4);
6356 
6357   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
6358   movq(carry2, rdx);
6359 
6360   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
6361   movq(carry, rdx);
6362   jmp(L_third_loop);
6363 
6364   bind (L_third_loop_exit);
6365 
6366   andl (idx, 0x3);
6367   jcc(Assembler::zero, L_post_third_loop_done);
6368 
6369   Label L_check_1;
6370   subl(idx, 2);
6371   jcc(Assembler::negative, L_check_1);
6372 
6373   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
6374   movq(carry, rdx);
6375 
6376   bind (L_check_1);
6377   addl (idx, 0x2);
6378   andl (idx, 0x1);
6379   subl(idx, 1);
6380   jcc(Assembler::negative, L_post_third_loop_done);
6381 
6382   movl(yz_idx, Address(y, idx, Address::times_4,  0));
6383   movq(product, x_xstart);
6384   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
6385   movl(yz_idx, Address(z, idx, Address::times_4,  0));
6386 
6387   add2_with_carry(rdx, product, yz_idx, carry);
6388 
6389   movl(Address(z, idx, Address::times_4,  0), product);
6390   shrq(product, 32);
6391 
6392   shlq(rdx, 32);
6393   orq(product, rdx);
6394   movq(carry, product);
6395 
6396   bind(L_post_third_loop_done);
6397 }
6398 
6399 /**
6400  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
6401  *
6402  */
6403 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
6404                                                   Register carry, Register carry2,
6405                                                   Register idx, Register jdx,
6406                                                   Register yz_idx1, Register yz_idx2,
6407                                                   Register tmp, Register tmp3, Register tmp4) {
6408   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
6409 
6410   //   jlong carry, x[], y[], z[];
6411   //   int kdx = ystart+1;
6412   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
6413   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
6414   //     jlong carry2  = (jlong)(tmp3 >>> 64);
6415   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
6416   //     carry  = (jlong)(tmp4 >>> 64);
6417   //     z[kdx+idx+1] = (jlong)tmp3;
6418   //     z[kdx+idx] = (jlong)tmp4;
6419   //   }
6420   //   idx += 2;
6421   //   if (idx > 0) {
6422   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
6423   //     z[kdx+idx] = (jlong)yz_idx1;
6424   //     carry  = (jlong)(yz_idx1 >>> 64);
6425   //   }
6426   //
6427 
6428   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
6429 
6430   movl(jdx, idx);
6431   andl(jdx, 0xFFFFFFFC);
6432   shrl(jdx, 2);
6433 
6434   bind(L_third_loop);
6435   subl(jdx, 1);
6436   jcc(Assembler::negative, L_third_loop_exit);
6437   subl(idx, 4);
6438 
6439   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
6440   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
6441   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
6442   rorxq(yz_idx2, yz_idx2, 32);
6443 
6444   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
6445   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
6446 
6447   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
6448   rorxq(yz_idx1, yz_idx1, 32);
6449   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
6450   rorxq(yz_idx2, yz_idx2, 32);
6451 
6452   if (VM_Version::supports_adx()) {
6453     adcxq(tmp3, carry);
6454     adoxq(tmp3, yz_idx1);
6455 
6456     adcxq(tmp4, tmp);
6457     adoxq(tmp4, yz_idx2);
6458 
6459     movl(carry, 0); // does not affect flags
6460     adcxq(carry2, carry);
6461     adoxq(carry2, carry);
6462   } else {
6463     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
6464     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
6465   }
6466   movq(carry, carry2);
6467 
6468   movl(Address(z, idx, Address::times_4, 12), tmp3);
6469   shrq(tmp3, 32);
6470   movl(Address(z, idx, Address::times_4,  8), tmp3);
6471 
6472   movl(Address(z, idx, Address::times_4,  4), tmp4);
6473   shrq(tmp4, 32);
6474   movl(Address(z, idx, Address::times_4,  0), tmp4);
6475 
6476   jmp(L_third_loop);
6477 
6478   bind (L_third_loop_exit);
6479 
6480   andl (idx, 0x3);
6481   jcc(Assembler::zero, L_post_third_loop_done);
6482 
6483   Label L_check_1;
6484   subl(idx, 2);
6485   jcc(Assembler::negative, L_check_1);
6486 
6487   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
6488   rorxq(yz_idx1, yz_idx1, 32);
6489   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
6490   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
6491   rorxq(yz_idx2, yz_idx2, 32);
6492 
6493   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
6494 
6495   movl(Address(z, idx, Address::times_4,  4), tmp3);
6496   shrq(tmp3, 32);
6497   movl(Address(z, idx, Address::times_4,  0), tmp3);
6498   movq(carry, tmp4);
6499 
6500   bind (L_check_1);
6501   addl (idx, 0x2);
6502   andl (idx, 0x1);
6503   subl(idx, 1);
6504   jcc(Assembler::negative, L_post_third_loop_done);
6505   movl(tmp4, Address(y, idx, Address::times_4,  0));
6506   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
6507   movl(tmp4, Address(z, idx, Address::times_4,  0));
6508 
6509   add2_with_carry(carry2, tmp3, tmp4, carry);
6510 
6511   movl(Address(z, idx, Address::times_4,  0), tmp3);
6512   shrq(tmp3, 32);
6513 
6514   shlq(carry2, 32);
6515   orq(tmp3, carry2);
6516   movq(carry, tmp3);
6517 
6518   bind(L_post_third_loop_done);
6519 }
6520 
6521 /**
6522  * Code for BigInteger::multiplyToLen() intrinsic.
6523  *
6524  * rdi: x
6525  * rax: xlen
6526  * rsi: y
6527  * rcx: ylen
6528  * r8:  z
6529  * r11: tmp0
6530  * r12: tmp1
6531  * r13: tmp2
6532  * r14: tmp3
6533  * r15: tmp4
6534  * rbx: tmp5
6535  *
6536  */
6537 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
6538                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
6539   ShortBranchVerifier sbv(this);
6540   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
6541 
6542   push(tmp0);
6543   push(tmp1);
6544   push(tmp2);
6545   push(tmp3);
6546   push(tmp4);
6547   push(tmp5);
6548 
6549   push(xlen);
6550 
6551   const Register idx = tmp1;
6552   const Register kdx = tmp2;
6553   const Register xstart = tmp3;
6554 
6555   const Register y_idx = tmp4;
6556   const Register carry = tmp5;
6557   const Register product  = xlen;
6558   const Register x_xstart = tmp0;
6559 
6560   // First Loop.
6561   //
6562   //  final static long LONG_MASK = 0xffffffffL;
6563   //  int xstart = xlen - 1;
6564   //  int ystart = ylen - 1;
6565   //  long carry = 0;
6566   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
6567   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
6568   //    z[kdx] = (int)product;
6569   //    carry = product >>> 32;
6570   //  }
6571   //  z[xstart] = (int)carry;
6572   //
6573 
6574   movl(idx, ylen);               // idx = ylen;
6575   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
6576   xorq(carry, carry);            // carry = 0;
6577 
6578   Label L_done;
6579 
6580   movl(xstart, xlen);
6581   decrementl(xstart);
6582   jcc(Assembler::negative, L_done);
6583 
6584   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
6585 
6586   Label L_second_loop;
6587   testl(kdx, kdx);
6588   jcc(Assembler::zero, L_second_loop);
6589 
6590   Label L_carry;
6591   subl(kdx, 1);
6592   jcc(Assembler::zero, L_carry);
6593 
6594   movl(Address(z, kdx, Address::times_4,  0), carry);
6595   shrq(carry, 32);
6596   subl(kdx, 1);
6597 
6598   bind(L_carry);
6599   movl(Address(z, kdx, Address::times_4,  0), carry);
6600 
6601   // Second and third (nested) loops.
6602   //
6603   // for (int i = xstart-1; i >= 0; i--) { // Second loop
6604   //   carry = 0;
6605   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
6606   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
6607   //                    (z[k] & LONG_MASK) + carry;
6608   //     z[k] = (int)product;
6609   //     carry = product >>> 32;
6610   //   }
6611   //   z[i] = (int)carry;
6612   // }
6613   //
6614   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
6615 
6616   const Register jdx = tmp1;
6617 
6618   bind(L_second_loop);
6619   xorl(carry, carry);    // carry = 0;
6620   movl(jdx, ylen);       // j = ystart+1
6621 
6622   subl(xstart, 1);       // i = xstart-1;
6623   jcc(Assembler::negative, L_done);
6624 
6625   push (z);
6626 
6627   Label L_last_x;
6628   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
6629   subl(xstart, 1);       // i = xstart-1;
6630   jcc(Assembler::negative, L_last_x);
6631 
6632   if (UseBMI2Instructions) {
6633     movq(rdx,  Address(x, xstart, Address::times_4,  0));
6634     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
6635   } else {
6636     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
6637     rorq(x_xstart, 32);  // convert big-endian to little-endian
6638   }
6639 
6640   Label L_third_loop_prologue;
6641   bind(L_third_loop_prologue);
6642 
6643   push (x);
6644   push (xstart);
6645   push (ylen);
6646 
6647 
6648   if (UseBMI2Instructions) {
6649     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
6650   } else { // !UseBMI2Instructions
6651     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
6652   }
6653 
6654   pop(ylen);
6655   pop(xlen);
6656   pop(x);
6657   pop(z);
6658 
6659   movl(tmp3, xlen);
6660   addl(tmp3, 1);
6661   movl(Address(z, tmp3, Address::times_4,  0), carry);
6662   subl(tmp3, 1);
6663   jccb(Assembler::negative, L_done);
6664 
6665   shrq(carry, 32);
6666   movl(Address(z, tmp3, Address::times_4,  0), carry);
6667   jmp(L_second_loop);
6668 
6669   // Next infrequent code is moved outside loops.
6670   bind(L_last_x);
6671   if (UseBMI2Instructions) {
6672     movl(rdx, Address(x,  0));
6673   } else {
6674     movl(x_xstart, Address(x,  0));
6675   }
6676   jmp(L_third_loop_prologue);
6677 
6678   bind(L_done);
6679 
6680   pop(xlen);
6681 
6682   pop(tmp5);
6683   pop(tmp4);
6684   pop(tmp3);
6685   pop(tmp2);
6686   pop(tmp1);
6687   pop(tmp0);
6688 }
6689 
6690 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
6691   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
6692   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
6693   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
6694   Label VECTOR8_TAIL, VECTOR4_TAIL;
6695   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
6696   Label SAME_TILL_END, DONE;
6697   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
6698 
6699   //scale is in rcx in both Win64 and Unix
6700   ShortBranchVerifier sbv(this);
6701 
6702   shlq(length);
6703   xorq(result, result);
6704 
6705   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
6706       VM_Version::supports_avx512vlbw()) {
6707     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
6708 
6709     cmpq(length, 64);
6710     jcc(Assembler::less, VECTOR32_TAIL);
6711 
6712     movq(tmp1, length);
6713     andq(tmp1, 0x3F);      // tail count
6714     andq(length, ~(0x3F)); //vector count
6715 
6716     bind(VECTOR64_LOOP);
6717     // AVX512 code to compare 64 byte vectors.
6718     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
6719     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
6720     kortestql(k7, k7);
6721     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
6722     addq(result, 64);
6723     subq(length, 64);
6724     jccb(Assembler::notZero, VECTOR64_LOOP);
6725 
6726     //bind(VECTOR64_TAIL);
6727     testq(tmp1, tmp1);
6728     jcc(Assembler::zero, SAME_TILL_END);
6729 
6730     //bind(VECTOR64_TAIL);
6731     // AVX512 code to compare up to 63 byte vectors.
6732     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
6733     shlxq(tmp2, tmp2, tmp1);
6734     notq(tmp2);
6735     kmovql(k3, tmp2);
6736 
6737     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
6738     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
6739 
6740     ktestql(k7, k3);
6741     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
6742 
6743     bind(VECTOR64_NOT_EQUAL);
6744     kmovql(tmp1, k7);
6745     notq(tmp1);
6746     tzcntq(tmp1, tmp1);
6747     addq(result, tmp1);
6748     shrq(result);
6749     jmp(DONE);
6750     bind(VECTOR32_TAIL);
6751   }
6752 
6753   cmpq(length, 8);
6754   jcc(Assembler::equal, VECTOR8_LOOP);
6755   jcc(Assembler::less, VECTOR4_TAIL);
6756 
6757   if (UseAVX >= 2) {
6758     Label VECTOR16_TAIL, VECTOR32_LOOP;
6759 
6760     cmpq(length, 16);
6761     jcc(Assembler::equal, VECTOR16_LOOP);
6762     jcc(Assembler::less, VECTOR8_LOOP);
6763 
6764     cmpq(length, 32);
6765     jccb(Assembler::less, VECTOR16_TAIL);
6766 
6767     subq(length, 32);
6768     bind(VECTOR32_LOOP);
6769     vmovdqu(rymm0, Address(obja, result));
6770     vmovdqu(rymm1, Address(objb, result));
6771     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
6772     vptest(rymm2, rymm2);
6773     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
6774     addq(result, 32);
6775     subq(length, 32);
6776     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
6777     addq(length, 32);
6778     jcc(Assembler::equal, SAME_TILL_END);
6779     //falling through if less than 32 bytes left //close the branch here.
6780 
6781     bind(VECTOR16_TAIL);
6782     cmpq(length, 16);
6783     jccb(Assembler::less, VECTOR8_TAIL);
6784     bind(VECTOR16_LOOP);
6785     movdqu(rymm0, Address(obja, result));
6786     movdqu(rymm1, Address(objb, result));
6787     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
6788     ptest(rymm2, rymm2);
6789     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6790     addq(result, 16);
6791     subq(length, 16);
6792     jcc(Assembler::equal, SAME_TILL_END);
6793     //falling through if less than 16 bytes left
6794   } else {//regular intrinsics
6795 
6796     cmpq(length, 16);
6797     jccb(Assembler::less, VECTOR8_TAIL);
6798 
6799     subq(length, 16);
6800     bind(VECTOR16_LOOP);
6801     movdqu(rymm0, Address(obja, result));
6802     movdqu(rymm1, Address(objb, result));
6803     pxor(rymm0, rymm1);
6804     ptest(rymm0, rymm0);
6805     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
6806     addq(result, 16);
6807     subq(length, 16);
6808     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
6809     addq(length, 16);
6810     jcc(Assembler::equal, SAME_TILL_END);
6811     //falling through if less than 16 bytes left
6812   }
6813 
6814   bind(VECTOR8_TAIL);
6815   cmpq(length, 8);
6816   jccb(Assembler::less, VECTOR4_TAIL);
6817   bind(VECTOR8_LOOP);
6818   movq(tmp1, Address(obja, result));
6819   movq(tmp2, Address(objb, result));
6820   xorq(tmp1, tmp2);
6821   testq(tmp1, tmp1);
6822   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
6823   addq(result, 8);
6824   subq(length, 8);
6825   jcc(Assembler::equal, SAME_TILL_END);
6826   //falling through if less than 8 bytes left
6827 
6828   bind(VECTOR4_TAIL);
6829   cmpq(length, 4);
6830   jccb(Assembler::less, BYTES_TAIL);
6831   bind(VECTOR4_LOOP);
6832   movl(tmp1, Address(obja, result));
6833   xorl(tmp1, Address(objb, result));
6834   testl(tmp1, tmp1);
6835   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
6836   addq(result, 4);
6837   subq(length, 4);
6838   jcc(Assembler::equal, SAME_TILL_END);
6839   //falling through if less than 4 bytes left
6840 
6841   bind(BYTES_TAIL);
6842   bind(BYTES_LOOP);
6843   load_unsigned_byte(tmp1, Address(obja, result));
6844   load_unsigned_byte(tmp2, Address(objb, result));
6845   xorl(tmp1, tmp2);
6846   testl(tmp1, tmp1);
6847   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6848   decq(length);
6849   jcc(Assembler::zero, SAME_TILL_END);
6850   incq(result);
6851   load_unsigned_byte(tmp1, Address(obja, result));
6852   load_unsigned_byte(tmp2, Address(objb, result));
6853   xorl(tmp1, tmp2);
6854   testl(tmp1, tmp1);
6855   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6856   decq(length);
6857   jcc(Assembler::zero, SAME_TILL_END);
6858   incq(result);
6859   load_unsigned_byte(tmp1, Address(obja, result));
6860   load_unsigned_byte(tmp2, Address(objb, result));
6861   xorl(tmp1, tmp2);
6862   testl(tmp1, tmp1);
6863   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
6864   jmp(SAME_TILL_END);
6865 
6866   if (UseAVX >= 2) {
6867     bind(VECTOR32_NOT_EQUAL);
6868     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
6869     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
6870     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
6871     vpmovmskb(tmp1, rymm0);
6872     bsfq(tmp1, tmp1);
6873     addq(result, tmp1);
6874     shrq(result);
6875     jmp(DONE);
6876   }
6877 
6878   bind(VECTOR16_NOT_EQUAL);
6879   if (UseAVX >= 2) {
6880     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
6881     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
6882     pxor(rymm0, rymm2);
6883   } else {
6884     pcmpeqb(rymm2, rymm2);
6885     pxor(rymm0, rymm1);
6886     pcmpeqb(rymm0, rymm1);
6887     pxor(rymm0, rymm2);
6888   }
6889   pmovmskb(tmp1, rymm0);
6890   bsfq(tmp1, tmp1);
6891   addq(result, tmp1);
6892   shrq(result);
6893   jmpb(DONE);
6894 
6895   bind(VECTOR8_NOT_EQUAL);
6896   bind(VECTOR4_NOT_EQUAL);
6897   bsfq(tmp1, tmp1);
6898   shrq(tmp1, 3);
6899   addq(result, tmp1);
6900   bind(BYTES_NOT_EQUAL);
6901   shrq(result);
6902   jmpb(DONE);
6903 
6904   bind(SAME_TILL_END);
6905   mov64(result, -1);
6906 
6907   bind(DONE);
6908 }
6909 
6910 //Helper functions for square_to_len()
6911 
6912 /**
6913  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
6914  * Preserves x and z and modifies rest of the registers.
6915  */
6916 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
6917   // Perform square and right shift by 1
6918   // Handle odd xlen case first, then for even xlen do the following
6919   // jlong carry = 0;
6920   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
6921   //     huge_128 product = x[j:j+1] * x[j:j+1];
6922   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
6923   //     z[i+2:i+3] = (jlong)(product >>> 1);
6924   //     carry = (jlong)product;
6925   // }
6926 
6927   xorq(tmp5, tmp5);     // carry
6928   xorq(rdxReg, rdxReg);
6929   xorl(tmp1, tmp1);     // index for x
6930   xorl(tmp4, tmp4);     // index for z
6931 
6932   Label L_first_loop, L_first_loop_exit;
6933 
6934   testl(xlen, 1);
6935   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
6936 
6937   // Square and right shift by 1 the odd element using 32 bit multiply
6938   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
6939   imulq(raxReg, raxReg);
6940   shrq(raxReg, 1);
6941   adcq(tmp5, 0);
6942   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
6943   incrementl(tmp1);
6944   addl(tmp4, 2);
6945 
6946   // Square and  right shift by 1 the rest using 64 bit multiply
6947   bind(L_first_loop);
6948   cmpptr(tmp1, xlen);
6949   jccb(Assembler::equal, L_first_loop_exit);
6950 
6951   // Square
6952   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
6953   rorq(raxReg, 32);    // convert big-endian to little-endian
6954   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
6955 
6956   // Right shift by 1 and save carry
6957   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
6958   rcrq(rdxReg, 1);
6959   rcrq(raxReg, 1);
6960   adcq(tmp5, 0);
6961 
6962   // Store result in z
6963   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
6964   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
6965 
6966   // Update indices for x and z
6967   addl(tmp1, 2);
6968   addl(tmp4, 4);
6969   jmp(L_first_loop);
6970 
6971   bind(L_first_loop_exit);
6972 }
6973 
6974 
6975 /**
6976  * Perform the following multiply add operation using BMI2 instructions
6977  * carry:sum = sum + op1*op2 + carry
6978  * op2 should be in rdx
6979  * op2 is preserved, all other registers are modified
6980  */
6981 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
6982   // assert op2 is rdx
6983   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
6984   addq(sum, carry);
6985   adcq(tmp2, 0);
6986   addq(sum, op1);
6987   adcq(tmp2, 0);
6988   movq(carry, tmp2);
6989 }
6990 
6991 /**
6992  * Perform the following multiply add operation:
6993  * carry:sum = sum + op1*op2 + carry
6994  * Preserves op1, op2 and modifies rest of registers
6995  */
6996 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
6997   // rdx:rax = op1 * op2
6998   movq(raxReg, op2);
6999   mulq(op1);
7000 
7001   //  rdx:rax = sum + carry + rdx:rax
7002   addq(sum, carry);
7003   adcq(rdxReg, 0);
7004   addq(sum, raxReg);
7005   adcq(rdxReg, 0);
7006 
7007   // carry:sum = rdx:sum
7008   movq(carry, rdxReg);
7009 }
7010 
7011 /**
7012  * Add 64 bit long carry into z[] with carry propagation.
7013  * Preserves z and carry register values and modifies rest of registers.
7014  *
7015  */
7016 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
7017   Label L_fourth_loop, L_fourth_loop_exit;
7018 
7019   movl(tmp1, 1);
7020   subl(zlen, 2);
7021   addq(Address(z, zlen, Address::times_4, 0), carry);
7022 
7023   bind(L_fourth_loop);
7024   jccb(Assembler::carryClear, L_fourth_loop_exit);
7025   subl(zlen, 2);
7026   jccb(Assembler::negative, L_fourth_loop_exit);
7027   addq(Address(z, zlen, Address::times_4, 0), tmp1);
7028   jmp(L_fourth_loop);
7029   bind(L_fourth_loop_exit);
7030 }
7031 
7032 /**
7033  * Shift z[] left by 1 bit.
7034  * Preserves x, len, z and zlen registers and modifies rest of the registers.
7035  *
7036  */
7037 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
7038 
7039   Label L_fifth_loop, L_fifth_loop_exit;
7040 
7041   // Fifth loop
7042   // Perform primitiveLeftShift(z, zlen, 1)
7043 
7044   const Register prev_carry = tmp1;
7045   const Register new_carry = tmp4;
7046   const Register value = tmp2;
7047   const Register zidx = tmp3;
7048 
7049   // int zidx, carry;
7050   // long value;
7051   // carry = 0;
7052   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
7053   //    (carry:value)  = (z[i] << 1) | carry ;
7054   //    z[i] = value;
7055   // }
7056 
7057   movl(zidx, zlen);
7058   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
7059 
7060   bind(L_fifth_loop);
7061   decl(zidx);  // Use decl to preserve carry flag
7062   decl(zidx);
7063   jccb(Assembler::negative, L_fifth_loop_exit);
7064 
7065   if (UseBMI2Instructions) {
7066      movq(value, Address(z, zidx, Address::times_4, 0));
7067      rclq(value, 1);
7068      rorxq(value, value, 32);
7069      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
7070   }
7071   else {
7072     // clear new_carry
7073     xorl(new_carry, new_carry);
7074 
7075     // Shift z[i] by 1, or in previous carry and save new carry
7076     movq(value, Address(z, zidx, Address::times_4, 0));
7077     shlq(value, 1);
7078     adcl(new_carry, 0);
7079 
7080     orq(value, prev_carry);
7081     rorq(value, 0x20);
7082     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
7083 
7084     // Set previous carry = new carry
7085     movl(prev_carry, new_carry);
7086   }
7087   jmp(L_fifth_loop);
7088 
7089   bind(L_fifth_loop_exit);
7090 }
7091 
7092 
7093 /**
7094  * Code for BigInteger::squareToLen() intrinsic
7095  *
7096  * rdi: x
7097  * rsi: len
7098  * r8:  z
7099  * rcx: zlen
7100  * r12: tmp1
7101  * r13: tmp2
7102  * r14: tmp3
7103  * r15: tmp4
7104  * rbx: tmp5
7105  *
7106  */
7107 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7108 
7109   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
7110   push(tmp1);
7111   push(tmp2);
7112   push(tmp3);
7113   push(tmp4);
7114   push(tmp5);
7115 
7116   // First loop
7117   // Store the squares, right shifted one bit (i.e., divided by 2).
7118   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
7119 
7120   // Add in off-diagonal sums.
7121   //
7122   // Second, third (nested) and fourth loops.
7123   // zlen +=2;
7124   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
7125   //    carry = 0;
7126   //    long op2 = x[xidx:xidx+1];
7127   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
7128   //       k -= 2;
7129   //       long op1 = x[j:j+1];
7130   //       long sum = z[k:k+1];
7131   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
7132   //       z[k:k+1] = sum;
7133   //    }
7134   //    add_one_64(z, k, carry, tmp_regs);
7135   // }
7136 
7137   const Register carry = tmp5;
7138   const Register sum = tmp3;
7139   const Register op1 = tmp4;
7140   Register op2 = tmp2;
7141 
7142   push(zlen);
7143   push(len);
7144   addl(zlen,2);
7145   bind(L_second_loop);
7146   xorq(carry, carry);
7147   subl(zlen, 4);
7148   subl(len, 2);
7149   push(zlen);
7150   push(len);
7151   cmpl(len, 0);
7152   jccb(Assembler::lessEqual, L_second_loop_exit);
7153 
7154   // Multiply an array by one 64 bit long.
7155   if (UseBMI2Instructions) {
7156     op2 = rdxReg;
7157     movq(op2, Address(x, len, Address::times_4,  0));
7158     rorxq(op2, op2, 32);
7159   }
7160   else {
7161     movq(op2, Address(x, len, Address::times_4,  0));
7162     rorq(op2, 32);
7163   }
7164 
7165   bind(L_third_loop);
7166   decrementl(len);
7167   jccb(Assembler::negative, L_third_loop_exit);
7168   decrementl(len);
7169   jccb(Assembler::negative, L_last_x);
7170 
7171   movq(op1, Address(x, len, Address::times_4,  0));
7172   rorq(op1, 32);
7173 
7174   bind(L_multiply);
7175   subl(zlen, 2);
7176   movq(sum, Address(z, zlen, Address::times_4,  0));
7177 
7178   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
7179   if (UseBMI2Instructions) {
7180     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
7181   }
7182   else {
7183     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7184   }
7185 
7186   movq(Address(z, zlen, Address::times_4, 0), sum);
7187 
7188   jmp(L_third_loop);
7189   bind(L_third_loop_exit);
7190 
7191   // Fourth loop
7192   // Add 64 bit long carry into z with carry propagation.
7193   // Uses offsetted zlen.
7194   add_one_64(z, zlen, carry, tmp1);
7195 
7196   pop(len);
7197   pop(zlen);
7198   jmp(L_second_loop);
7199 
7200   // Next infrequent code is moved outside loops.
7201   bind(L_last_x);
7202   movl(op1, Address(x, 0));
7203   jmp(L_multiply);
7204 
7205   bind(L_second_loop_exit);
7206   pop(len);
7207   pop(zlen);
7208   pop(len);
7209   pop(zlen);
7210 
7211   // Fifth loop
7212   // Shift z left 1 bit.
7213   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
7214 
7215   // z[zlen-1] |= x[len-1] & 1;
7216   movl(tmp3, Address(x, len, Address::times_4, -4));
7217   andl(tmp3, 1);
7218   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
7219 
7220   pop(tmp5);
7221   pop(tmp4);
7222   pop(tmp3);
7223   pop(tmp2);
7224   pop(tmp1);
7225 }
7226 
7227 /**
7228  * Helper function for mul_add()
7229  * Multiply the in[] by int k and add to out[] starting at offset offs using
7230  * 128 bit by 32 bit multiply and return the carry in tmp5.
7231  * Only quad int aligned length of in[] is operated on in this function.
7232  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
7233  * This function preserves out, in and k registers.
7234  * len and offset point to the appropriate index in "in" & "out" correspondingly
7235  * tmp5 has the carry.
7236  * other registers are temporary and are modified.
7237  *
7238  */
7239 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
7240   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
7241   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7242 
7243   Label L_first_loop, L_first_loop_exit;
7244 
7245   movl(tmp1, len);
7246   shrl(tmp1, 2);
7247 
7248   bind(L_first_loop);
7249   subl(tmp1, 1);
7250   jccb(Assembler::negative, L_first_loop_exit);
7251 
7252   subl(len, 4);
7253   subl(offset, 4);
7254 
7255   Register op2 = tmp2;
7256   const Register sum = tmp3;
7257   const Register op1 = tmp4;
7258   const Register carry = tmp5;
7259 
7260   if (UseBMI2Instructions) {
7261     op2 = rdxReg;
7262   }
7263 
7264   movq(op1, Address(in, len, Address::times_4,  8));
7265   rorq(op1, 32);
7266   movq(sum, Address(out, offset, Address::times_4,  8));
7267   rorq(sum, 32);
7268   if (UseBMI2Instructions) {
7269     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7270   }
7271   else {
7272     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7273   }
7274   // Store back in big endian from little endian
7275   rorq(sum, 0x20);
7276   movq(Address(out, offset, Address::times_4,  8), sum);
7277 
7278   movq(op1, Address(in, len, Address::times_4,  0));
7279   rorq(op1, 32);
7280   movq(sum, Address(out, offset, Address::times_4,  0));
7281   rorq(sum, 32);
7282   if (UseBMI2Instructions) {
7283     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7284   }
7285   else {
7286     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7287   }
7288   // Store back in big endian from little endian
7289   rorq(sum, 0x20);
7290   movq(Address(out, offset, Address::times_4,  0), sum);
7291 
7292   jmp(L_first_loop);
7293   bind(L_first_loop_exit);
7294 }
7295 
7296 /**
7297  * Code for BigInteger::mulAdd() intrinsic
7298  *
7299  * rdi: out
7300  * rsi: in
7301  * r11: offs (out.length - offset)
7302  * rcx: len
7303  * r8:  k
7304  * r12: tmp1
7305  * r13: tmp2
7306  * r14: tmp3
7307  * r15: tmp4
7308  * rbx: tmp5
7309  * Multiply the in[] by word k and add to out[], return the carry in rax
7310  */
7311 void MacroAssembler::mul_add(Register out, Register in, Register offs,
7312    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
7313    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
7314 
7315   Label L_carry, L_last_in, L_done;
7316 
7317 // carry = 0;
7318 // for (int j=len-1; j >= 0; j--) {
7319 //    long product = (in[j] & LONG_MASK) * kLong +
7320 //                   (out[offs] & LONG_MASK) + carry;
7321 //    out[offs--] = (int)product;
7322 //    carry = product >>> 32;
7323 // }
7324 //
7325   push(tmp1);
7326   push(tmp2);
7327   push(tmp3);
7328   push(tmp4);
7329   push(tmp5);
7330 
7331   Register op2 = tmp2;
7332   const Register sum = tmp3;
7333   const Register op1 = tmp4;
7334   const Register carry =  tmp5;
7335 
7336   if (UseBMI2Instructions) {
7337     op2 = rdxReg;
7338     movl(op2, k);
7339   }
7340   else {
7341     movl(op2, k);
7342   }
7343 
7344   xorq(carry, carry);
7345 
7346   //First loop
7347 
7348   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
7349   //The carry is in tmp5
7350   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
7351 
7352   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
7353   decrementl(len);
7354   jccb(Assembler::negative, L_carry);
7355   decrementl(len);
7356   jccb(Assembler::negative, L_last_in);
7357 
7358   movq(op1, Address(in, len, Address::times_4,  0));
7359   rorq(op1, 32);
7360 
7361   subl(offs, 2);
7362   movq(sum, Address(out, offs, Address::times_4,  0));
7363   rorq(sum, 32);
7364 
7365   if (UseBMI2Instructions) {
7366     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
7367   }
7368   else {
7369     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
7370   }
7371 
7372   // Store back in big endian from little endian
7373   rorq(sum, 0x20);
7374   movq(Address(out, offs, Address::times_4,  0), sum);
7375 
7376   testl(len, len);
7377   jccb(Assembler::zero, L_carry);
7378 
7379   //Multiply the last in[] entry, if any
7380   bind(L_last_in);
7381   movl(op1, Address(in, 0));
7382   movl(sum, Address(out, offs, Address::times_4,  -4));
7383 
7384   movl(raxReg, k);
7385   mull(op1); //tmp4 * eax -> edx:eax
7386   addl(sum, carry);
7387   adcl(rdxReg, 0);
7388   addl(sum, raxReg);
7389   adcl(rdxReg, 0);
7390   movl(carry, rdxReg);
7391 
7392   movl(Address(out, offs, Address::times_4,  -4), sum);
7393 
7394   bind(L_carry);
7395   //return tmp5/carry as carry in rax
7396   movl(rax, carry);
7397 
7398   bind(L_done);
7399   pop(tmp5);
7400   pop(tmp4);
7401   pop(tmp3);
7402   pop(tmp2);
7403   pop(tmp1);
7404 }
7405 
7406 /**
7407  * Emits code to update CRC-32 with a byte value according to constants in table
7408  *
7409  * @param [in,out]crc   Register containing the crc.
7410  * @param [in]val       Register containing the byte to fold into the CRC.
7411  * @param [in]table     Register containing the table of crc constants.
7412  *
7413  * uint32_t crc;
7414  * val = crc_table[(val ^ crc) & 0xFF];
7415  * crc = val ^ (crc >> 8);
7416  *
7417  */
7418 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
7419   xorl(val, crc);
7420   andl(val, 0xFF);
7421   shrl(crc, 8); // unsigned shift
7422   xorl(crc, Address(table, val, Address::times_4, 0));
7423 }
7424 
7425 /**
7426  * Fold 128-bit data chunk
7427  */
7428 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
7429   if (UseAVX > 0) {
7430     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
7431     vpclmulldq(xcrc, xK, xcrc); // [63:0]
7432     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
7433     pxor(xcrc, xtmp);
7434   } else {
7435     movdqa(xtmp, xcrc);
7436     pclmulhdq(xtmp, xK);   // [123:64]
7437     pclmulldq(xcrc, xK);   // [63:0]
7438     pxor(xcrc, xtmp);
7439     movdqu(xtmp, Address(buf, offset));
7440     pxor(xcrc, xtmp);
7441   }
7442 }
7443 
7444 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
7445   if (UseAVX > 0) {
7446     vpclmulhdq(xtmp, xK, xcrc);
7447     vpclmulldq(xcrc, xK, xcrc);
7448     pxor(xcrc, xbuf);
7449     pxor(xcrc, xtmp);
7450   } else {
7451     movdqa(xtmp, xcrc);
7452     pclmulhdq(xtmp, xK);
7453     pclmulldq(xcrc, xK);
7454     pxor(xcrc, xbuf);
7455     pxor(xcrc, xtmp);
7456   }
7457 }
7458 
7459 /**
7460  * 8-bit folds to compute 32-bit CRC
7461  *
7462  * uint64_t xcrc;
7463  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
7464  */
7465 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
7466   movdl(tmp, xcrc);
7467   andl(tmp, 0xFF);
7468   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
7469   psrldq(xcrc, 1); // unsigned shift one byte
7470   pxor(xcrc, xtmp);
7471 }
7472 
7473 /**
7474  * uint32_t crc;
7475  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
7476  */
7477 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
7478   movl(tmp, crc);
7479   andl(tmp, 0xFF);
7480   shrl(crc, 8);
7481   xorl(crc, Address(table, tmp, Address::times_4, 0));
7482 }
7483 
7484 /**
7485  * @param crc   register containing existing CRC (32-bit)
7486  * @param buf   register pointing to input byte buffer (byte*)
7487  * @param len   register containing number of bytes
7488  * @param table register that will contain address of CRC table
7489  * @param tmp   scratch register
7490  */
7491 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
7492   assert_different_registers(crc, buf, len, table, tmp, rax);
7493 
7494   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
7495   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
7496 
7497   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
7498   // context for the registers used, where all instructions below are using 128-bit mode
7499   // On EVEX without VL and BW, these instructions will all be AVX.
7500   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
7501   notl(crc); // ~crc
7502   cmpl(len, 16);
7503   jcc(Assembler::less, L_tail);
7504 
7505   // Align buffer to 16 bytes
7506   movl(tmp, buf);
7507   andl(tmp, 0xF);
7508   jccb(Assembler::zero, L_aligned);
7509   subl(tmp,  16);
7510   addl(len, tmp);
7511 
7512   align(4);
7513   BIND(L_align_loop);
7514   movsbl(rax, Address(buf, 0)); // load byte with sign extension
7515   update_byte_crc32(crc, rax, table);
7516   increment(buf);
7517   incrementl(tmp);
7518   jccb(Assembler::less, L_align_loop);
7519 
7520   BIND(L_aligned);
7521   movl(tmp, len); // save
7522   shrl(len, 4);
7523   jcc(Assembler::zero, L_tail_restore);
7524 
7525   // Fold crc into first bytes of vector
7526   movdqa(xmm1, Address(buf, 0));
7527   movdl(rax, xmm1);
7528   xorl(crc, rax);
7529   if (VM_Version::supports_sse4_1()) {
7530     pinsrd(xmm1, crc, 0);
7531   } else {
7532     pinsrw(xmm1, crc, 0);
7533     shrl(crc, 16);
7534     pinsrw(xmm1, crc, 1);
7535   }
7536   addptr(buf, 16);
7537   subl(len, 4); // len > 0
7538   jcc(Assembler::less, L_fold_tail);
7539 
7540   movdqa(xmm2, Address(buf,  0));
7541   movdqa(xmm3, Address(buf, 16));
7542   movdqa(xmm4, Address(buf, 32));
7543   addptr(buf, 48);
7544   subl(len, 3);
7545   jcc(Assembler::lessEqual, L_fold_512b);
7546 
7547   // Fold total 512 bits of polynomial on each iteration,
7548   // 128 bits per each of 4 parallel streams.
7549   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
7550 
7551   align32();
7552   BIND(L_fold_512b_loop);
7553   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7554   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
7555   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
7556   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
7557   addptr(buf, 64);
7558   subl(len, 4);
7559   jcc(Assembler::greater, L_fold_512b_loop);
7560 
7561   // Fold 512 bits to 128 bits.
7562   BIND(L_fold_512b);
7563   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
7564   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
7565   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
7566   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
7567 
7568   // Fold the rest of 128 bits data chunks
7569   BIND(L_fold_tail);
7570   addl(len, 3);
7571   jccb(Assembler::lessEqual, L_fold_128b);
7572   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
7573 
7574   BIND(L_fold_tail_loop);
7575   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7576   addptr(buf, 16);
7577   decrementl(len);
7578   jccb(Assembler::greater, L_fold_tail_loop);
7579 
7580   // Fold 128 bits in xmm1 down into 32 bits in crc register.
7581   BIND(L_fold_128b);
7582   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
7583   if (UseAVX > 0) {
7584     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
7585     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
7586     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
7587   } else {
7588     movdqa(xmm2, xmm0);
7589     pclmulqdq(xmm2, xmm1, 0x1);
7590     movdqa(xmm3, xmm0);
7591     pand(xmm3, xmm2);
7592     pclmulqdq(xmm0, xmm3, 0x1);
7593   }
7594   psrldq(xmm1, 8);
7595   psrldq(xmm2, 4);
7596   pxor(xmm0, xmm1);
7597   pxor(xmm0, xmm2);
7598 
7599   // 8 8-bit folds to compute 32-bit CRC.
7600   for (int j = 0; j < 4; j++) {
7601     fold_8bit_crc32(xmm0, table, xmm1, rax);
7602   }
7603   movdl(crc, xmm0); // mov 32 bits to general register
7604   for (int j = 0; j < 4; j++) {
7605     fold_8bit_crc32(crc, table, rax);
7606   }
7607 
7608   BIND(L_tail_restore);
7609   movl(len, tmp); // restore
7610   BIND(L_tail);
7611   andl(len, 0xf);
7612   jccb(Assembler::zero, L_exit);
7613 
7614   // Fold the rest of bytes
7615   align(4);
7616   BIND(L_tail_loop);
7617   movsbl(rax, Address(buf, 0)); // load byte with sign extension
7618   update_byte_crc32(crc, rax, table);
7619   increment(buf);
7620   decrementl(len);
7621   jccb(Assembler::greater, L_tail_loop);
7622 
7623   BIND(L_exit);
7624   notl(crc); // ~c
7625 }
7626 
7627 // Helper function for AVX 512 CRC32
7628 // Fold 512-bit data chunks
7629 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
7630                                              Register pos, int offset) {
7631   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
7632   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
7633   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
7634   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
7635   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
7636 }
7637 
7638 // Helper function for AVX 512 CRC32
7639 // Compute CRC32 for < 256B buffers
7640 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
7641                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
7642                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
7643 
7644   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
7645   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
7646   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
7647 
7648   // check if there is enough buffer to be able to fold 16B at a time
7649   cmpl(len, 32);
7650   jcc(Assembler::less, L_less_than_32);
7651 
7652   // if there is, load the constants
7653   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
7654   movdl(xmm0, crc);                        // get the initial crc value
7655   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7656   pxor(xmm7, xmm0);
7657 
7658   // update the buffer pointer
7659   addl(pos, 16);
7660   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
7661   subl(len, 32);
7662   jmp(L_16B_reduction_loop);
7663 
7664   bind(L_less_than_32);
7665   //mov initial crc to the return value. this is necessary for zero - length buffers.
7666   movl(rax, crc);
7667   testl(len, len);
7668   jcc(Assembler::equal, L_cleanup);
7669 
7670   movdl(xmm0, crc);                        //get the initial crc value
7671 
7672   cmpl(len, 16);
7673   jcc(Assembler::equal, L_exact_16_left);
7674   jcc(Assembler::less, L_less_than_16_left);
7675 
7676   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
7677   pxor(xmm7, xmm0);                       //xor the initial crc value
7678   addl(pos, 16);
7679   subl(len, 16);
7680   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
7681   jmp(L_get_last_two_xmms);
7682 
7683   bind(L_less_than_16_left);
7684   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
7685   pxor(xmm1, xmm1);
7686   movptr(tmp1, rsp);
7687   movdqu(Address(tmp1, 0 * 16), xmm1);
7688 
7689   cmpl(len, 4);
7690   jcc(Assembler::less, L_only_less_than_4);
7691 
7692   //backup the counter value
7693   movl(tmp2, len);
7694   cmpl(len, 8);
7695   jcc(Assembler::less, L_less_than_8_left);
7696 
7697   //load 8 Bytes
7698   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
7699   movq(Address(tmp1, 0 * 16), rax);
7700   addptr(tmp1, 8);
7701   subl(len, 8);
7702   addl(pos, 8);
7703 
7704   bind(L_less_than_8_left);
7705   cmpl(len, 4);
7706   jcc(Assembler::less, L_less_than_4_left);
7707 
7708   //load 4 Bytes
7709   movl(rax, Address(buf, pos, Address::times_1, 0));
7710   movl(Address(tmp1, 0 * 16), rax);
7711   addptr(tmp1, 4);
7712   subl(len, 4);
7713   addl(pos, 4);
7714 
7715   bind(L_less_than_4_left);
7716   cmpl(len, 2);
7717   jcc(Assembler::less, L_less_than_2_left);
7718 
7719   // load 2 Bytes
7720   movw(rax, Address(buf, pos, Address::times_1, 0));
7721   movl(Address(tmp1, 0 * 16), rax);
7722   addptr(tmp1, 2);
7723   subl(len, 2);
7724   addl(pos, 2);
7725 
7726   bind(L_less_than_2_left);
7727   cmpl(len, 1);
7728   jcc(Assembler::less, L_zero_left);
7729 
7730   // load 1 Byte
7731   movb(rax, Address(buf, pos, Address::times_1, 0));
7732   movb(Address(tmp1, 0 * 16), rax);
7733 
7734   bind(L_zero_left);
7735   movdqu(xmm7, Address(rsp, 0));
7736   pxor(xmm7, xmm0);                       //xor the initial crc value
7737 
7738   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7739   movdqu(xmm0, Address(rax, tmp2));
7740   pshufb(xmm7, xmm0);
7741   jmp(L_128_done);
7742 
7743   bind(L_exact_16_left);
7744   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
7745   pxor(xmm7, xmm0);                       //xor the initial crc value
7746   jmp(L_128_done);
7747 
7748   bind(L_only_less_than_4);
7749   cmpl(len, 3);
7750   jcc(Assembler::less, L_only_less_than_3);
7751 
7752   // load 3 Bytes
7753   movb(rax, Address(buf, pos, Address::times_1, 0));
7754   movb(Address(tmp1, 0), rax);
7755 
7756   movb(rax, Address(buf, pos, Address::times_1, 1));
7757   movb(Address(tmp1, 1), rax);
7758 
7759   movb(rax, Address(buf, pos, Address::times_1, 2));
7760   movb(Address(tmp1, 2), rax);
7761 
7762   movdqu(xmm7, Address(rsp, 0));
7763   pxor(xmm7, xmm0);                     //xor the initial crc value
7764 
7765   pslldq(xmm7, 0x5);
7766   jmp(L_barrett);
7767   bind(L_only_less_than_3);
7768   cmpl(len, 2);
7769   jcc(Assembler::less, L_only_less_than_2);
7770 
7771   // load 2 Bytes
7772   movb(rax, Address(buf, pos, Address::times_1, 0));
7773   movb(Address(tmp1, 0), rax);
7774 
7775   movb(rax, Address(buf, pos, Address::times_1, 1));
7776   movb(Address(tmp1, 1), rax);
7777 
7778   movdqu(xmm7, Address(rsp, 0));
7779   pxor(xmm7, xmm0);                     //xor the initial crc value
7780 
7781   pslldq(xmm7, 0x6);
7782   jmp(L_barrett);
7783 
7784   bind(L_only_less_than_2);
7785   //load 1 Byte
7786   movb(rax, Address(buf, pos, Address::times_1, 0));
7787   movb(Address(tmp1, 0), rax);
7788 
7789   movdqu(xmm7, Address(rsp, 0));
7790   pxor(xmm7, xmm0);                     //xor the initial crc value
7791 
7792   pslldq(xmm7, 0x7);
7793 }
7794 
7795 /**
7796 * Compute CRC32 using AVX512 instructions
7797 * param crc   register containing existing CRC (32-bit)
7798 * param buf   register pointing to input byte buffer (byte*)
7799 * param len   register containing number of bytes
7800 * param table address of crc or crc32c table
7801 * param tmp1  scratch register
7802 * param tmp2  scratch register
7803 * return rax  result register
7804 *
7805 * This routine is identical for crc32c with the exception of the precomputed constant
7806 * table which will be passed as the table argument.  The calculation steps are
7807 * the same for both variants.
7808 */
7809 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
7810   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
7811 
7812   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
7813   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
7814   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
7815   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
7816   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
7817 
7818   const Register pos = r12;
7819   push(r12);
7820   subptr(rsp, 16 * 2 + 8);
7821 
7822   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
7823   // context for the registers used, where all instructions below are using 128-bit mode
7824   // On EVEX without VL and BW, these instructions will all be AVX.
7825   movl(pos, 0);
7826 
7827   // check if smaller than 256B
7828   cmpl(len, 256);
7829   jcc(Assembler::less, L_less_than_256);
7830 
7831   // load the initial crc value
7832   movdl(xmm10, crc);
7833 
7834   // receive the initial 64B data, xor the initial crc value
7835   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
7836   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
7837   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
7838   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
7839 
7840   subl(len, 256);
7841   cmpl(len, 256);
7842   jcc(Assembler::less, L_fold_128_B_loop);
7843 
7844   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
7845   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
7846   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
7847   subl(len, 256);
7848 
7849   bind(L_fold_256_B_loop);
7850   addl(pos, 256);
7851   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
7852   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
7853   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
7854   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
7855 
7856   subl(len, 256);
7857   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
7858 
7859   // Fold 256 into 128
7860   addl(pos, 256);
7861   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
7862   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
7863   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
7864 
7865   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
7866   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
7867   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
7868 
7869   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
7870   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
7871 
7872   addl(len, 128);
7873   jmp(L_fold_128_B_register);
7874 
7875   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
7876   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
7877 
7878   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
7879   bind(L_fold_128_B_loop);
7880   addl(pos, 128);
7881   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
7882   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
7883 
7884   subl(len, 128);
7885   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
7886 
7887   addl(pos, 128);
7888 
7889   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
7890   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
7891   bind(L_fold_128_B_register);
7892   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
7893   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
7894   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
7895   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
7896   // save last that has no multiplicand
7897   vextracti64x2(xmm7, xmm4, 3);
7898 
7899   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
7900   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
7901   // Needed later in reduction loop
7902   movdqu(xmm10, Address(table, 1 * 16));
7903   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
7904   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
7905 
7906   // Swap 1,0,3,2 - 01 00 11 10
7907   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
7908   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
7909   vextracti128(xmm5, xmm8, 1);
7910   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
7911 
7912   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
7913   // instead of a cmp instruction, we use the negative flag with the jl instruction
7914   addl(len, 128 - 16);
7915   jcc(Assembler::less, L_final_reduction_for_128);
7916 
7917   bind(L_16B_reduction_loop);
7918   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
7919   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7920   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7921   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
7922   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7923   addl(pos, 16);
7924   subl(len, 16);
7925   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
7926 
7927   bind(L_final_reduction_for_128);
7928   addl(len, 16);
7929   jcc(Assembler::equal, L_128_done);
7930 
7931   bind(L_get_last_two_xmms);
7932   movdqu(xmm2, xmm7);
7933   addl(pos, len);
7934   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
7935   subl(pos, len);
7936 
7937   // get rid of the extra data that was loaded before
7938   // load the shift constant
7939   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
7940   movdqu(xmm0, Address(rax, len));
7941   addl(rax, len);
7942 
7943   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7944   //Change mask to 512
7945   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
7946   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
7947 
7948   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
7949   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
7950   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7951   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
7952   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
7953 
7954   bind(L_128_done);
7955   // compute crc of a 128-bit value
7956   movdqu(xmm10, Address(table, 3 * 16));
7957   movdqu(xmm0, xmm7);
7958 
7959   // 64b fold
7960   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
7961   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
7962   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7963 
7964   // 32b fold
7965   movdqu(xmm0, xmm7);
7966   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
7967   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
7968   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
7969   jmp(L_barrett);
7970 
7971   bind(L_less_than_256);
7972   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
7973 
7974   //barrett reduction
7975   bind(L_barrett);
7976   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
7977   movdqu(xmm1, xmm7);
7978   movdqu(xmm2, xmm7);
7979   movdqu(xmm10, Address(table, 4 * 16));
7980 
7981   pclmulqdq(xmm7, xmm10, 0x0);
7982   pxor(xmm7, xmm2);
7983   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
7984   movdqu(xmm2, xmm7);
7985   pclmulqdq(xmm7, xmm10, 0x10);
7986   pxor(xmm7, xmm2);
7987   pxor(xmm7, xmm1);
7988   pextrd(crc, xmm7, 2);
7989 
7990   bind(L_cleanup);
7991   addptr(rsp, 16 * 2 + 8);
7992   pop(r12);
7993 }
7994 
7995 // S. Gueron / Information Processing Letters 112 (2012) 184
7996 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
7997 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
7998 // Output: the 64-bit carry-less product of B * CONST
7999 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
8000                                      Register tmp1, Register tmp2, Register tmp3) {
8001   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
8002   if (n > 0) {
8003     addq(tmp3, n * 256 * 8);
8004   }
8005   //    Q1 = TABLEExt[n][B & 0xFF];
8006   movl(tmp1, in);
8007   andl(tmp1, 0x000000FF);
8008   shll(tmp1, 3);
8009   addq(tmp1, tmp3);
8010   movq(tmp1, Address(tmp1, 0));
8011 
8012   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
8013   movl(tmp2, in);
8014   shrl(tmp2, 8);
8015   andl(tmp2, 0x000000FF);
8016   shll(tmp2, 3);
8017   addq(tmp2, tmp3);
8018   movq(tmp2, Address(tmp2, 0));
8019 
8020   shlq(tmp2, 8);
8021   xorq(tmp1, tmp2);
8022 
8023   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
8024   movl(tmp2, in);
8025   shrl(tmp2, 16);
8026   andl(tmp2, 0x000000FF);
8027   shll(tmp2, 3);
8028   addq(tmp2, tmp3);
8029   movq(tmp2, Address(tmp2, 0));
8030 
8031   shlq(tmp2, 16);
8032   xorq(tmp1, tmp2);
8033 
8034   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
8035   shrl(in, 24);
8036   andl(in, 0x000000FF);
8037   shll(in, 3);
8038   addq(in, tmp3);
8039   movq(in, Address(in, 0));
8040 
8041   shlq(in, 24);
8042   xorq(in, tmp1);
8043   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
8044 }
8045 
8046 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
8047                                       Register in_out,
8048                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
8049                                       XMMRegister w_xtmp2,
8050                                       Register tmp1,
8051                                       Register n_tmp2, Register n_tmp3) {
8052   if (is_pclmulqdq_supported) {
8053     movdl(w_xtmp1, in_out); // modified blindly
8054 
8055     movl(tmp1, const_or_pre_comp_const_index);
8056     movdl(w_xtmp2, tmp1);
8057     pclmulqdq(w_xtmp1, w_xtmp2, 0);
8058 
8059     movdq(in_out, w_xtmp1);
8060   } else {
8061     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
8062   }
8063 }
8064 
8065 // Recombination Alternative 2: No bit-reflections
8066 // T1 = (CRC_A * U1) << 1
8067 // T2 = (CRC_B * U2) << 1
8068 // C1 = T1 >> 32
8069 // C2 = T2 >> 32
8070 // T1 = T1 & 0xFFFFFFFF
8071 // T2 = T2 & 0xFFFFFFFF
8072 // T1 = CRC32(0, T1)
8073 // T2 = CRC32(0, T2)
8074 // C1 = C1 ^ T1
8075 // C2 = C2 ^ T2
8076 // CRC = C1 ^ C2 ^ CRC_C
8077 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
8078                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8079                                      Register tmp1, Register tmp2,
8080                                      Register n_tmp3) {
8081   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
8082   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
8083   shlq(in_out, 1);
8084   movl(tmp1, in_out);
8085   shrq(in_out, 32);
8086   xorl(tmp2, tmp2);
8087   crc32(tmp2, tmp1, 4);
8088   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
8089   shlq(in1, 1);
8090   movl(tmp1, in1);
8091   shrq(in1, 32);
8092   xorl(tmp2, tmp2);
8093   crc32(tmp2, tmp1, 4);
8094   xorl(in1, tmp2);
8095   xorl(in_out, in1);
8096   xorl(in_out, in2);
8097 }
8098 
8099 // Set N to predefined value
8100 // Subtract from a length of a buffer
8101 // execute in a loop:
8102 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
8103 // for i = 1 to N do
8104 //  CRC_A = CRC32(CRC_A, A[i])
8105 //  CRC_B = CRC32(CRC_B, B[i])
8106 //  CRC_C = CRC32(CRC_C, C[i])
8107 // end for
8108 // Recombine
8109 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
8110                                        Register in_out1, Register in_out2, Register in_out3,
8111                                        Register tmp1, Register tmp2, Register tmp3,
8112                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8113                                        Register tmp4, Register tmp5,
8114                                        Register n_tmp6) {
8115   Label L_processPartitions;
8116   Label L_processPartition;
8117   Label L_exit;
8118 
8119   bind(L_processPartitions);
8120   cmpl(in_out1, 3 * size);
8121   jcc(Assembler::less, L_exit);
8122     xorl(tmp1, tmp1);
8123     xorl(tmp2, tmp2);
8124     movq(tmp3, in_out2);
8125     addq(tmp3, size);
8126 
8127     bind(L_processPartition);
8128       crc32(in_out3, Address(in_out2, 0), 8);
8129       crc32(tmp1, Address(in_out2, size), 8);
8130       crc32(tmp2, Address(in_out2, size * 2), 8);
8131       addq(in_out2, 8);
8132       cmpq(in_out2, tmp3);
8133       jcc(Assembler::less, L_processPartition);
8134     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
8135             w_xtmp1, w_xtmp2, w_xtmp3,
8136             tmp4, tmp5,
8137             n_tmp6);
8138     addq(in_out2, 2 * size);
8139     subl(in_out1, 3 * size);
8140     jmp(L_processPartitions);
8141 
8142   bind(L_exit);
8143 }
8144 
8145 // Algorithm 2: Pipelined usage of the CRC32 instruction.
8146 // Input: A buffer I of L bytes.
8147 // Output: the CRC32C value of the buffer.
8148 // Notations:
8149 // Write L = 24N + r, with N = floor (L/24).
8150 // r = L mod 24 (0 <= r < 24).
8151 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
8152 // N quadwords, and R consists of r bytes.
8153 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
8154 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
8155 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
8156 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
8157 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
8158                                           Register tmp1, Register tmp2, Register tmp3,
8159                                           Register tmp4, Register tmp5, Register tmp6,
8160                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
8161                                           bool is_pclmulqdq_supported) {
8162   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
8163   Label L_wordByWord;
8164   Label L_byteByByteProlog;
8165   Label L_byteByByte;
8166   Label L_exit;
8167 
8168   if (is_pclmulqdq_supported ) {
8169     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
8170     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
8171 
8172     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
8173     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
8174 
8175     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
8176     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
8177     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
8178   } else {
8179     const_or_pre_comp_const_index[0] = 1;
8180     const_or_pre_comp_const_index[1] = 0;
8181 
8182     const_or_pre_comp_const_index[2] = 3;
8183     const_or_pre_comp_const_index[3] = 2;
8184 
8185     const_or_pre_comp_const_index[4] = 5;
8186     const_or_pre_comp_const_index[5] = 4;
8187    }
8188   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
8189                     in2, in1, in_out,
8190                     tmp1, tmp2, tmp3,
8191                     w_xtmp1, w_xtmp2, w_xtmp3,
8192                     tmp4, tmp5,
8193                     tmp6);
8194   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
8195                     in2, in1, in_out,
8196                     tmp1, tmp2, tmp3,
8197                     w_xtmp1, w_xtmp2, w_xtmp3,
8198                     tmp4, tmp5,
8199                     tmp6);
8200   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
8201                     in2, in1, in_out,
8202                     tmp1, tmp2, tmp3,
8203                     w_xtmp1, w_xtmp2, w_xtmp3,
8204                     tmp4, tmp5,
8205                     tmp6);
8206   movl(tmp1, in2);
8207   andl(tmp1, 0x00000007);
8208   negl(tmp1);
8209   addl(tmp1, in2);
8210   addq(tmp1, in1);
8211 
8212   cmpq(in1, tmp1);
8213   jccb(Assembler::greaterEqual, L_byteByByteProlog);
8214   align(16);
8215   BIND(L_wordByWord);
8216     crc32(in_out, Address(in1, 0), 8);
8217     addq(in1, 8);
8218     cmpq(in1, tmp1);
8219     jcc(Assembler::less, L_wordByWord);
8220 
8221   BIND(L_byteByByteProlog);
8222   andl(in2, 0x00000007);
8223   movl(tmp2, 1);
8224 
8225   cmpl(tmp2, in2);
8226   jccb(Assembler::greater, L_exit);
8227   BIND(L_byteByByte);
8228     crc32(in_out, Address(in1, 0), 1);
8229     incq(in1);
8230     incl(tmp2);
8231     cmpl(tmp2, in2);
8232     jcc(Assembler::lessEqual, L_byteByByte);
8233 
8234   BIND(L_exit);
8235 }
8236 #undef BIND
8237 #undef BLOCK_COMMENT
8238 
8239 // Compress char[] array to byte[].
8240 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
8241 // Return the array length if every element in array can be encoded,
8242 // otherwise, the index of first non-latin1 (> 0xff) character.
8243 //   @IntrinsicCandidate
8244 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
8245 //     for (int i = 0; i < len; i++) {
8246 //       char c = src[srcOff];
8247 //       if (c > 0xff) {
8248 //           return i;  // return index of non-latin1 char
8249 //       }
8250 //       dst[dstOff] = (byte)c;
8251 //       srcOff++;
8252 //       dstOff++;
8253 //     }
8254 //     return len;
8255 //   }
8256 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
8257   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8258   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8259   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
8260   Label copy_chars_loop, done, reset_sp, copy_tail;
8261 
8262   // rsi: src
8263   // rdi: dst
8264   // rdx: len
8265   // rcx: tmp5
8266   // rax: result
8267 
8268   // rsi holds start addr of source char[] to be compressed
8269   // rdi holds start addr of destination byte[]
8270   // rdx holds length
8271 
8272   assert(len != result, "");
8273 
8274   // save length for return
8275   movl(result, len);
8276 
8277   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
8278     VM_Version::supports_avx512vlbw() &&
8279     VM_Version::supports_bmi2()) {
8280 
8281     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
8282 
8283     // alignment
8284     Label post_alignment;
8285 
8286     // if length of the string is less than 32, handle it the old fashioned way
8287     testl(len, -32);
8288     jcc(Assembler::zero, below_threshold);
8289 
8290     // First check whether a character is compressible ( <= 0xFF).
8291     // Create mask to test for Unicode chars inside zmm vector
8292     movl(tmp5, 0x00FF);
8293     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
8294 
8295     testl(len, -64);
8296     jccb(Assembler::zero, post_alignment);
8297 
8298     movl(tmp5, dst);
8299     andl(tmp5, (32 - 1));
8300     negl(tmp5);
8301     andl(tmp5, (32 - 1));
8302 
8303     // bail out when there is nothing to be done
8304     testl(tmp5, 0xFFFFFFFF);
8305     jccb(Assembler::zero, post_alignment);
8306 
8307     // ~(~0 << len), where len is the # of remaining elements to process
8308     movl(len, 0xFFFFFFFF);
8309     shlxl(len, len, tmp5);
8310     notl(len);
8311     kmovdl(mask2, len);
8312     movl(len, result);
8313 
8314     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
8315     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
8316     ktestd(mask1, mask2);
8317     jcc(Assembler::carryClear, copy_tail);
8318 
8319     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
8320 
8321     addptr(src, tmp5);
8322     addptr(src, tmp5);
8323     addptr(dst, tmp5);
8324     subl(len, tmp5);
8325 
8326     bind(post_alignment);
8327     // end of alignment
8328 
8329     movl(tmp5, len);
8330     andl(tmp5, (32 - 1));    // tail count (in chars)
8331     andl(len, ~(32 - 1));    // vector count (in chars)
8332     jccb(Assembler::zero, copy_loop_tail);
8333 
8334     lea(src, Address(src, len, Address::times_2));
8335     lea(dst, Address(dst, len, Address::times_1));
8336     negptr(len);
8337 
8338     bind(copy_32_loop);
8339     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
8340     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
8341     kortestdl(mask1, mask1);
8342     jccb(Assembler::carryClear, reset_for_copy_tail);
8343 
8344     // All elements in current processed chunk are valid candidates for
8345     // compression. Write a truncated byte elements to the memory.
8346     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
8347     addptr(len, 32);
8348     jccb(Assembler::notZero, copy_32_loop);
8349 
8350     bind(copy_loop_tail);
8351     // bail out when there is nothing to be done
8352     testl(tmp5, 0xFFFFFFFF);
8353     jcc(Assembler::zero, done);
8354 
8355     movl(len, tmp5);
8356 
8357     // ~(~0 << len), where len is the # of remaining elements to process
8358     movl(tmp5, 0xFFFFFFFF);
8359     shlxl(tmp5, tmp5, len);
8360     notl(tmp5);
8361 
8362     kmovdl(mask2, tmp5);
8363 
8364     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
8365     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
8366     ktestd(mask1, mask2);
8367     jcc(Assembler::carryClear, copy_tail);
8368 
8369     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
8370     jmp(done);
8371 
8372     bind(reset_for_copy_tail);
8373     lea(src, Address(src, tmp5, Address::times_2));
8374     lea(dst, Address(dst, tmp5, Address::times_1));
8375     subptr(len, tmp5);
8376     jmp(copy_chars_loop);
8377 
8378     bind(below_threshold);
8379   }
8380 
8381   if (UseSSE42Intrinsics) {
8382     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
8383 
8384     // vectored compression
8385     testl(len, 0xfffffff8);
8386     jcc(Assembler::zero, copy_tail);
8387 
8388     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
8389     movdl(tmp1Reg, tmp5);
8390     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
8391 
8392     andl(len, 0xfffffff0);
8393     jccb(Assembler::zero, copy_16);
8394 
8395     // compress 16 chars per iter
8396     pxor(tmp4Reg, tmp4Reg);
8397 
8398     lea(src, Address(src, len, Address::times_2));
8399     lea(dst, Address(dst, len, Address::times_1));
8400     negptr(len);
8401 
8402     bind(copy_32_loop);
8403     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
8404     por(tmp4Reg, tmp2Reg);
8405     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
8406     por(tmp4Reg, tmp3Reg);
8407     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
8408     jccb(Assembler::notZero, reset_for_copy_tail);
8409     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
8410     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
8411     addptr(len, 16);
8412     jccb(Assembler::notZero, copy_32_loop);
8413 
8414     // compress next vector of 8 chars (if any)
8415     bind(copy_16);
8416     // len = 0
8417     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
8418     jccb(Assembler::zero, copy_tail_sse);
8419 
8420     pxor(tmp3Reg, tmp3Reg);
8421 
8422     movdqu(tmp2Reg, Address(src, 0));
8423     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
8424     jccb(Assembler::notZero, reset_for_copy_tail);
8425     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
8426     movq(Address(dst, 0), tmp2Reg);
8427     addptr(src, 16);
8428     addptr(dst, 8);
8429     jmpb(copy_tail_sse);
8430 
8431     bind(reset_for_copy_tail);
8432     movl(tmp5, result);
8433     andl(tmp5, 0x0000000f);
8434     lea(src, Address(src, tmp5, Address::times_2));
8435     lea(dst, Address(dst, tmp5, Address::times_1));
8436     subptr(len, tmp5);
8437     jmpb(copy_chars_loop);
8438 
8439     bind(copy_tail_sse);
8440     movl(len, result);
8441     andl(len, 0x00000007);    // tail count (in chars)
8442   }
8443   // compress 1 char per iter
8444   bind(copy_tail);
8445   testl(len, len);
8446   jccb(Assembler::zero, done);
8447   lea(src, Address(src, len, Address::times_2));
8448   lea(dst, Address(dst, len, Address::times_1));
8449   negptr(len);
8450 
8451   bind(copy_chars_loop);
8452   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
8453   testl(tmp5, 0xff00);      // check if Unicode char
8454   jccb(Assembler::notZero, reset_sp);
8455   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
8456   increment(len);
8457   jccb(Assembler::notZero, copy_chars_loop);
8458 
8459   // add len then return (len will be zero if compress succeeded, otherwise negative)
8460   bind(reset_sp);
8461   addl(result, len);
8462 
8463   bind(done);
8464 }
8465 
8466 // Inflate byte[] array to char[].
8467 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
8468 //   @IntrinsicCandidate
8469 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
8470 //     for (int i = 0; i < len; i++) {
8471 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
8472 //     }
8473 //   }
8474 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
8475   XMMRegister tmp1, Register tmp2, KRegister mask) {
8476   Label copy_chars_loop, done, below_threshold, avx3_threshold;
8477   // rsi: src
8478   // rdi: dst
8479   // rdx: len
8480   // rcx: tmp2
8481 
8482   // rsi holds start addr of source byte[] to be inflated
8483   // rdi holds start addr of destination char[]
8484   // rdx holds length
8485   assert_different_registers(src, dst, len, tmp2);
8486   movl(tmp2, len);
8487   if ((UseAVX > 2) && // AVX512
8488     VM_Version::supports_avx512vlbw() &&
8489     VM_Version::supports_bmi2()) {
8490 
8491     Label copy_32_loop, copy_tail;
8492     Register tmp3_aliased = len;
8493 
8494     // if length of the string is less than 16, handle it in an old fashioned way
8495     testl(len, -16);
8496     jcc(Assembler::zero, below_threshold);
8497 
8498     testl(len, -1 * AVX3Threshold);
8499     jcc(Assembler::zero, avx3_threshold);
8500 
8501     // In order to use only one arithmetic operation for the main loop we use
8502     // this pre-calculation
8503     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
8504     andl(len, -32);     // vector count
8505     jccb(Assembler::zero, copy_tail);
8506 
8507     lea(src, Address(src, len, Address::times_1));
8508     lea(dst, Address(dst, len, Address::times_2));
8509     negptr(len);
8510 
8511 
8512     // inflate 32 chars per iter
8513     bind(copy_32_loop);
8514     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
8515     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
8516     addptr(len, 32);
8517     jcc(Assembler::notZero, copy_32_loop);
8518 
8519     bind(copy_tail);
8520     // bail out when there is nothing to be done
8521     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
8522     jcc(Assembler::zero, done);
8523 
8524     // ~(~0 << length), where length is the # of remaining elements to process
8525     movl(tmp3_aliased, -1);
8526     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
8527     notl(tmp3_aliased);
8528     kmovdl(mask, tmp3_aliased);
8529     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
8530     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
8531 
8532     jmp(done);
8533     bind(avx3_threshold);
8534   }
8535   if (UseSSE42Intrinsics) {
8536     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
8537 
8538     if (UseAVX > 1) {
8539       andl(tmp2, (16 - 1));
8540       andl(len, -16);
8541       jccb(Assembler::zero, copy_new_tail);
8542     } else {
8543       andl(tmp2, 0x00000007);   // tail count (in chars)
8544       andl(len, 0xfffffff8);    // vector count (in chars)
8545       jccb(Assembler::zero, copy_tail);
8546     }
8547 
8548     // vectored inflation
8549     lea(src, Address(src, len, Address::times_1));
8550     lea(dst, Address(dst, len, Address::times_2));
8551     negptr(len);
8552 
8553     if (UseAVX > 1) {
8554       bind(copy_16_loop);
8555       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
8556       vmovdqu(Address(dst, len, Address::times_2), tmp1);
8557       addptr(len, 16);
8558       jcc(Assembler::notZero, copy_16_loop);
8559 
8560       bind(below_threshold);
8561       bind(copy_new_tail);
8562       movl(len, tmp2);
8563       andl(tmp2, 0x00000007);
8564       andl(len, 0xFFFFFFF8);
8565       jccb(Assembler::zero, copy_tail);
8566 
8567       pmovzxbw(tmp1, Address(src, 0));
8568       movdqu(Address(dst, 0), tmp1);
8569       addptr(src, 8);
8570       addptr(dst, 2 * 8);
8571 
8572       jmp(copy_tail, true);
8573     }
8574 
8575     // inflate 8 chars per iter
8576     bind(copy_8_loop);
8577     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
8578     movdqu(Address(dst, len, Address::times_2), tmp1);
8579     addptr(len, 8);
8580     jcc(Assembler::notZero, copy_8_loop);
8581 
8582     bind(copy_tail);
8583     movl(len, tmp2);
8584 
8585     cmpl(len, 4);
8586     jccb(Assembler::less, copy_bytes);
8587 
8588     movdl(tmp1, Address(src, 0));  // load 4 byte chars
8589     pmovzxbw(tmp1, tmp1);
8590     movq(Address(dst, 0), tmp1);
8591     subptr(len, 4);
8592     addptr(src, 4);
8593     addptr(dst, 8);
8594 
8595     bind(copy_bytes);
8596   } else {
8597     bind(below_threshold);
8598   }
8599 
8600   testl(len, len);
8601   jccb(Assembler::zero, done);
8602   lea(src, Address(src, len, Address::times_1));
8603   lea(dst, Address(dst, len, Address::times_2));
8604   negptr(len);
8605 
8606   // inflate 1 char per iter
8607   bind(copy_chars_loop);
8608   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
8609   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
8610   increment(len);
8611   jcc(Assembler::notZero, copy_chars_loop);
8612 
8613   bind(done);
8614 }
8615 
8616 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
8617   switch(type) {
8618     case T_BYTE:
8619     case T_BOOLEAN:
8620       evmovdqub(dst, kmask, src, merge, vector_len);
8621       break;
8622     case T_CHAR:
8623     case T_SHORT:
8624       evmovdquw(dst, kmask, src, merge, vector_len);
8625       break;
8626     case T_INT:
8627     case T_FLOAT:
8628       evmovdqul(dst, kmask, src, merge, vector_len);
8629       break;
8630     case T_LONG:
8631     case T_DOUBLE:
8632       evmovdquq(dst, kmask, src, merge, vector_len);
8633       break;
8634     default:
8635       fatal("Unexpected type argument %s", type2name(type));
8636       break;
8637   }
8638 }
8639 
8640 
8641 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
8642   switch(type) {
8643     case T_BYTE:
8644     case T_BOOLEAN:
8645       evmovdqub(dst, kmask, src, merge, vector_len);
8646       break;
8647     case T_CHAR:
8648     case T_SHORT:
8649       evmovdquw(dst, kmask, src, merge, vector_len);
8650       break;
8651     case T_INT:
8652     case T_FLOAT:
8653       evmovdqul(dst, kmask, src, merge, vector_len);
8654       break;
8655     case T_LONG:
8656     case T_DOUBLE:
8657       evmovdquq(dst, kmask, src, merge, vector_len);
8658       break;
8659     default:
8660       fatal("Unexpected type argument %s", type2name(type));
8661       break;
8662   }
8663 }
8664 
8665 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
8666   switch(type) {
8667     case T_BYTE:
8668     case T_BOOLEAN:
8669       evmovdqub(dst, kmask, src, merge, vector_len);
8670       break;
8671     case T_CHAR:
8672     case T_SHORT:
8673       evmovdquw(dst, kmask, src, merge, vector_len);
8674       break;
8675     case T_INT:
8676     case T_FLOAT:
8677       evmovdqul(dst, kmask, src, merge, vector_len);
8678       break;
8679     case T_LONG:
8680     case T_DOUBLE:
8681       evmovdquq(dst, kmask, src, merge, vector_len);
8682       break;
8683     default:
8684       fatal("Unexpected type argument %s", type2name(type));
8685       break;
8686   }
8687 }
8688 
8689 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
8690   switch(masklen) {
8691     case 2:
8692        knotbl(dst, src);
8693        movl(rtmp, 3);
8694        kmovbl(ktmp, rtmp);
8695        kandbl(dst, ktmp, dst);
8696        break;
8697     case 4:
8698        knotbl(dst, src);
8699        movl(rtmp, 15);
8700        kmovbl(ktmp, rtmp);
8701        kandbl(dst, ktmp, dst);
8702        break;
8703     case 8:
8704        knotbl(dst, src);
8705        break;
8706     case 16:
8707        knotwl(dst, src);
8708        break;
8709     case 32:
8710        knotdl(dst, src);
8711        break;
8712     case 64:
8713        knotql(dst, src);
8714        break;
8715     default:
8716       fatal("Unexpected vector length %d", masklen);
8717       break;
8718   }
8719 }
8720 
8721 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8722   switch(type) {
8723     case T_BOOLEAN:
8724     case T_BYTE:
8725        kandbl(dst, src1, src2);
8726        break;
8727     case T_CHAR:
8728     case T_SHORT:
8729        kandwl(dst, src1, src2);
8730        break;
8731     case T_INT:
8732     case T_FLOAT:
8733        kanddl(dst, src1, src2);
8734        break;
8735     case T_LONG:
8736     case T_DOUBLE:
8737        kandql(dst, src1, src2);
8738        break;
8739     default:
8740       fatal("Unexpected type argument %s", type2name(type));
8741       break;
8742   }
8743 }
8744 
8745 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8746   switch(type) {
8747     case T_BOOLEAN:
8748     case T_BYTE:
8749        korbl(dst, src1, src2);
8750        break;
8751     case T_CHAR:
8752     case T_SHORT:
8753        korwl(dst, src1, src2);
8754        break;
8755     case T_INT:
8756     case T_FLOAT:
8757        kordl(dst, src1, src2);
8758        break;
8759     case T_LONG:
8760     case T_DOUBLE:
8761        korql(dst, src1, src2);
8762        break;
8763     default:
8764       fatal("Unexpected type argument %s", type2name(type));
8765       break;
8766   }
8767 }
8768 
8769 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8770   switch(type) {
8771     case T_BOOLEAN:
8772     case T_BYTE:
8773        kxorbl(dst, src1, src2);
8774        break;
8775     case T_CHAR:
8776     case T_SHORT:
8777        kxorwl(dst, src1, src2);
8778        break;
8779     case T_INT:
8780     case T_FLOAT:
8781        kxordl(dst, src1, src2);
8782        break;
8783     case T_LONG:
8784     case T_DOUBLE:
8785        kxorql(dst, src1, src2);
8786        break;
8787     default:
8788       fatal("Unexpected type argument %s", type2name(type));
8789       break;
8790   }
8791 }
8792 
8793 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8794   switch(type) {
8795     case T_BOOLEAN:
8796     case T_BYTE:
8797       evpermb(dst, mask, nds, src, merge, vector_len); break;
8798     case T_CHAR:
8799     case T_SHORT:
8800       evpermw(dst, mask, nds, src, merge, vector_len); break;
8801     case T_INT:
8802     case T_FLOAT:
8803       evpermd(dst, mask, nds, src, merge, vector_len); break;
8804     case T_LONG:
8805     case T_DOUBLE:
8806       evpermq(dst, mask, nds, src, merge, vector_len); break;
8807     default:
8808       fatal("Unexpected type argument %s", type2name(type)); break;
8809   }
8810 }
8811 
8812 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8813   switch(type) {
8814     case T_BOOLEAN:
8815     case T_BYTE:
8816       evpermb(dst, mask, nds, src, merge, vector_len); break;
8817     case T_CHAR:
8818     case T_SHORT:
8819       evpermw(dst, mask, nds, src, merge, vector_len); break;
8820     case T_INT:
8821     case T_FLOAT:
8822       evpermd(dst, mask, nds, src, merge, vector_len); break;
8823     case T_LONG:
8824     case T_DOUBLE:
8825       evpermq(dst, mask, nds, src, merge, vector_len); break;
8826     default:
8827       fatal("Unexpected type argument %s", type2name(type)); break;
8828   }
8829 }
8830 
8831 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8832   switch(type) {
8833     case T_BYTE:
8834       evpminub(dst, mask, nds, src, merge, vector_len); break;
8835     case T_SHORT:
8836       evpminuw(dst, mask, nds, src, merge, vector_len); break;
8837     case T_INT:
8838       evpminud(dst, mask, nds, src, merge, vector_len); break;
8839     case T_LONG:
8840       evpminuq(dst, mask, nds, src, merge, vector_len); break;
8841     default:
8842       fatal("Unexpected type argument %s", type2name(type)); break;
8843   }
8844 }
8845 
8846 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8847   switch(type) {
8848     case T_BYTE:
8849       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
8850     case T_SHORT:
8851       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
8852     case T_INT:
8853       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
8854     case T_LONG:
8855       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
8856     default:
8857       fatal("Unexpected type argument %s", type2name(type)); break;
8858   }
8859 }
8860 
8861 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8862   switch(type) {
8863     case T_BYTE:
8864       evpminub(dst, mask, nds, src, merge, vector_len); break;
8865     case T_SHORT:
8866       evpminuw(dst, mask, nds, src, merge, vector_len); break;
8867     case T_INT:
8868       evpminud(dst, mask, nds, src, merge, vector_len); break;
8869     case T_LONG:
8870       evpminuq(dst, mask, nds, src, merge, vector_len); break;
8871     default:
8872       fatal("Unexpected type argument %s", type2name(type)); break;
8873   }
8874 }
8875 
8876 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8877   switch(type) {
8878     case T_BYTE:
8879       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
8880     case T_SHORT:
8881       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
8882     case T_INT:
8883       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
8884     case T_LONG:
8885       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
8886     default:
8887       fatal("Unexpected type argument %s", type2name(type)); break;
8888   }
8889 }
8890 
8891 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8892   switch(type) {
8893     case T_BYTE:
8894       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8895     case T_SHORT:
8896       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8897     case T_INT:
8898       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8899     case T_LONG:
8900       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8901     case T_FLOAT:
8902       evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8903     case T_DOUBLE:
8904       evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8905     default:
8906       fatal("Unexpected type argument %s", type2name(type)); break;
8907   }
8908 }
8909 
8910 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8911   switch(type) {
8912     case T_BYTE:
8913       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8914     case T_SHORT:
8915       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8916     case T_INT:
8917       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8918     case T_LONG:
8919       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8920     case T_FLOAT:
8921       evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8922     case T_DOUBLE:
8923       evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8924     default:
8925       fatal("Unexpected type argument %s", type2name(type)); break;
8926   }
8927 }
8928 
8929 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8930   switch(type) {
8931     case T_BYTE:
8932       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8933     case T_SHORT:
8934       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8935     case T_INT:
8936       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8937     case T_LONG:
8938       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8939     case T_FLOAT:
8940       evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8941     case T_DOUBLE:
8942       evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vector_len); break;
8943     default:
8944       fatal("Unexpected type argument %s", type2name(type)); break;
8945   }
8946 }
8947 
8948 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8949   switch(type) {
8950     case T_BYTE:
8951       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8952     case T_SHORT:
8953       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8954     case T_INT:
8955       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8956     case T_LONG:
8957       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8958     case T_FLOAT:
8959       evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8960     case T_DOUBLE:
8961       evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
8962     default:
8963       fatal("Unexpected type argument %s", type2name(type)); break;
8964   }
8965 }
8966 
8967 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8968   switch(type) {
8969     case T_INT:
8970       evpxord(dst, mask, nds, src, merge, vector_len); break;
8971     case T_LONG:
8972       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8973     default:
8974       fatal("Unexpected type argument %s", type2name(type)); break;
8975   }
8976 }
8977 
8978 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8979   switch(type) {
8980     case T_INT:
8981       evpxord(dst, mask, nds, src, merge, vector_len); break;
8982     case T_LONG:
8983       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8984     default:
8985       fatal("Unexpected type argument %s", type2name(type)); break;
8986   }
8987 }
8988 
8989 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8990   switch(type) {
8991     case T_INT:
8992       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8993     case T_LONG:
8994       evporq(dst, mask, nds, src, merge, vector_len); break;
8995     default:
8996       fatal("Unexpected type argument %s", type2name(type)); break;
8997   }
8998 }
8999 
9000 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9001   switch(type) {
9002     case T_INT:
9003       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
9004     case T_LONG:
9005       evporq(dst, mask, nds, src, merge, vector_len); break;
9006     default:
9007       fatal("Unexpected type argument %s", type2name(type)); break;
9008   }
9009 }
9010 
9011 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9012   switch(type) {
9013     case T_INT:
9014       evpandd(dst, mask, nds, src, merge, vector_len); break;
9015     case T_LONG:
9016       evpandq(dst, mask, nds, src, merge, vector_len); break;
9017     default:
9018       fatal("Unexpected type argument %s", type2name(type)); break;
9019   }
9020 }
9021 
9022 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9023   switch(type) {
9024     case T_INT:
9025       evpandd(dst, mask, nds, src, merge, vector_len); break;
9026     case T_LONG:
9027       evpandq(dst, mask, nds, src, merge, vector_len); break;
9028     default:
9029       fatal("Unexpected type argument %s", type2name(type)); break;
9030   }
9031 }
9032 
9033 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
9034   switch(masklen) {
9035     case 8:
9036        kortestbl(src1, src2);
9037        break;
9038     case 16:
9039        kortestwl(src1, src2);
9040        break;
9041     case 32:
9042        kortestdl(src1, src2);
9043        break;
9044     case 64:
9045        kortestql(src1, src2);
9046        break;
9047     default:
9048       fatal("Unexpected mask length %d", masklen);
9049       break;
9050   }
9051 }
9052 
9053 
9054 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
9055   switch(masklen)  {
9056     case 8:
9057        ktestbl(src1, src2);
9058        break;
9059     case 16:
9060        ktestwl(src1, src2);
9061        break;
9062     case 32:
9063        ktestdl(src1, src2);
9064        break;
9065     case 64:
9066        ktestql(src1, src2);
9067        break;
9068     default:
9069       fatal("Unexpected mask length %d", masklen);
9070       break;
9071   }
9072 }
9073 
9074 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
9075   switch(type) {
9076     case T_INT:
9077       evprold(dst, mask, src, shift, merge, vlen_enc); break;
9078     case T_LONG:
9079       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
9080     default:
9081       fatal("Unexpected type argument %s", type2name(type)); break;
9082       break;
9083   }
9084 }
9085 
9086 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
9087   switch(type) {
9088     case T_INT:
9089       evprord(dst, mask, src, shift, merge, vlen_enc); break;
9090     case T_LONG:
9091       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
9092     default:
9093       fatal("Unexpected type argument %s", type2name(type)); break;
9094   }
9095 }
9096 
9097 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
9098   switch(type) {
9099     case T_INT:
9100       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
9101     case T_LONG:
9102       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
9103     default:
9104       fatal("Unexpected type argument %s", type2name(type)); break;
9105   }
9106 }
9107 
9108 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
9109   switch(type) {
9110     case T_INT:
9111       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
9112     case T_LONG:
9113       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
9114     default:
9115       fatal("Unexpected type argument %s", type2name(type)); break;
9116   }
9117 }
9118 
9119 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9120   assert(rscratch != noreg || always_reachable(src), "missing");
9121 
9122   if (reachable(src)) {
9123     evpandq(dst, nds, as_Address(src), vector_len);
9124   } else {
9125     lea(rscratch, src);
9126     evpandq(dst, nds, Address(rscratch, 0), vector_len);
9127   }
9128 }
9129 
9130 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
9131   assert(rscratch != noreg || always_reachable(src), "missing");
9132 
9133   if (reachable(src)) {
9134     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
9135   } else {
9136     lea(rscratch, src);
9137     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
9138   }
9139 }
9140 
9141 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9142   assert(rscratch != noreg || always_reachable(src), "missing");
9143 
9144   if (reachable(src)) {
9145     evporq(dst, nds, as_Address(src), vector_len);
9146   } else {
9147     lea(rscratch, src);
9148     evporq(dst, nds, Address(rscratch, 0), vector_len);
9149   }
9150 }
9151 
9152 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9153   assert(rscratch != noreg || always_reachable(src), "missing");
9154 
9155   if (reachable(src)) {
9156     vpshufb(dst, nds, as_Address(src), vector_len);
9157   } else {
9158     lea(rscratch, src);
9159     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
9160   }
9161 }
9162 
9163 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
9164   assert(rscratch != noreg || always_reachable(src), "missing");
9165 
9166   if (reachable(src)) {
9167     Assembler::vpor(dst, nds, as_Address(src), vector_len);
9168   } else {
9169     lea(rscratch, src);
9170     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
9171   }
9172 }
9173 
9174 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
9175   assert(rscratch != noreg || always_reachable(src3), "missing");
9176 
9177   if (reachable(src3)) {
9178     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
9179   } else {
9180     lea(rscratch, src3);
9181     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
9182   }
9183 }
9184 
9185 #if COMPILER2_OR_JVMCI
9186 
9187 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
9188                                  Register length, Register temp, int vec_enc) {
9189   // Computing mask for predicated vector store.
9190   movptr(temp, -1);
9191   bzhiq(temp, temp, length);
9192   kmov(mask, temp);
9193   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
9194 }
9195 
9196 // Set memory operation for length "less than" 64 bytes.
9197 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
9198                                        XMMRegister xmm, KRegister mask, Register length,
9199                                        Register temp, bool use64byteVector) {
9200   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9201   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
9202   if (!use64byteVector) {
9203     fill32(dst, disp, xmm);
9204     subptr(length, 32 >> shift);
9205     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
9206   } else {
9207     assert(MaxVectorSize == 64, "vector length != 64");
9208     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
9209   }
9210 }
9211 
9212 
9213 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
9214                                        XMMRegister xmm, KRegister mask, Register length,
9215                                        Register temp) {
9216   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9217   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
9218   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
9219 }
9220 
9221 
9222 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
9223   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9224   vmovdqu(dst, xmm);
9225 }
9226 
9227 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
9228   fill32(Address(dst, disp), xmm);
9229 }
9230 
9231 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
9232   assert(MaxVectorSize >= 32, "vector length should be >= 32");
9233   if (!use64byteVector) {
9234     fill32(dst, xmm);
9235     fill32(dst.plus_disp(32), xmm);
9236   } else {
9237     evmovdquq(dst, xmm, Assembler::AVX_512bit);
9238   }
9239 }
9240 
9241 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
9242   fill64(Address(dst, disp), xmm, use64byteVector);
9243 }
9244 
9245 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
9246                                         Register count, Register rtmp, XMMRegister xtmp) {
9247   Label L_exit;
9248   Label L_fill_start;
9249   Label L_fill_64_bytes;
9250   Label L_fill_96_bytes;
9251   Label L_fill_128_bytes;
9252   Label L_fill_128_bytes_loop;
9253   Label L_fill_128_loop_header;
9254   Label L_fill_128_bytes_loop_header;
9255   Label L_fill_128_bytes_loop_pre_header;
9256   Label L_fill_zmm_sequence;
9257 
9258   int shift = -1;
9259   int avx3threshold = VM_Version::avx3_threshold();
9260   switch(type) {
9261     case T_BYTE:  shift = 0;
9262       break;
9263     case T_SHORT: shift = 1;
9264       break;
9265     case T_INT:   shift = 2;
9266       break;
9267     /* Uncomment when LONG fill stubs are supported.
9268     case T_LONG:  shift = 3;
9269       break;
9270     */
9271     default:
9272       fatal("Unhandled type: %s\n", type2name(type));
9273   }
9274 
9275   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
9276 
9277     if (MaxVectorSize == 64) {
9278       cmpq(count, avx3threshold >> shift);
9279       jcc(Assembler::greater, L_fill_zmm_sequence);
9280     }
9281 
9282     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
9283 
9284     bind(L_fill_start);
9285 
9286     cmpq(count, 32 >> shift);
9287     jccb(Assembler::greater, L_fill_64_bytes);
9288     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
9289     jmp(L_exit);
9290 
9291     bind(L_fill_64_bytes);
9292     cmpq(count, 64 >> shift);
9293     jccb(Assembler::greater, L_fill_96_bytes);
9294     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
9295     jmp(L_exit);
9296 
9297     bind(L_fill_96_bytes);
9298     cmpq(count, 96 >> shift);
9299     jccb(Assembler::greater, L_fill_128_bytes);
9300     fill64(to, 0, xtmp);
9301     subq(count, 64 >> shift);
9302     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
9303     jmp(L_exit);
9304 
9305     bind(L_fill_128_bytes);
9306     cmpq(count, 128 >> shift);
9307     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
9308     fill64(to, 0, xtmp);
9309     fill32(to, 64, xtmp);
9310     subq(count, 96 >> shift);
9311     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
9312     jmp(L_exit);
9313 
9314     bind(L_fill_128_bytes_loop_pre_header);
9315     {
9316       mov(rtmp, to);
9317       andq(rtmp, 31);
9318       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
9319       negq(rtmp);
9320       addq(rtmp, 32);
9321       mov64(r8, -1L);
9322       bzhiq(r8, r8, rtmp);
9323       kmovql(k2, r8);
9324       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
9325       addq(to, rtmp);
9326       shrq(rtmp, shift);
9327       subq(count, rtmp);
9328     }
9329 
9330     cmpq(count, 128 >> shift);
9331     jcc(Assembler::less, L_fill_start);
9332 
9333     bind(L_fill_128_bytes_loop_header);
9334     subq(count, 128 >> shift);
9335 
9336     align32();
9337     bind(L_fill_128_bytes_loop);
9338       fill64(to, 0, xtmp);
9339       fill64(to, 64, xtmp);
9340       addq(to, 128);
9341       subq(count, 128 >> shift);
9342       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
9343 
9344     addq(count, 128 >> shift);
9345     jcc(Assembler::zero, L_exit);
9346     jmp(L_fill_start);
9347   }
9348 
9349   if (MaxVectorSize == 64) {
9350     // Sequence using 64 byte ZMM register.
9351     Label L_fill_128_bytes_zmm;
9352     Label L_fill_192_bytes_zmm;
9353     Label L_fill_192_bytes_loop_zmm;
9354     Label L_fill_192_bytes_loop_header_zmm;
9355     Label L_fill_192_bytes_loop_pre_header_zmm;
9356     Label L_fill_start_zmm_sequence;
9357 
9358     bind(L_fill_zmm_sequence);
9359     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
9360 
9361     bind(L_fill_start_zmm_sequence);
9362     cmpq(count, 64 >> shift);
9363     jccb(Assembler::greater, L_fill_128_bytes_zmm);
9364     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
9365     jmp(L_exit);
9366 
9367     bind(L_fill_128_bytes_zmm);
9368     cmpq(count, 128 >> shift);
9369     jccb(Assembler::greater, L_fill_192_bytes_zmm);
9370     fill64(to, 0, xtmp, true);
9371     subq(count, 64 >> shift);
9372     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
9373     jmp(L_exit);
9374 
9375     bind(L_fill_192_bytes_zmm);
9376     cmpq(count, 192 >> shift);
9377     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
9378     fill64(to, 0, xtmp, true);
9379     fill64(to, 64, xtmp, true);
9380     subq(count, 128 >> shift);
9381     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
9382     jmp(L_exit);
9383 
9384     bind(L_fill_192_bytes_loop_pre_header_zmm);
9385     {
9386       movq(rtmp, to);
9387       andq(rtmp, 63);
9388       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
9389       negq(rtmp);
9390       addq(rtmp, 64);
9391       mov64(r8, -1L);
9392       bzhiq(r8, r8, rtmp);
9393       kmovql(k2, r8);
9394       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
9395       addq(to, rtmp);
9396       shrq(rtmp, shift);
9397       subq(count, rtmp);
9398     }
9399 
9400     cmpq(count, 192 >> shift);
9401     jcc(Assembler::less, L_fill_start_zmm_sequence);
9402 
9403     bind(L_fill_192_bytes_loop_header_zmm);
9404     subq(count, 192 >> shift);
9405 
9406     align32();
9407     bind(L_fill_192_bytes_loop_zmm);
9408       fill64(to, 0, xtmp, true);
9409       fill64(to, 64, xtmp, true);
9410       fill64(to, 128, xtmp, true);
9411       addq(to, 192);
9412       subq(count, 192 >> shift);
9413       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
9414 
9415     addq(count, 192 >> shift);
9416     jcc(Assembler::zero, L_exit);
9417     jmp(L_fill_start_zmm_sequence);
9418   }
9419   bind(L_exit);
9420 }
9421 #endif //COMPILER2_OR_JVMCI
9422 
9423 
9424 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
9425   Label done;
9426   cvttss2sil(dst, src);
9427   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9428   cmpl(dst, 0x80000000); // float_sign_flip
9429   jccb(Assembler::notEqual, done);
9430   subptr(rsp, 8);
9431   movflt(Address(rsp, 0), src);
9432   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9433   pop(dst);
9434   bind(done);
9435 }
9436 
9437 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
9438   Label done;
9439   cvttsd2sil(dst, src);
9440   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9441   cmpl(dst, 0x80000000); // float_sign_flip
9442   jccb(Assembler::notEqual, done);
9443   subptr(rsp, 8);
9444   movdbl(Address(rsp, 0), src);
9445   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9446   pop(dst);
9447   bind(done);
9448 }
9449 
9450 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
9451   Label done;
9452   cvttss2siq(dst, src);
9453   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9454   jccb(Assembler::notEqual, done);
9455   subptr(rsp, 8);
9456   movflt(Address(rsp, 0), src);
9457   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9458   pop(dst);
9459   bind(done);
9460 }
9461 
9462 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
9463   // Following code is line by line assembly translation rounding algorithm.
9464   // Please refer to java.lang.Math.round(float) algorithm for details.
9465   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
9466   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
9467   const int32_t FloatConsts_EXP_BIAS = 127;
9468   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
9469   const int32_t MINUS_32 = 0xFFFFFFE0;
9470   Label L_special_case, L_block1, L_exit;
9471   movl(rtmp, FloatConsts_EXP_BIT_MASK);
9472   movdl(dst, src);
9473   andl(dst, rtmp);
9474   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
9475   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
9476   subl(rtmp, dst);
9477   movl(rcx, rtmp);
9478   movl(dst, MINUS_32);
9479   testl(rtmp, dst);
9480   jccb(Assembler::notEqual, L_special_case);
9481   movdl(dst, src);
9482   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
9483   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
9484   movdl(rtmp, src);
9485   testl(rtmp, rtmp);
9486   jccb(Assembler::greaterEqual, L_block1);
9487   negl(dst);
9488   bind(L_block1);
9489   sarl(dst);
9490   addl(dst, 0x1);
9491   sarl(dst, 0x1);
9492   jmp(L_exit);
9493   bind(L_special_case);
9494   convert_f2i(dst, src);
9495   bind(L_exit);
9496 }
9497 
9498 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
9499   // Following code is line by line assembly translation rounding algorithm.
9500   // Please refer to java.lang.Math.round(double) algorithm for details.
9501   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
9502   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
9503   const int64_t DoubleConsts_EXP_BIAS = 1023;
9504   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
9505   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
9506   Label L_special_case, L_block1, L_exit;
9507   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
9508   movq(dst, src);
9509   andq(dst, rtmp);
9510   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
9511   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
9512   subq(rtmp, dst);
9513   movq(rcx, rtmp);
9514   mov64(dst, MINUS_64);
9515   testq(rtmp, dst);
9516   jccb(Assembler::notEqual, L_special_case);
9517   movq(dst, src);
9518   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
9519   andq(dst, rtmp);
9520   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
9521   orq(dst, rtmp);
9522   movq(rtmp, src);
9523   testq(rtmp, rtmp);
9524   jccb(Assembler::greaterEqual, L_block1);
9525   negq(dst);
9526   bind(L_block1);
9527   sarq(dst);
9528   addq(dst, 0x1);
9529   sarq(dst, 0x1);
9530   jmp(L_exit);
9531   bind(L_special_case);
9532   convert_d2l(dst, src);
9533   bind(L_exit);
9534 }
9535 
9536 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
9537   Label done;
9538   cvttsd2siq(dst, src);
9539   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9540   jccb(Assembler::notEqual, done);
9541   subptr(rsp, 8);
9542   movdbl(Address(rsp, 0), src);
9543   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9544   pop(dst);
9545   bind(done);
9546 }
9547 
9548 void MacroAssembler::cache_wb(Address line)
9549 {
9550   // 64 bit cpus always support clflush
9551   assert(VM_Version::supports_clflush(), "clflush should be available");
9552   bool optimized = VM_Version::supports_clflushopt();
9553   bool no_evict = VM_Version::supports_clwb();
9554 
9555   // prefer clwb (writeback without evict) otherwise
9556   // prefer clflushopt (potentially parallel writeback with evict)
9557   // otherwise fallback on clflush (serial writeback with evict)
9558 
9559   if (optimized) {
9560     if (no_evict) {
9561       clwb(line);
9562     } else {
9563       clflushopt(line);
9564     }
9565   } else {
9566     // no need for fence when using CLFLUSH
9567     clflush(line);
9568   }
9569 }
9570 
9571 void MacroAssembler::cache_wbsync(bool is_pre)
9572 {
9573   assert(VM_Version::supports_clflush(), "clflush should be available");
9574   bool optimized = VM_Version::supports_clflushopt();
9575   bool no_evict = VM_Version::supports_clwb();
9576 
9577   // pick the correct implementation
9578 
9579   if (!is_pre && (optimized || no_evict)) {
9580     // need an sfence for post flush when using clflushopt or clwb
9581     // otherwise no no need for any synchroniaztion
9582 
9583     sfence();
9584   }
9585 }
9586 
9587 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
9588   switch (cond) {
9589     // Note some conditions are synonyms for others
9590     case Assembler::zero:         return Assembler::notZero;
9591     case Assembler::notZero:      return Assembler::zero;
9592     case Assembler::less:         return Assembler::greaterEqual;
9593     case Assembler::lessEqual:    return Assembler::greater;
9594     case Assembler::greater:      return Assembler::lessEqual;
9595     case Assembler::greaterEqual: return Assembler::less;
9596     case Assembler::below:        return Assembler::aboveEqual;
9597     case Assembler::belowEqual:   return Assembler::above;
9598     case Assembler::above:        return Assembler::belowEqual;
9599     case Assembler::aboveEqual:   return Assembler::below;
9600     case Assembler::overflow:     return Assembler::noOverflow;
9601     case Assembler::noOverflow:   return Assembler::overflow;
9602     case Assembler::negative:     return Assembler::positive;
9603     case Assembler::positive:     return Assembler::negative;
9604     case Assembler::parity:       return Assembler::noParity;
9605     case Assembler::noParity:     return Assembler::parity;
9606   }
9607   ShouldNotReachHere(); return Assembler::overflow;
9608 }
9609 
9610 // This is simply a call to Thread::current()
9611 void MacroAssembler::get_thread_slow(Register thread) {
9612   if (thread != rax) {
9613     push(rax);
9614   }
9615   push(rdi);
9616   push(rsi);
9617   push(rdx);
9618   push(rcx);
9619   push(r8);
9620   push(r9);
9621   push(r10);
9622   push(r11);
9623 
9624   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
9625 
9626   pop(r11);
9627   pop(r10);
9628   pop(r9);
9629   pop(r8);
9630   pop(rcx);
9631   pop(rdx);
9632   pop(rsi);
9633   pop(rdi);
9634   if (thread != rax) {
9635     mov(thread, rax);
9636     pop(rax);
9637   }
9638 }
9639 
9640 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
9641   Label L_stack_ok;
9642   if (bias == 0) {
9643     testptr(sp, 2 * wordSize - 1);
9644   } else {
9645     // lea(tmp, Address(rsp, bias);
9646     mov(tmp, sp);
9647     addptr(tmp, bias);
9648     testptr(tmp, 2 * wordSize - 1);
9649   }
9650   jcc(Assembler::equal, L_stack_ok);
9651   block_comment(msg);
9652   stop(msg);
9653   bind(L_stack_ok);
9654 }
9655 
9656 // Implements fast-locking.
9657 //
9658 // obj: the object to be locked
9659 // reg_rax: rax
9660 // thread: the thread which attempts to lock obj
9661 // tmp: a temporary register
9662 void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register reg_rax, Register tmp, Label& slow) {
9663   Register thread = r15_thread;
9664 
9665   assert(reg_rax == rax, "");
9666   assert_different_registers(basic_lock, obj, reg_rax, thread, tmp);
9667 
9668   Label push;
9669   const Register top = tmp;
9670 
9671   // Preload the markWord. It is important that this is the first
9672   // instruction emitted as it is part of C1's null check semantics.
9673   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
9674 
9675   if (UseObjectMonitorTable) {
9676     // Clear cache in case fast locking succeeds or we need to take the slow-path.
9677     movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0);
9678   }
9679 
9680   if (DiagnoseSyncOnValueBasedClasses != 0) {
9681     load_klass(tmp, obj, rscratch1);
9682     testb(Address(tmp, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
9683     jcc(Assembler::notZero, slow);
9684   }
9685 
9686   // Load top.
9687   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9688 
9689   // Check if the lock-stack is full.
9690   cmpl(top, LockStack::end_offset());
9691   jcc(Assembler::greaterEqual, slow);
9692 
9693   // Check for recursion.
9694   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
9695   jcc(Assembler::equal, push);
9696 
9697   // Check header for monitor (0b10).
9698   testptr(reg_rax, markWord::monitor_value);
9699   jcc(Assembler::notZero, slow);
9700 
9701   // Try to lock. Transition lock bits 0b01 => 0b00
9702   movptr(tmp, reg_rax);
9703   andptr(tmp, ~(int32_t)markWord::unlocked_value);
9704   orptr(reg_rax, markWord::unlocked_value);



9705   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
9706   jcc(Assembler::notEqual, slow);
9707 
9708   // Restore top, CAS clobbers register.
9709   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9710 
9711   bind(push);
9712   // After successful lock, push object on lock-stack.
9713   movptr(Address(thread, top), obj);
9714   incrementl(top, oopSize);
9715   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
9716 }
9717 
9718 // Implements fast-unlocking.
9719 //
9720 // obj: the object to be unlocked
9721 // reg_rax: rax
9722 // thread: the thread
9723 // tmp: a temporary register
9724 void MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register tmp, Label& slow) {
9725   Register thread = r15_thread;
9726 
9727   assert(reg_rax == rax, "");
9728   assert_different_registers(obj, reg_rax, thread, tmp);
9729 
9730   Label unlocked, push_and_slow;
9731   const Register top = tmp;
9732 
9733   // Check if obj is top of lock-stack.
9734   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9735   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
9736   jcc(Assembler::notEqual, slow);
9737 
9738   // Pop lock-stack.
9739   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
9740   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
9741 
9742   // Check if recursive.
9743   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
9744   jcc(Assembler::equal, unlocked);
9745 
9746   // Not recursive. Check header for monitor (0b10).
9747   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
9748   testptr(reg_rax, markWord::monitor_value);
9749   jcc(Assembler::notZero, push_and_slow);
9750 
9751 #ifdef ASSERT
9752   // Check header not unlocked (0b01).
9753   Label not_unlocked;
9754   testptr(reg_rax, markWord::unlocked_value);
9755   jcc(Assembler::zero, not_unlocked);
9756   stop("fast_unlock already unlocked");
9757   bind(not_unlocked);
9758 #endif
9759 
9760   // Try to unlock. Transition lock bits 0b00 => 0b01
9761   movptr(tmp, reg_rax);
9762   orptr(tmp, markWord::unlocked_value);
9763   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
9764   jcc(Assembler::equal, unlocked);
9765 
9766   bind(push_and_slow);
9767   // Restore lock-stack and handle the unlock in runtime.
9768 #ifdef ASSERT
9769   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
9770   movptr(Address(thread, top), obj);
9771 #endif
9772   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
9773   jmp(slow);
9774 
9775   bind(unlocked);
9776 }
9777 
9778 // Saves legacy GPRs state on stack.
9779 void MacroAssembler::save_legacy_gprs() {
9780   subq(rsp, 16 * wordSize);
9781   movq(Address(rsp, 15 * wordSize), rax);
9782   movq(Address(rsp, 14 * wordSize), rcx);
9783   movq(Address(rsp, 13 * wordSize), rdx);
9784   movq(Address(rsp, 12 * wordSize), rbx);
9785   movq(Address(rsp, 10 * wordSize), rbp);
9786   movq(Address(rsp, 9 * wordSize), rsi);
9787   movq(Address(rsp, 8 * wordSize), rdi);
9788   movq(Address(rsp, 7 * wordSize), r8);
9789   movq(Address(rsp, 6 * wordSize), r9);
9790   movq(Address(rsp, 5 * wordSize), r10);
9791   movq(Address(rsp, 4 * wordSize), r11);
9792   movq(Address(rsp, 3 * wordSize), r12);
9793   movq(Address(rsp, 2 * wordSize), r13);
9794   movq(Address(rsp, wordSize), r14);
9795   movq(Address(rsp, 0), r15);
9796 }
9797 
9798 // Resotres back legacy GPRs state from stack.
9799 void MacroAssembler::restore_legacy_gprs() {
9800   movq(r15, Address(rsp, 0));
9801   movq(r14, Address(rsp, wordSize));
9802   movq(r13, Address(rsp, 2 * wordSize));
9803   movq(r12, Address(rsp, 3 * wordSize));
9804   movq(r11, Address(rsp, 4 * wordSize));
9805   movq(r10, Address(rsp, 5 * wordSize));
9806   movq(r9,  Address(rsp, 6 * wordSize));
9807   movq(r8,  Address(rsp, 7 * wordSize));
9808   movq(rdi, Address(rsp, 8 * wordSize));
9809   movq(rsi, Address(rsp, 9 * wordSize));
9810   movq(rbp, Address(rsp, 10 * wordSize));
9811   movq(rbx, Address(rsp, 12 * wordSize));
9812   movq(rdx, Address(rsp, 13 * wordSize));
9813   movq(rcx, Address(rsp, 14 * wordSize));
9814   movq(rax, Address(rsp, 15 * wordSize));
9815   addq(rsp, 16 * wordSize);
9816 }
9817 
9818 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
9819   if (VM_Version::supports_apx_f()) {
9820     esetzucc(comparison, dst);
9821   } else {
9822     setb(comparison, dst);
9823     movzbl(dst, dst);
9824   }
9825 }
--- EOF ---