1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef CPU_X86_MACROASSEMBLER_X86_HPP
  26 #define CPU_X86_MACROASSEMBLER_X86_HPP
  27 
  28 #include "asm/assembler.hpp"
  29 #include "asm/register.hpp"
  30 #include "code/vmreg.inline.hpp"
  31 #include "compiler/oopMap.hpp"
  32 #include "utilities/macros.hpp"
  33 #include "runtime/rtmLocking.hpp"
  34 #include "runtime/signature.hpp"
  35 #include "runtime/vm_version.hpp"
  36 
  37 class ciInlineKlass;
  38 
  39 // MacroAssembler extends Assembler by frequently used macros.
  40 //
  41 // Instructions for which a 'better' code sequence exists depending
  42 // on arguments should also go in here.
  43 
  44 class MacroAssembler: public Assembler {
  45   friend class LIR_Assembler;
  46   friend class Runtime1;      // as_Address()
  47 
  48  public:
  49   // Support for VM calls
  50   //
  51   // This is the base routine called by the different versions of call_VM_leaf. The interpreter
  52   // may customize this version by overriding it for its purposes (e.g., to save/restore
  53   // additional registers when doing a VM call).
  54 
  55   virtual void call_VM_leaf_base(
  56     address entry_point,               // the entry point
  57     int     number_of_arguments        // the number of arguments to pop after the call
  58   );
  59 
  60  protected:
  61   // This is the base routine called by the different versions of call_VM. The interpreter
  62   // may customize this version by overriding it for its purposes (e.g., to save/restore
  63   // additional registers when doing a VM call).
  64   //
  65   // If no java_thread register is specified (noreg) than rdi will be used instead. call_VM_base
  66   // returns the register which contains the thread upon return. If a thread register has been
  67   // specified, the return value will correspond to that register. If no last_java_sp is specified
  68   // (noreg) than rsp will be used instead.
  69   virtual void call_VM_base(           // returns the register containing the thread upon return
  70     Register oop_result,               // where an oop-result ends up if any; use noreg otherwise
  71     Register java_thread,              // the thread if computed before     ; use noreg otherwise
  72     Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise
  73     address  entry_point,              // the entry point
  74     int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call
  75     bool     check_exceptions          // whether to check for pending exceptions after return
  76   );
  77 
  78   void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
  79 
  80   // helpers for FPU flag access
  81   // tmp is a temporary register, if none is available use noreg
  82   void save_rax   (Register tmp);
  83   void restore_rax(Register tmp);
  84 
  85  public:
  86   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
  87 
  88  // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
  89  // The implementation is only non-empty for the InterpreterMacroAssembler,
  90  // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
  91  virtual void check_and_handle_popframe(Register java_thread);
  92  virtual void check_and_handle_earlyret(Register java_thread);
  93 
  94   Address as_Address(AddressLiteral adr);
  95   Address as_Address(ArrayAddress adr, Register rscratch);
  96 
  97   // Support for null-checks
  98   //
  99   // Generates code that causes a null OS exception if the content of reg is null.
 100   // If the accessed location is M[reg + offset] and the offset is known, provide the
 101   // offset. No explicit code generation is needed if the offset is within a certain
 102   // range (0 <= offset <= page_size).
 103 
 104   void null_check(Register reg, int offset = -1);
 105   static bool needs_explicit_null_check(intptr_t offset);
 106   static bool uses_implicit_null_check(void* address);
 107 
 108   // markWord tests, kills markWord reg
 109   void test_markword_is_inline_type(Register markword, Label& is_inline_type);
 110 
 111   // inlineKlass queries, kills temp_reg
 112   void test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type);
 113   void test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type);
 114   void test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type);
 115 
 116   // Get the default value oop for the given InlineKlass
 117   void get_default_value_oop(Register inline_klass, Register temp_reg, Register obj);
 118   // The empty value oop, for the given InlineKlass ("empty" as in no instance fields)
 119   // get_default_value_oop with extra assertion for empty inline klass
 120   void get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj);
 121 
 122   void test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free);
 123   void test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free);
 124   void test_field_is_flat(Register flags, Register temp_reg, Label& is_flat);
 125 
 126   // Check oops for special arrays, i.e. flat arrays and/or null-free arrays
 127   void test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label);
 128   void test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array);
 129   void test_non_flat_array_oop(Register oop, Register temp_reg, Label& is_non_flat_array);
 130   void test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array);
 131   void test_non_null_free_array_oop(Register oop, Register temp_reg, Label& is_non_null_free_array);
 132 
 133   // Check array klass layout helper for flat or null-free arrays...
 134   void test_flat_array_layout(Register lh, Label& is_flat_array);
 135   void test_non_flat_array_layout(Register lh, Label& is_non_flat_array);
 136   void test_null_free_array_layout(Register lh, Label& is_null_free_array);
 137   void test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array);
 138 
 139   // Required platform-specific helpers for Label::patch_instructions.
 140   // They _shadow_ the declarations in AbstractAssembler, which are undefined.
 141   void pd_patch_instruction(address branch, address target, const char* file, int line) {
 142     unsigned char op = branch[0];
 143     assert(op == 0xE8 /* call */ ||
 144         op == 0xE9 /* jmp */ ||
 145         op == 0xEB /* short jmp */ ||
 146         (op & 0xF0) == 0x70 /* short jcc */ ||
 147         op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
 148         op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
 149         "Invalid opcode at patch point");
 150 
 151     if (op == 0xEB || (op & 0xF0) == 0x70) {
 152       // short offset operators (jmp and jcc)
 153       char* disp = (char*) &branch[1];
 154       int imm8 = checked_cast<int>(target - (address) &disp[1]);
 155       guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset at %s:%d",
 156                 file == nullptr ? "<null>" : file, line);
 157       *disp = (char)imm8;
 158     } else {
 159       int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1];
 160       int imm32 = checked_cast<int>(target - (address) &disp[1]);
 161       *disp = imm32;
 162     }
 163   }
 164 
 165   // The following 4 methods return the offset of the appropriate move instruction
 166 
 167   // Support for fast byte/short loading with zero extension (depending on particular CPU)
 168   int load_unsigned_byte(Register dst, Address src);
 169   int load_unsigned_short(Register dst, Address src);
 170 
 171   // Support for fast byte/short loading with sign extension (depending on particular CPU)
 172   int load_signed_byte(Register dst, Address src);
 173   int load_signed_short(Register dst, Address src);
 174 
 175   // Support for sign-extension (hi:lo = extend_sign(lo))
 176   void extend_sign(Register hi, Register lo);
 177 
 178   // Load and store values by size and signed-ness
 179   void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
 180   void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
 181 
 182   // Support for inc/dec with optimal instruction selection depending on value
 183 
 184   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
 185   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
 186   void increment(Address dst, int value = 1)  { LP64_ONLY(incrementq(dst, value)) NOT_LP64(incrementl(dst, value)) ; }
 187   void decrement(Address dst, int value = 1)  { LP64_ONLY(decrementq(dst, value)) NOT_LP64(decrementl(dst, value)) ; }
 188 
 189   void decrementl(Address dst, int value = 1);
 190   void decrementl(Register reg, int value = 1);
 191 
 192   void decrementq(Register reg, int value = 1);
 193   void decrementq(Address dst, int value = 1);
 194 
 195   void incrementl(Address dst, int value = 1);
 196   void incrementl(Register reg, int value = 1);
 197 
 198   void incrementq(Register reg, int value = 1);
 199   void incrementq(Address dst, int value = 1);
 200 
 201   void incrementl(AddressLiteral dst, Register rscratch = noreg);
 202   void incrementl(ArrayAddress   dst, Register rscratch);
 203 
 204   void incrementq(AddressLiteral dst, Register rscratch = noreg);
 205 
 206   // Support optimal SSE move instructions.
 207   void movflt(XMMRegister dst, XMMRegister src) {
 208     if (dst-> encoding() == src->encoding()) return;
 209     if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
 210     else                       { movss (dst, src); return; }
 211   }
 212   void movflt(XMMRegister dst, Address src) { movss(dst, src); }
 213   void movflt(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 214   void movflt(Address dst, XMMRegister src) { movss(dst, src); }
 215 
 216   // Move with zero extension
 217   void movfltz(XMMRegister dst, XMMRegister src) { movss(dst, src); }
 218 
 219   void movdbl(XMMRegister dst, XMMRegister src) {
 220     if (dst-> encoding() == src->encoding()) return;
 221     if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
 222     else                       { movsd (dst, src); return; }
 223   }
 224 
 225   void movdbl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 226 
 227   void movdbl(XMMRegister dst, Address src) {
 228     if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
 229     else                         { movlpd(dst, src); return; }
 230   }
 231   void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
 232 
 233   void flt_to_flt16(Register dst, XMMRegister src, XMMRegister tmp) {
 234     // Use separate tmp XMM register because caller may
 235     // requires src XMM register to be unchanged (as in x86.ad).
 236     vcvtps2ph(tmp, src, 0x04, Assembler::AVX_128bit);
 237     movdl(dst, tmp);
 238     movswl(dst, dst);
 239   }
 240 
 241   void flt16_to_flt(XMMRegister dst, Register src) {
 242     movdl(dst, src);
 243     vcvtph2ps(dst, dst, Assembler::AVX_128bit);
 244   }
 245 
 246   // Alignment
 247   void align32();
 248   void align64();
 249   void align(int modulus);
 250   void align(int modulus, int target);
 251 
 252   void post_call_nop();
 253   // A 5 byte nop that is safe for patching (see patch_verified_entry)
 254   void fat_nop();
 255 
 256   // Stack frame creation/removal
 257   void enter();
 258   void leave();
 259 
 260   // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
 261   // The pointer will be loaded into the thread register.
 262   void get_thread(Register thread);
 263 
 264 #ifdef _LP64
 265   // Support for argument shuffling
 266 
 267   // bias in bytes
 268   void move32_64(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 269   void long_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 270   void float_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 271   void double_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 272   void move_ptr(VMRegPair src, VMRegPair dst);
 273   void object_move(OopMap* map,
 274                    int oop_handle_offset,
 275                    int framesize_in_slots,
 276                    VMRegPair src,
 277                    VMRegPair dst,
 278                    bool is_receiver,
 279                    int* receiver_offset);
 280 #endif // _LP64
 281 
 282   // Support for VM calls
 283   //
 284   // It is imperative that all calls into the VM are handled via the call_VM macros.
 285   // They make sure that the stack linkage is setup correctly. call_VM's correspond
 286   // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
 287 
 288 
 289   void call_VM(Register oop_result,
 290                address entry_point,
 291                bool check_exceptions = true);
 292   void call_VM(Register oop_result,
 293                address entry_point,
 294                Register arg_1,
 295                bool check_exceptions = true);
 296   void call_VM(Register oop_result,
 297                address entry_point,
 298                Register arg_1, Register arg_2,
 299                bool check_exceptions = true);
 300   void call_VM(Register oop_result,
 301                address entry_point,
 302                Register arg_1, Register arg_2, Register arg_3,
 303                bool check_exceptions = true);
 304 
 305   // Overloadings with last_Java_sp
 306   void call_VM(Register oop_result,
 307                Register last_java_sp,
 308                address entry_point,
 309                int number_of_arguments = 0,
 310                bool check_exceptions = true);
 311   void call_VM(Register oop_result,
 312                Register last_java_sp,
 313                address entry_point,
 314                Register arg_1, bool
 315                check_exceptions = true);
 316   void call_VM(Register oop_result,
 317                Register last_java_sp,
 318                address entry_point,
 319                Register arg_1, Register arg_2,
 320                bool check_exceptions = true);
 321   void call_VM(Register oop_result,
 322                Register last_java_sp,
 323                address entry_point,
 324                Register arg_1, Register arg_2, Register arg_3,
 325                bool check_exceptions = true);
 326 
 327   void get_vm_result  (Register oop_result, Register thread);
 328   void get_vm_result_2(Register metadata_result, Register thread);
 329 
 330   // These always tightly bind to MacroAssembler::call_VM_base
 331   // bypassing the virtual implementation
 332   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
 333   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
 334   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
 335   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
 336   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
 337 
 338   void call_VM_leaf0(address entry_point);
 339   void call_VM_leaf(address entry_point,
 340                     int number_of_arguments = 0);
 341   void call_VM_leaf(address entry_point,
 342                     Register arg_1);
 343   void call_VM_leaf(address entry_point,
 344                     Register arg_1, Register arg_2);
 345   void call_VM_leaf(address entry_point,
 346                     Register arg_1, Register arg_2, Register arg_3);
 347 
 348   void call_VM_leaf(address entry_point,
 349                     Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 350 
 351   // These always tightly bind to MacroAssembler::call_VM_leaf_base
 352   // bypassing the virtual implementation
 353   void super_call_VM_leaf(address entry_point);
 354   void super_call_VM_leaf(address entry_point, Register arg_1);
 355   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
 356   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
 357   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 358 
 359   // last Java Frame (fills frame anchor)
 360   void set_last_Java_frame(Register thread,
 361                            Register last_java_sp,
 362                            Register last_java_fp,
 363                            address  last_java_pc,
 364                            Register rscratch);
 365 
 366   // thread in the default location (r15_thread on 64bit)
 367   void set_last_Java_frame(Register last_java_sp,
 368                            Register last_java_fp,
 369                            address  last_java_pc,
 370                            Register rscratch);
 371 
 372   void reset_last_Java_frame(Register thread, bool clear_fp);
 373 
 374   // thread in the default location (r15_thread on 64bit)
 375   void reset_last_Java_frame(bool clear_fp);
 376 
 377   // jobjects
 378   void clear_jobject_tag(Register possibly_non_local);
 379   void resolve_jobject(Register value, Register thread, Register tmp);
 380   void resolve_global_jobject(Register value, Register thread, Register tmp);
 381 
 382   // C 'boolean' to Java boolean: x == 0 ? 0 : 1
 383   void c2bool(Register x);
 384 
 385   // C++ bool manipulation
 386 
 387   void movbool(Register dst, Address src);
 388   void movbool(Address dst, bool boolconst);
 389   void movbool(Address dst, Register src);
 390   void testbool(Register dst);
 391 
 392   void resolve_oop_handle(Register result, Register tmp);
 393   void resolve_weak_handle(Register result, Register tmp);
 394   void load_mirror(Register mirror, Register method, Register tmp);
 395   void load_method_holder_cld(Register rresult, Register rmethod);
 396 
 397   void load_method_holder(Register holder, Register method);
 398 
 399   // oop manipulations
 400   void load_metadata(Register dst, Register src);
 401   void load_klass(Register dst, Register src, Register tmp);
 402   void store_klass(Register dst, Register src, Register tmp);
 403 
 404   void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 405                       Register tmp1, Register thread_tmp);
 406   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 407                        Register tmp1, Register tmp2, Register tmp3);
 408 
 409   void access_value_copy(DecoratorSet decorators, Register src, Register dst, Register inline_klass);
 410 
 411   // inline type data payload offsets...
 412   void first_field_offset(Register inline_klass, Register offset);
 413   void data_for_oop(Register oop, Register data, Register inline_klass);
 414   // get data payload ptr a flat value array at index, kills rcx and index
 415   void data_for_value_array_index(Register array, Register array_klass,
 416                                   Register index, Register data);
 417 
 418 
 419   void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
 420                      Register thread_tmp = noreg, DecoratorSet decorators = 0);
 421   void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
 422                               Register thread_tmp = noreg, DecoratorSet decorators = 0);
 423   void store_heap_oop(Address dst, Register val, Register tmp1 = noreg,
 424                       Register tmp2 = noreg, Register tmp3 = noreg, DecoratorSet decorators = 0);
 425 
 426   // Used for storing null. All other oop constants should be
 427   // stored using routines that take a jobject.
 428   void store_heap_oop_null(Address dst);
 429 
 430   void load_prototype_header(Register dst, Register src, Register tmp);
 431 
 432 #ifdef _LP64
 433   void store_klass_gap(Register dst, Register src);
 434 
 435   // This dummy is to prevent a call to store_heap_oop from
 436   // converting a zero (like null) into a Register by giving
 437   // the compiler two choices it can't resolve
 438 
 439   void store_heap_oop(Address dst, void* dummy);
 440 
 441   void encode_heap_oop(Register r);
 442   void decode_heap_oop(Register r);
 443   void encode_heap_oop_not_null(Register r);
 444   void decode_heap_oop_not_null(Register r);
 445   void encode_heap_oop_not_null(Register dst, Register src);
 446   void decode_heap_oop_not_null(Register dst, Register src);
 447 
 448   void set_narrow_oop(Register dst, jobject obj);
 449   void set_narrow_oop(Address dst, jobject obj);
 450   void cmp_narrow_oop(Register dst, jobject obj);
 451   void cmp_narrow_oop(Address dst, jobject obj);
 452 
 453   void encode_klass_not_null(Register r, Register tmp);
 454   void decode_klass_not_null(Register r, Register tmp);
 455   void encode_and_move_klass_not_null(Register dst, Register src);
 456   void decode_and_move_klass_not_null(Register dst, Register src);
 457   void set_narrow_klass(Register dst, Klass* k);
 458   void set_narrow_klass(Address dst, Klass* k);
 459   void cmp_narrow_klass(Register dst, Klass* k);
 460   void cmp_narrow_klass(Address dst, Klass* k);
 461 
 462   // if heap base register is used - reinit it with the correct value
 463   void reinit_heapbase();
 464 
 465   DEBUG_ONLY(void verify_heapbase(const char* msg);)
 466 
 467 #endif // _LP64
 468 
 469   // Int division/remainder for Java
 470   // (as idivl, but checks for special case as described in JVM spec.)
 471   // returns idivl instruction offset for implicit exception handling
 472   int corrected_idivl(Register reg);
 473 
 474   // Long division/remainder for Java
 475   // (as idivq, but checks for special case as described in JVM spec.)
 476   // returns idivq instruction offset for implicit exception handling
 477   int corrected_idivq(Register reg);
 478 
 479   void int3();
 480 
 481   // Long operation macros for a 32bit cpu
 482   // Long negation for Java
 483   void lneg(Register hi, Register lo);
 484 
 485   // Long multiplication for Java
 486   // (destroys contents of eax, ebx, ecx and edx)
 487   void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
 488 
 489   // Long shifts for Java
 490   // (semantics as described in JVM spec.)
 491   void lshl(Register hi, Register lo);                               // hi:lo << (rcx & 0x3f)
 492   void lshr(Register hi, Register lo, bool sign_extension = false);  // hi:lo >> (rcx & 0x3f)
 493 
 494   // Long compare for Java
 495   // (semantics as described in JVM spec.)
 496   void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
 497 
 498 
 499   // misc
 500 
 501   // Sign extension
 502   void sign_extend_short(Register reg);
 503   void sign_extend_byte(Register reg);
 504 
 505   // Division by power of 2, rounding towards 0
 506   void division_with_shift(Register reg, int shift_value);
 507 
 508 #ifndef _LP64
 509   // Compares the top-most stack entries on the FPU stack and sets the eflags as follows:
 510   //
 511   // CF (corresponds to C0) if x < y
 512   // PF (corresponds to C2) if unordered
 513   // ZF (corresponds to C3) if x = y
 514   //
 515   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 516   // tmp is a temporary register, if none is available use noreg (only matters for non-P6 code)
 517   void fcmp(Register tmp);
 518   // Variant of the above which allows y to be further down the stack
 519   // and which only pops x and y if specified. If pop_right is
 520   // specified then pop_left must also be specified.
 521   void fcmp(Register tmp, int index, bool pop_left, bool pop_right);
 522 
 523   // Floating-point comparison for Java
 524   // Compares the top-most stack entries on the FPU stack and stores the result in dst.
 525   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 526   // (semantics as described in JVM spec.)
 527   void fcmp2int(Register dst, bool unordered_is_less);
 528   // Variant of the above which allows y to be further down the stack
 529   // and which only pops x and y if specified. If pop_right is
 530   // specified then pop_left must also be specified.
 531   void fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right);
 532 
 533   // Floating-point remainder for Java (ST0 = ST0 fremr ST1, ST1 is empty afterwards)
 534   // tmp is a temporary register, if none is available use noreg
 535   void fremr(Register tmp);
 536 
 537   // only if +VerifyFPU
 538   void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
 539 #endif // !LP64
 540 
 541   // dst = c = a * b + c
 542   void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 543   void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 544 
 545   void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 546   void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 547   void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 548   void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 549 
 550 
 551   // same as fcmp2int, but using SSE2
 552   void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 553   void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 554 
 555   // branch to L if FPU flag C2 is set/not set
 556   // tmp is a temporary register, if none is available use noreg
 557   void jC2 (Register tmp, Label& L);
 558   void jnC2(Register tmp, Label& L);
 559 
 560   // Load float value from 'address'. If UseSSE >= 1, the value is loaded into
 561   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 562   void load_float(Address src);
 563 
 564   // Store float value to 'address'. If UseSSE >= 1, the value is stored
 565   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 566   void store_float(Address dst);
 567 
 568   // Load double value from 'address'. If UseSSE >= 2, the value is loaded into
 569   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 570   void load_double(Address src);
 571 
 572   // Store double value to 'address'. If UseSSE >= 2, the value is stored
 573   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 574   void store_double(Address dst);
 575 
 576 #ifndef _LP64
 577   // Pop ST (ffree & fincstp combined)
 578   void fpop();
 579 
 580   void empty_FPU_stack();
 581 #endif // !_LP64
 582 
 583   void push_IU_state();
 584   void pop_IU_state();
 585 
 586   void push_FPU_state();
 587   void pop_FPU_state();
 588 
 589   void push_CPU_state();
 590   void pop_CPU_state();
 591 
 592   void push_cont_fastpath();
 593   void pop_cont_fastpath();
 594 
 595   void inc_held_monitor_count();
 596   void dec_held_monitor_count();
 597 
 598   DEBUG_ONLY(void stop_if_in_cont(Register cont_reg, const char* name);)
 599 
 600   // Round up to a power of two
 601   void round_to(Register reg, int modulus);
 602 
 603 private:
 604   // General purpose and XMM registers potentially clobbered by native code; there
 605   // is no need for FPU or AVX opmask related methods because C1/interpreter
 606   // - we save/restore FPU state as a whole always
 607   // - do not care about AVX-512 opmask
 608   static RegSet call_clobbered_gp_registers();
 609   static XMMRegSet call_clobbered_xmm_registers();
 610 
 611   void push_set(XMMRegSet set, int offset);
 612   void pop_set(XMMRegSet set, int offset);
 613 
 614 public:
 615   void push_set(RegSet set, int offset = -1);
 616   void pop_set(RegSet set, int offset = -1);
 617 
 618   // Push and pop everything that might be clobbered by a native
 619   // runtime call.
 620   // Only save the lower 64 bits of each vector register.
 621   // Additional registers can be excluded in a passed RegSet.
 622   void push_call_clobbered_registers_except(RegSet exclude, bool save_fpu = true);
 623   void pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu = true);
 624 
 625   void push_call_clobbered_registers(bool save_fpu = true) {
 626     push_call_clobbered_registers_except(RegSet(), save_fpu);
 627   }
 628   void pop_call_clobbered_registers(bool restore_fpu = true) {
 629     pop_call_clobbered_registers_except(RegSet(), restore_fpu);
 630   }
 631 
 632   // allocation
 633 
 634   // Object / value buffer allocation...
 635   // Allocate instance of klass, assumes klass initialized by caller
 636   // new_obj prefers to be rax
 637   // Kills t1 and t2, perserves klass, return allocation in new_obj (rsi on LP64)
 638   void allocate_instance(Register klass, Register new_obj,
 639                          Register t1, Register t2,
 640                          bool clear_fields, Label& alloc_failed);
 641 
 642   void tlab_allocate(
 643     Register thread,                   // Current thread
 644     Register obj,                      // result: pointer to object after successful allocation
 645     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 646     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 647     Register t1,                       // temp register
 648     Register t2,                       // temp register
 649     Label&   slow_case                 // continuation point if fast allocation fails
 650   );
 651   void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp);
 652 
 653   // For field "index" within "klass", return inline_klass ...
 654   void get_inline_type_field_klass(Register klass, Register index, Register inline_klass);
 655 
 656   // interface method calling
 657   void lookup_interface_method(Register recv_klass,
 658                                Register intf_klass,
 659                                RegisterOrConstant itable_index,
 660                                Register method_result,
 661                                Register scan_temp,
 662                                Label& no_such_interface,
 663                                bool return_method = true);
 664 
 665   void lookup_interface_method_stub(Register recv_klass,
 666                                     Register holder_klass,
 667                                     Register resolved_klass,
 668                                     Register method_result,
 669                                     Register scan_temp,
 670                                     Register temp_reg2,
 671                                     Register receiver,
 672                                     int itable_index,
 673                                     Label& L_no_such_interface);
 674 
 675   // virtual method calling
 676   void lookup_virtual_method(Register recv_klass,
 677                              RegisterOrConstant vtable_index,
 678                              Register method_result);
 679 
 680   // Test sub_klass against super_klass, with fast and slow paths.
 681 
 682   // The fast path produces a tri-state answer: yes / no / maybe-slow.
 683   // One of the three labels can be null, meaning take the fall-through.
 684   // If super_check_offset is -1, the value is loaded up from super_klass.
 685   // No registers are killed, except temp_reg.
 686   void check_klass_subtype_fast_path(Register sub_klass,
 687                                      Register super_klass,
 688                                      Register temp_reg,
 689                                      Label* L_success,
 690                                      Label* L_failure,
 691                                      Label* L_slow_path,
 692                 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
 693 
 694   // The rest of the type check; must be wired to a corresponding fast path.
 695   // It does not repeat the fast path logic, so don't use it standalone.
 696   // The temp_reg and temp2_reg can be noreg, if no temps are available.
 697   // Updates the sub's secondary super cache as necessary.
 698   // If set_cond_codes, condition codes will be Z on success, NZ on failure.
 699   void check_klass_subtype_slow_path(Register sub_klass,
 700                                      Register super_klass,
 701                                      Register temp_reg,
 702                                      Register temp2_reg,
 703                                      Label* L_success,
 704                                      Label* L_failure,
 705                                      bool set_cond_codes = false);
 706 
 707   // Simplified, combined version, good for typical uses.
 708   // Falls through on failure.
 709   void check_klass_subtype(Register sub_klass,
 710                            Register super_klass,
 711                            Register temp_reg,
 712                            Label& L_success);
 713 
 714   void clinit_barrier(Register klass,
 715                       Register thread,
 716                       Label* L_fast_path = nullptr,
 717                       Label* L_slow_path = nullptr);
 718 
 719   // method handles (JSR 292)
 720   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
 721 
 722   // Debugging
 723 
 724   // only if +VerifyOops
 725   void _verify_oop(Register reg, const char* s, const char* file, int line);
 726   void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
 727 
 728   void _verify_oop_checked(Register reg, const char* s, const char* file, int line) {
 729     if (VerifyOops) {
 730       _verify_oop(reg, s, file, line);
 731     }
 732   }
 733   void _verify_oop_addr_checked(Address reg, const char* s, const char* file, int line) {
 734     if (VerifyOops) {
 735       _verify_oop_addr(reg, s, file, line);
 736     }
 737   }
 738 
 739   // TODO: verify method and klass metadata (compare against vptr?)
 740   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
 741   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
 742 
 743 #define verify_oop(reg) _verify_oop_checked(reg, "broken oop " #reg, __FILE__, __LINE__)
 744 #define verify_oop_msg(reg, msg) _verify_oop_checked(reg, "broken oop " #reg ", " #msg, __FILE__, __LINE__)
 745 #define verify_oop_addr(addr) _verify_oop_addr_checked(addr, "broken oop addr " #addr, __FILE__, __LINE__)
 746 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
 747 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
 748 
 749   // Verify or restore cpu control state after JNI call
 750   void restore_cpu_control_state_after_jni(Register rscratch);
 751 
 752   // prints msg, dumps registers and stops execution
 753   void stop(const char* msg);
 754 
 755   // prints msg and continues
 756   void warn(const char* msg);
 757 
 758   // dumps registers and other state
 759   void print_state();
 760 
 761   static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
 762   static void debug64(char* msg, int64_t pc, int64_t regs[]);
 763   static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
 764   static void print_state64(int64_t pc, int64_t regs[]);
 765 
 766   void os_breakpoint();
 767 
 768   void untested()                                { stop("untested"); }
 769 
 770   void unimplemented(const char* what = "");
 771 
 772   void should_not_reach_here()                   { stop("should not reach here"); }
 773 
 774   void print_CPU_state();
 775 
 776   // Stack overflow checking
 777   void bang_stack_with_offset(int offset) {
 778     // stack grows down, caller passes positive offset
 779     assert(offset > 0, "must bang with negative offset");
 780     movl(Address(rsp, (-offset)), rax);
 781   }
 782 
 783   // Writes to stack successive pages until offset reached to check for
 784   // stack overflow + shadow pages.  Also, clobbers tmp
 785   void bang_stack_size(Register size, Register tmp);
 786 
 787   // Check for reserved stack access in method being exited (for JIT)
 788   void reserved_stack_check();
 789 
 790   void safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod);
 791 
 792   void verify_tlab();
 793 
 794   static Condition negate_condition(Condition cond);
 795 
 796   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
 797   // operands. In general the names are modified to avoid hiding the instruction in Assembler
 798   // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
 799   // here in MacroAssembler. The major exception to this rule is call
 800 
 801   // Arithmetics
 802 
 803 
 804   void addptr(Address dst, int32_t src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)) ; }
 805   void addptr(Address dst, Register src);
 806 
 807   void addptr(Register dst, Address src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); }
 808   void addptr(Register dst, int32_t src);
 809   void addptr(Register dst, Register src);
 810   void addptr(Register dst, RegisterOrConstant src) {
 811     if (src.is_constant()) addptr(dst, checked_cast<int>(src.as_constant()));
 812     else                   addptr(dst, src.as_register());
 813   }
 814 
 815   void andptr(Register dst, int32_t src);
 816   void andptr(Register dst, Register src) { LP64_ONLY(andq(dst, src)) NOT_LP64(andl(dst, src)) ; }
 817   void andptr(Register dst, Address src) { LP64_ONLY(andq(dst, src)) NOT_LP64(andl(dst, src)) ; }
 818 
 819 #ifdef _LP64
 820   using Assembler::andq;
 821   void andq(Register dst, AddressLiteral src, Register rscratch = noreg);
 822 #endif
 823 
 824   void cmp8(AddressLiteral src1, int imm, Register rscratch = noreg);
 825 
 826   // renamed to drag out the casting of address to int32_t/intptr_t
 827   void cmp32(Register src1, int32_t imm);
 828 
 829   void cmp32(AddressLiteral src1, int32_t imm, Register rscratch = noreg);
 830   // compare reg - mem, or reg - &mem
 831   void cmp32(Register src1, AddressLiteral src2, Register rscratch = noreg);
 832 
 833   void cmp32(Register src1, Address src2);
 834 
 835 #ifndef _LP64
 836   void cmpklass(Address dst, Metadata* obj);
 837   void cmpklass(Register dst, Metadata* obj);
 838   void cmpoop(Address dst, jobject obj);
 839 #endif // _LP64
 840 
 841   void cmpoop(Register src1, Register src2);
 842   void cmpoop(Register src1, Address src2);
 843   void cmpoop(Register dst, jobject obj, Register rscratch);
 844 
 845   // NOTE src2 must be the lval. This is NOT an mem-mem compare
 846   void cmpptr(Address src1, AddressLiteral src2, Register rscratch);
 847 
 848   void cmpptr(Register src1, AddressLiteral src2, Register rscratch = noreg);
 849 
 850   void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 851   void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 852   // void cmpptr(Address src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 853 
 854   void cmpptr(Register src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 855   void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 856 
 857   // cmp64 to avoild hiding cmpq
 858   void cmp64(Register src1, AddressLiteral src, Register rscratch = noreg);
 859 
 860   void cmpxchgptr(Register reg, Address adr);
 861 
 862   void locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch = noreg);
 863 
 864   void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
 865   void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
 866 
 867 
 868   void negptr(Register dst) { LP64_ONLY(negq(dst)) NOT_LP64(negl(dst)); }
 869 
 870   void notptr(Register dst) { LP64_ONLY(notq(dst)) NOT_LP64(notl(dst)); }
 871 
 872   void shlptr(Register dst, int32_t shift);
 873   void shlptr(Register dst) { LP64_ONLY(shlq(dst)) NOT_LP64(shll(dst)); }
 874 
 875   void shrptr(Register dst, int32_t shift);
 876   void shrptr(Register dst) { LP64_ONLY(shrq(dst)) NOT_LP64(shrl(dst)); }
 877 
 878   void sarptr(Register dst) { LP64_ONLY(sarq(dst)) NOT_LP64(sarl(dst)); }
 879   void sarptr(Register dst, int32_t src) { LP64_ONLY(sarq(dst, src)) NOT_LP64(sarl(dst, src)); }
 880 
 881   void subptr(Address dst, int32_t src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 882 
 883   void subptr(Register dst, Address src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 884   void subptr(Register dst, int32_t src);
 885   // Force generation of a 4 byte immediate value even if it fits into 8bit
 886   void subptr_imm32(Register dst, int32_t src);
 887   void subptr(Register dst, Register src);
 888   void subptr(Register dst, RegisterOrConstant src) {
 889     if (src.is_constant()) subptr(dst, (int) src.as_constant());
 890     else                   subptr(dst,       src.as_register());
 891   }
 892 
 893   void sbbptr(Address dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 894   void sbbptr(Register dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 895 
 896   void xchgptr(Register src1, Register src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 897   void xchgptr(Register src1, Address src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 898 
 899   void xaddptr(Address src1, Register src2) { LP64_ONLY(xaddq(src1, src2)) NOT_LP64(xaddl(src1, src2)) ; }
 900 
 901 
 902 
 903   // Helper functions for statistics gathering.
 904   // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
 905   void cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch = noreg);
 906   // Unconditional atomic increment.
 907   void atomic_incl(Address counter_addr);
 908   void atomic_incl(AddressLiteral counter_addr, Register rscratch = noreg);
 909 #ifdef _LP64
 910   void atomic_incq(Address counter_addr);
 911   void atomic_incq(AddressLiteral counter_addr, Register rscratch = noreg);
 912 #endif
 913   void atomic_incptr(AddressLiteral counter_addr, Register rscratch = noreg) { LP64_ONLY(atomic_incq(counter_addr, rscratch)) NOT_LP64(atomic_incl(counter_addr, rscratch)) ; }
 914   void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
 915 
 916   void lea(Register dst, Address        adr) { Assembler::lea(dst, adr); }
 917   void lea(Register dst, AddressLiteral adr);
 918   void lea(Address  dst, AddressLiteral adr, Register rscratch);
 919 
 920   void leal32(Register dst, Address src) { leal(dst, src); }
 921 
 922   // Import other testl() methods from the parent class or else
 923   // they will be hidden by the following overriding declaration.
 924   using Assembler::testl;
 925   void testl(Address dst, int32_t imm32);
 926   void testl(Register dst, int32_t imm32);
 927   void testl(Register dst, AddressLiteral src); // requires reachable address
 928   using Assembler::testq;
 929   void testq(Address dst, int32_t imm32);
 930   void testq(Register dst, int32_t imm32);
 931 
 932   void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 933   void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 934   void orptr(Register dst, int32_t src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 935   void orptr(Address dst, int32_t imm32) { LP64_ONLY(orq(dst, imm32)) NOT_LP64(orl(dst, imm32)); }
 936 
 937   void testptr(Register src, int32_t imm32) {  LP64_ONLY(testq(src, imm32)) NOT_LP64(testl(src, imm32)); }
 938   void testptr(Register src1, Address src2) { LP64_ONLY(testq(src1, src2)) NOT_LP64(testl(src1, src2)); }
 939   void testptr(Register src1, Register src2);
 940 
 941   void xorptr(Register dst, Register src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 942   void xorptr(Register dst, Address src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 943 
 944   // Calls
 945 
 946   void call(Label& L, relocInfo::relocType rtype);
 947   void call(Register entry);
 948   void call(Address addr) { Assembler::call(addr); }
 949 
 950   // NOTE: this call transfers to the effective address of entry NOT
 951   // the address contained by entry. This is because this is more natural
 952   // for jumps/calls.
 953   void call(AddressLiteral entry, Register rscratch = rax);
 954 
 955   // Emit the CompiledIC call idiom
 956   void ic_call(address entry, jint method_index = 0);
 957 
 958   void emit_static_call_stub();
 959 
 960   // Jumps
 961 
 962   // NOTE: these jumps transfer to the effective address of dst NOT
 963   // the address contained by dst. This is because this is more natural
 964   // for jumps/calls.
 965   void jump(AddressLiteral dst, Register rscratch = noreg);
 966 
 967   void jump_cc(Condition cc, AddressLiteral dst, Register rscratch = noreg);
 968 
 969   // 32bit can do a case table jump in one instruction but we no longer allow the base
 970   // to be installed in the Address class. This jump will transfer to the address
 971   // contained in the location described by entry (not the address of entry)
 972   void jump(ArrayAddress entry, Register rscratch);
 973 
 974   // Floating
 975 
 976   void push_f(XMMRegister r);
 977   void pop_f(XMMRegister r);
 978   void push_d(XMMRegister r);
 979   void pop_d(XMMRegister r);
 980 
 981   void andpd(XMMRegister dst, XMMRegister    src) { Assembler::andpd(dst, src); }
 982   void andpd(XMMRegister dst, Address        src) { Assembler::andpd(dst, src); }
 983   void andpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 984 
 985   void andps(XMMRegister dst, XMMRegister    src) { Assembler::andps(dst, src); }
 986   void andps(XMMRegister dst, Address        src) { Assembler::andps(dst, src); }
 987   void andps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 988 
 989   void comiss(XMMRegister dst, XMMRegister    src) { Assembler::comiss(dst, src); }
 990   void comiss(XMMRegister dst, Address        src) { Assembler::comiss(dst, src); }
 991   void comiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 992 
 993   void comisd(XMMRegister dst, XMMRegister    src) { Assembler::comisd(dst, src); }
 994   void comisd(XMMRegister dst, Address        src) { Assembler::comisd(dst, src); }
 995   void comisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 996 
 997 #ifndef _LP64
 998   void fadd_s(Address        src) { Assembler::fadd_s(src); }
 999   void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
1000 
1001   void fldcw(Address        src) { Assembler::fldcw(src); }
1002   void fldcw(AddressLiteral src);
1003 
1004   void fld_s(int index)          { Assembler::fld_s(index); }
1005   void fld_s(Address        src) { Assembler::fld_s(src); }
1006   void fld_s(AddressLiteral src);
1007 
1008   void fld_d(Address        src) { Assembler::fld_d(src); }
1009   void fld_d(AddressLiteral src);
1010 
1011   void fld_x(Address        src) { Assembler::fld_x(src); }
1012   void fld_x(AddressLiteral src) { Assembler::fld_x(as_Address(src)); }
1013 
1014   void fmul_s(Address        src) { Assembler::fmul_s(src); }
1015   void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
1016 #endif // !_LP64
1017 
1018   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
1019   void ldmxcsr(AddressLiteral src, Register rscratch = noreg);
1020 
1021 #ifdef _LP64
1022  private:
1023   void sha256_AVX2_one_round_compute(
1024     Register  reg_old_h,
1025     Register  reg_a,
1026     Register  reg_b,
1027     Register  reg_c,
1028     Register  reg_d,
1029     Register  reg_e,
1030     Register  reg_f,
1031     Register  reg_g,
1032     Register  reg_h,
1033     int iter);
1034   void sha256_AVX2_four_rounds_compute_first(int start);
1035   void sha256_AVX2_four_rounds_compute_last(int start);
1036   void sha256_AVX2_one_round_and_sched(
1037         XMMRegister xmm_0,     /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */
1038         XMMRegister xmm_1,     /* ymm5 */  /* full cycle is 16 iterations */
1039         XMMRegister xmm_2,     /* ymm6 */
1040         XMMRegister xmm_3,     /* ymm7 */
1041         Register    reg_a,      /* == eax on 0 iteration, then rotate 8 register right on each next iteration */
1042         Register    reg_b,      /* ebx */    /* full cycle is 8 iterations */
1043         Register    reg_c,      /* edi */
1044         Register    reg_d,      /* esi */
1045         Register    reg_e,      /* r8d */
1046         Register    reg_f,      /* r9d */
1047         Register    reg_g,      /* r10d */
1048         Register    reg_h,      /* r11d */
1049         int iter);
1050 
1051   void addm(int disp, Register r1, Register r2);
1052 
1053   void sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, Register d,
1054                                      Register e, Register f, Register g, Register h, int iteration);
1055 
1056   void sha512_AVX2_one_round_and_schedule(XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1057                                           Register a, Register b, Register c, Register d, Register e, Register f,
1058                                           Register g, Register h, int iteration);
1059 
1060   void addmq(int disp, Register r1, Register r2);
1061  public:
1062   void sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1063                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1064                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1065                    bool multi_block, XMMRegister shuf_mask);
1066   void sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1067                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1068                    Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
1069                    XMMRegister shuf_mask);
1070 #endif // _LP64
1071 
1072   void fast_md5(Register buf, Address state, Address ofs, Address limit,
1073                 bool multi_block);
1074 
1075   void fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0,
1076                  XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask,
1077                  Register buf, Register state, Register ofs, Register limit, Register rsp,
1078                  bool multi_block);
1079 
1080 #ifdef _LP64
1081   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1082                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1083                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1084                    bool multi_block, XMMRegister shuf_mask);
1085 #else
1086   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1087                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1088                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1089                    bool multi_block);
1090 #endif
1091 
1092   void fast_exp(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1093                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1094                 Register rax, Register rcx, Register rdx, Register tmp);
1095 
1096 #ifndef _LP64
1097  private:
1098   // Initialized in macroAssembler_x86_constants.cpp
1099   static address ONES;
1100   static address L_2IL0FLOATPACKET_0;
1101   static address PI4_INV;
1102   static address PI4X3;
1103   static address PI4X4;
1104 
1105  public:
1106   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1107                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1108                 Register rax, Register rcx, Register rdx, Register tmp1);
1109 
1110   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1111                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1112                 Register rax, Register rcx, Register rdx, Register tmp);
1113 
1114   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1115                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
1116                 Register rdx, Register tmp);
1117 
1118   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1119                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1120                 Register rax, Register rbx, Register rdx);
1121 
1122   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1123                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1124                 Register rax, Register rcx, Register rdx, Register tmp);
1125 
1126   void libm_sincos_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1127                         Register edx, Register ebx, Register esi, Register edi,
1128                         Register ebp, Register esp);
1129 
1130   void libm_reduce_pi04l(Register eax, Register ecx, Register edx, Register ebx,
1131                          Register esi, Register edi, Register ebp, Register esp);
1132 
1133   void libm_tancot_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1134                         Register edx, Register ebx, Register esi, Register edi,
1135                         Register ebp, Register esp);
1136 
1137   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1138                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1139                 Register rax, Register rcx, Register rdx, Register tmp);
1140 #endif // !_LP64
1141 
1142 private:
1143 
1144   // these are private because users should be doing movflt/movdbl
1145 
1146   void movss(Address     dst, XMMRegister    src) { Assembler::movss(dst, src); }
1147   void movss(XMMRegister dst, XMMRegister    src) { Assembler::movss(dst, src); }
1148   void movss(XMMRegister dst, Address        src) { Assembler::movss(dst, src); }
1149   void movss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1150 
1151   void movlpd(XMMRegister dst, Address        src) {Assembler::movlpd(dst, src); }
1152   void movlpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1153 
1154 public:
1155 
1156   void addsd(XMMRegister dst, XMMRegister    src) { Assembler::addsd(dst, src); }
1157   void addsd(XMMRegister dst, Address        src) { Assembler::addsd(dst, src); }
1158   void addsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1159 
1160   void addss(XMMRegister dst, XMMRegister    src) { Assembler::addss(dst, src); }
1161   void addss(XMMRegister dst, Address        src) { Assembler::addss(dst, src); }
1162   void addss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1163 
1164   void addpd(XMMRegister dst, XMMRegister    src) { Assembler::addpd(dst, src); }
1165   void addpd(XMMRegister dst, Address        src) { Assembler::addpd(dst, src); }
1166   void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1167 
1168   using Assembler::vbroadcastsd;
1169   void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1170 
1171   using Assembler::vbroadcastss;
1172   void vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1173 
1174   void divsd(XMMRegister dst, XMMRegister    src) { Assembler::divsd(dst, src); }
1175   void divsd(XMMRegister dst, Address        src) { Assembler::divsd(dst, src); }
1176   void divsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1177 
1178   void divss(XMMRegister dst, XMMRegister    src) { Assembler::divss(dst, src); }
1179   void divss(XMMRegister dst, Address        src) { Assembler::divss(dst, src); }
1180   void divss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1181 
1182   // Move Unaligned Double Quadword
1183   void movdqu(Address     dst, XMMRegister    src);
1184   void movdqu(XMMRegister dst, XMMRegister    src);
1185   void movdqu(XMMRegister dst, Address        src);
1186   void movdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1187 
1188   void kmovwl(Register  dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1189   void kmovwl(Address   dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1190   void kmovwl(KRegister dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1191   void kmovwl(KRegister dst, Register       src) { Assembler::kmovwl(dst, src); }
1192   void kmovwl(KRegister dst, Address        src) { Assembler::kmovwl(dst, src); }
1193   void kmovwl(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1194 
1195   void kmovql(KRegister dst, KRegister      src) { Assembler::kmovql(dst, src); }
1196   void kmovql(KRegister dst, Register       src) { Assembler::kmovql(dst, src); }
1197   void kmovql(Register  dst, KRegister      src) { Assembler::kmovql(dst, src); }
1198   void kmovql(KRegister dst, Address        src) { Assembler::kmovql(dst, src); }
1199   void kmovql(Address   dst, KRegister      src) { Assembler::kmovql(dst, src); }
1200   void kmovql(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1201 
1202   // Safe move operation, lowers down to 16bit moves for targets supporting
1203   // AVX512F feature and 64bit moves for targets supporting AVX512BW feature.
1204   void kmov(Address  dst, KRegister src);
1205   void kmov(KRegister dst, Address src);
1206   void kmov(KRegister dst, KRegister src);
1207   void kmov(Register dst, KRegister src);
1208   void kmov(KRegister dst, Register src);
1209 
1210   using Assembler::movddup;
1211   void movddup(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1212 
1213   using Assembler::vmovddup;
1214   void vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1215 
1216   // AVX Unaligned forms
1217   void vmovdqu(Address     dst, XMMRegister    src);
1218   void vmovdqu(XMMRegister dst, Address        src);
1219   void vmovdqu(XMMRegister dst, XMMRegister    src);
1220   void vmovdqu(XMMRegister dst, AddressLiteral src,                 Register rscratch = noreg);
1221   void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1222 
1223   // AVX512 Unaligned
1224   void evmovdqu(BasicType type, KRegister kmask, Address     dst, XMMRegister src, bool merge, int vector_len);
1225   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address     src, bool merge, int vector_len);
1226 
1227   void evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1228   void evmovdqub(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1229 
1230   void evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1231     if (dst->encoding() != src->encoding() || mask != k0)  {
1232       Assembler::evmovdqub(dst, mask, src, merge, vector_len);
1233     }
1234   }
1235   void evmovdqub(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1236   void evmovdqub(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1237   void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1238 
1239   void evmovdquw(Address     dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1240   void evmovdquw(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1241 
1242   void evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1243     if (dst->encoding() != src->encoding() || mask != k0) {
1244       Assembler::evmovdquw(dst, mask, src, merge, vector_len);
1245     }
1246   }
1247   void evmovdquw(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1248   void evmovdquw(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1249   void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1250 
1251   void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
1252      if (dst->encoding() != src->encoding()) {
1253        Assembler::evmovdqul(dst, src, vector_len);
1254      }
1255   }
1256   void evmovdqul(Address     dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1257   void evmovdqul(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1258 
1259   void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1260     if (dst->encoding() != src->encoding() || mask != k0)  {
1261       Assembler::evmovdqul(dst, mask, src, merge, vector_len);
1262     }
1263   }
1264   void evmovdqul(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1265   void evmovdqul(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1266   void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1267 
1268   void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
1269     if (dst->encoding() != src->encoding()) {
1270       Assembler::evmovdquq(dst, src, vector_len);
1271     }
1272   }
1273   void evmovdquq(XMMRegister dst, Address        src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1274   void evmovdquq(Address     dst, XMMRegister    src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1275   void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1276 
1277   void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1278     if (dst->encoding() != src->encoding() || mask != k0) {
1279       Assembler::evmovdquq(dst, mask, src, merge, vector_len);
1280     }
1281   }
1282   void evmovdquq(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1283   void evmovdquq(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1284   void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1285 
1286   // Move Aligned Double Quadword
1287   void movdqa(XMMRegister dst, XMMRegister    src) { Assembler::movdqa(dst, src); }
1288   void movdqa(XMMRegister dst, Address        src) { Assembler::movdqa(dst, src); }
1289   void movdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1290 
1291   void movsd(Address     dst, XMMRegister    src) { Assembler::movsd(dst, src); }
1292   void movsd(XMMRegister dst, XMMRegister    src) { Assembler::movsd(dst, src); }
1293   void movsd(XMMRegister dst, Address        src) { Assembler::movsd(dst, src); }
1294   void movsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1295 
1296   void mulpd(XMMRegister dst, XMMRegister    src) { Assembler::mulpd(dst, src); }
1297   void mulpd(XMMRegister dst, Address        src) { Assembler::mulpd(dst, src); }
1298   void mulpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1299 
1300   void mulsd(XMMRegister dst, XMMRegister    src) { Assembler::mulsd(dst, src); }
1301   void mulsd(XMMRegister dst, Address        src) { Assembler::mulsd(dst, src); }
1302   void mulsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1303 
1304   void mulss(XMMRegister dst, XMMRegister    src) { Assembler::mulss(dst, src); }
1305   void mulss(XMMRegister dst, Address        src) { Assembler::mulss(dst, src); }
1306   void mulss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1307 
1308   // Carry-Less Multiplication Quadword
1309   void pclmulldq(XMMRegister dst, XMMRegister src) {
1310     // 0x00 - multiply lower 64 bits [0:63]
1311     Assembler::pclmulqdq(dst, src, 0x00);
1312   }
1313   void pclmulhdq(XMMRegister dst, XMMRegister src) {
1314     // 0x11 - multiply upper 64 bits [64:127]
1315     Assembler::pclmulqdq(dst, src, 0x11);
1316   }
1317 
1318   void pcmpeqb(XMMRegister dst, XMMRegister src);
1319   void pcmpeqw(XMMRegister dst, XMMRegister src);
1320 
1321   void pcmpestri(XMMRegister dst, Address src, int imm8);
1322   void pcmpestri(XMMRegister dst, XMMRegister src, int imm8);
1323 
1324   void pmovzxbw(XMMRegister dst, XMMRegister src);
1325   void pmovzxbw(XMMRegister dst, Address src);
1326 
1327   void pmovmskb(Register dst, XMMRegister src);
1328 
1329   void ptest(XMMRegister dst, XMMRegister src);
1330 
1331   void roundsd(XMMRegister dst, XMMRegister    src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1332   void roundsd(XMMRegister dst, Address        src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1333   void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch = noreg);
1334 
1335   void sqrtss(XMMRegister dst, XMMRegister     src) { Assembler::sqrtss(dst, src); }
1336   void sqrtss(XMMRegister dst, Address         src) { Assembler::sqrtss(dst, src); }
1337   void sqrtss(XMMRegister dst, AddressLiteral  src, Register rscratch = noreg);
1338 
1339   void subsd(XMMRegister dst, XMMRegister    src) { Assembler::subsd(dst, src); }
1340   void subsd(XMMRegister dst, Address        src) { Assembler::subsd(dst, src); }
1341   void subsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1342 
1343   void subss(XMMRegister dst, XMMRegister    src) { Assembler::subss(dst, src); }
1344   void subss(XMMRegister dst, Address        src) { Assembler::subss(dst, src); }
1345   void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1346 
1347   void ucomiss(XMMRegister dst, XMMRegister    src) { Assembler::ucomiss(dst, src); }
1348   void ucomiss(XMMRegister dst, Address        src) { Assembler::ucomiss(dst, src); }
1349   void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1350 
1351   void ucomisd(XMMRegister dst, XMMRegister    src) { Assembler::ucomisd(dst, src); }
1352   void ucomisd(XMMRegister dst, Address        src) { Assembler::ucomisd(dst, src); }
1353   void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1354 
1355   // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
1356   void xorpd(XMMRegister dst, XMMRegister    src);
1357   void xorpd(XMMRegister dst, Address        src) { Assembler::xorpd(dst, src); }
1358   void xorpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1359 
1360   // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
1361   void xorps(XMMRegister dst, XMMRegister    src);
1362   void xorps(XMMRegister dst, Address        src) { Assembler::xorps(dst, src); }
1363   void xorps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1364 
1365   // Shuffle Bytes
1366   void pshufb(XMMRegister dst, XMMRegister    src) { Assembler::pshufb(dst, src); }
1367   void pshufb(XMMRegister dst, Address        src) { Assembler::pshufb(dst, src); }
1368   void pshufb(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1369   // AVX 3-operands instructions
1370 
1371   void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vaddsd(dst, nds, src); }
1372   void vaddsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vaddsd(dst, nds, src); }
1373   void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1374 
1375   void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vaddss(dst, nds, src); }
1376   void vaddss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vaddss(dst, nds, src); }
1377   void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1378 
1379   void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1380   void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1381 
1382   void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len);
1383   void vpaddb(XMMRegister dst, XMMRegister nds, Address        src, int vector_len);
1384   void vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1385 
1386   void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1387   void vpaddw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1388 
1389   void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1390   void vpaddd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1391   void vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1392 
1393   void vpand(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1394   void vpand(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1395   void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1396 
1397   using Assembler::vpbroadcastd;
1398   void vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1399 
1400   using Assembler::vpbroadcastq;
1401   void vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1402 
1403   void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1404 
1405   void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1406   void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1407 
1408   // Vector compares
1409   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1410     Assembler::evpcmpd(kdst, mask, nds, src, comparison, is_signed, vector_len);
1411   }
1412   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1413 
1414   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1415     Assembler::evpcmpq(kdst, mask, nds, src, comparison, is_signed, vector_len);
1416   }
1417   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1418 
1419   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1420     Assembler::evpcmpb(kdst, mask, nds, src, comparison, is_signed, vector_len);
1421   }
1422   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1423 
1424   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1425     Assembler::evpcmpw(kdst, mask, nds, src, comparison, is_signed, vector_len);
1426   }
1427   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1428 
1429   void evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len);
1430 
1431   // Emit comparison instruction for the specified comparison predicate.
1432   void vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len);
1433   void vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len);
1434 
1435   void vpmovzxbw(XMMRegister dst, Address     src, int vector_len);
1436   void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
1437 
1438   void vpmovmskb(Register dst, XMMRegister src, int vector_len = Assembler::AVX_256bit);
1439 
1440   void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1441   void vpmullw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1442 
1443   void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1444   void vpmulld(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1445   void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1446 
1447   void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1448   void vpsubb(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1449 
1450   void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1451   void vpsubw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1452 
1453   void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1454   void vpsraw(XMMRegister dst, XMMRegister nds, int         shift, int vector_len);
1455 
1456   void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1457   void evpsraq(XMMRegister dst, XMMRegister nds, int         shift, int vector_len);
1458 
1459   void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1460     if (!is_varshift) {
1461       Assembler::evpsllw(dst, mask, nds, src, merge, vector_len);
1462     } else {
1463       Assembler::evpsllvw(dst, mask, nds, src, merge, vector_len);
1464     }
1465   }
1466   void evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1467     if (!is_varshift) {
1468       Assembler::evpslld(dst, mask, nds, src, merge, vector_len);
1469     } else {
1470       Assembler::evpsllvd(dst, mask, nds, src, merge, vector_len);
1471     }
1472   }
1473   void evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1474     if (!is_varshift) {
1475       Assembler::evpsllq(dst, mask, nds, src, merge, vector_len);
1476     } else {
1477       Assembler::evpsllvq(dst, mask, nds, src, merge, vector_len);
1478     }
1479   }
1480   void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1481     if (!is_varshift) {
1482       Assembler::evpsrlw(dst, mask, nds, src, merge, vector_len);
1483     } else {
1484       Assembler::evpsrlvw(dst, mask, nds, src, merge, vector_len);
1485     }
1486   }
1487   void evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1488     if (!is_varshift) {
1489       Assembler::evpsrld(dst, mask, nds, src, merge, vector_len);
1490     } else {
1491       Assembler::evpsrlvd(dst, mask, nds, src, merge, vector_len);
1492     }
1493   }
1494   void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1495     if (!is_varshift) {
1496       Assembler::evpsrlq(dst, mask, nds, src, merge, vector_len);
1497     } else {
1498       Assembler::evpsrlvq(dst, mask, nds, src, merge, vector_len);
1499     }
1500   }
1501   void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1502     if (!is_varshift) {
1503       Assembler::evpsraw(dst, mask, nds, src, merge, vector_len);
1504     } else {
1505       Assembler::evpsravw(dst, mask, nds, src, merge, vector_len);
1506     }
1507   }
1508   void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1509     if (!is_varshift) {
1510       Assembler::evpsrad(dst, mask, nds, src, merge, vector_len);
1511     } else {
1512       Assembler::evpsravd(dst, mask, nds, src, merge, vector_len);
1513     }
1514   }
1515   void evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1516     if (!is_varshift) {
1517       Assembler::evpsraq(dst, mask, nds, src, merge, vector_len);
1518     } else {
1519       Assembler::evpsravq(dst, mask, nds, src, merge, vector_len);
1520     }
1521   }
1522 
1523   void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1524   void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1525   void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1526   void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1527 
1528   void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1529   void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1530 
1531   void vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1532   void vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1533 
1534   void vptest(XMMRegister dst, XMMRegister src);
1535   void vptest(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vptest(dst, src, vector_len); }
1536 
1537   void punpcklbw(XMMRegister dst, XMMRegister src);
1538   void punpcklbw(XMMRegister dst, Address src) { Assembler::punpcklbw(dst, src); }
1539 
1540   void pshufd(XMMRegister dst, Address src, int mode);
1541   void pshufd(XMMRegister dst, XMMRegister src, int mode) { Assembler::pshufd(dst, src, mode); }
1542 
1543   void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1544   void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
1545 
1546   void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1547   void vandpd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1548   void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1549 
1550   void vandps(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1551   void vandps(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1552   void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1553 
1554   void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1555 
1556   void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vdivsd(dst, nds, src); }
1557   void vdivsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vdivsd(dst, nds, src); }
1558   void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1559 
1560   void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vdivss(dst, nds, src); }
1561   void vdivss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vdivss(dst, nds, src); }
1562   void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1563 
1564   void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vmulsd(dst, nds, src); }
1565   void vmulsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vmulsd(dst, nds, src); }
1566   void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1567 
1568   void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vmulss(dst, nds, src); }
1569   void vmulss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vmulss(dst, nds, src); }
1570   void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1571 
1572   void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vsubsd(dst, nds, src); }
1573   void vsubsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vsubsd(dst, nds, src); }
1574   void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1575 
1576   void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vsubss(dst, nds, src); }
1577   void vsubss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vsubss(dst, nds, src); }
1578   void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1579 
1580   void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1581   void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1582 
1583   // AVX Vector instructions
1584 
1585   void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1586   void vxorpd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1587   void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1588 
1589   void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1590   void vxorps(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1591   void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1592 
1593   void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1594     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1595       Assembler::vpxor(dst, nds, src, vector_len);
1596     else
1597       Assembler::vxorpd(dst, nds, src, vector_len);
1598   }
1599   void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
1600     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1601       Assembler::vpxor(dst, nds, src, vector_len);
1602     else
1603       Assembler::vxorpd(dst, nds, src, vector_len);
1604   }
1605   void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1606 
1607   // Simple version for AVX2 256bit vectors
1608   void vpxor(XMMRegister dst, XMMRegister src) {
1609     assert(UseAVX >= 2, "Should be at least AVX2");
1610     Assembler::vpxor(dst, dst, src, AVX_256bit);
1611   }
1612   void vpxor(XMMRegister dst, Address src) {
1613     assert(UseAVX >= 2, "Should be at least AVX2");
1614     Assembler::vpxor(dst, dst, src, AVX_256bit);
1615   }
1616 
1617   void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
1618   void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1619 
1620   void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
1621     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1622       Assembler::vinserti32x4(dst, nds, src, imm8);
1623     } else if (UseAVX > 1) {
1624       // vinserti128 is available only in AVX2
1625       Assembler::vinserti128(dst, nds, src, imm8);
1626     } else {
1627       Assembler::vinsertf128(dst, nds, src, imm8);
1628     }
1629   }
1630 
1631   void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
1632     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1633       Assembler::vinserti32x4(dst, nds, src, imm8);
1634     } else if (UseAVX > 1) {
1635       // vinserti128 is available only in AVX2
1636       Assembler::vinserti128(dst, nds, src, imm8);
1637     } else {
1638       Assembler::vinsertf128(dst, nds, src, imm8);
1639     }
1640   }
1641 
1642   void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
1643     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1644       Assembler::vextracti32x4(dst, src, imm8);
1645     } else if (UseAVX > 1) {
1646       // vextracti128 is available only in AVX2
1647       Assembler::vextracti128(dst, src, imm8);
1648     } else {
1649       Assembler::vextractf128(dst, src, imm8);
1650     }
1651   }
1652 
1653   void vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
1654     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1655       Assembler::vextracti32x4(dst, src, imm8);
1656     } else if (UseAVX > 1) {
1657       // vextracti128 is available only in AVX2
1658       Assembler::vextracti128(dst, src, imm8);
1659     } else {
1660       Assembler::vextractf128(dst, src, imm8);
1661     }
1662   }
1663 
1664   // 128bit copy to/from high 128 bits of 256bit (YMM) vector registers
1665   void vinserti128_high(XMMRegister dst, XMMRegister src) {
1666     vinserti128(dst, dst, src, 1);
1667   }
1668   void vinserti128_high(XMMRegister dst, Address src) {
1669     vinserti128(dst, dst, src, 1);
1670   }
1671   void vextracti128_high(XMMRegister dst, XMMRegister src) {
1672     vextracti128(dst, src, 1);
1673   }
1674   void vextracti128_high(Address dst, XMMRegister src) {
1675     vextracti128(dst, src, 1);
1676   }
1677 
1678   void vinsertf128_high(XMMRegister dst, XMMRegister src) {
1679     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1680       Assembler::vinsertf32x4(dst, dst, src, 1);
1681     } else {
1682       Assembler::vinsertf128(dst, dst, src, 1);
1683     }
1684   }
1685 
1686   void vinsertf128_high(XMMRegister dst, Address src) {
1687     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1688       Assembler::vinsertf32x4(dst, dst, src, 1);
1689     } else {
1690       Assembler::vinsertf128(dst, dst, src, 1);
1691     }
1692   }
1693 
1694   void vextractf128_high(XMMRegister dst, XMMRegister src) {
1695     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1696       Assembler::vextractf32x4(dst, src, 1);
1697     } else {
1698       Assembler::vextractf128(dst, src, 1);
1699     }
1700   }
1701 
1702   void vextractf128_high(Address dst, XMMRegister src) {
1703     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1704       Assembler::vextractf32x4(dst, src, 1);
1705     } else {
1706       Assembler::vextractf128(dst, src, 1);
1707     }
1708   }
1709 
1710   // 256bit copy to/from high 256 bits of 512bit (ZMM) vector registers
1711   void vinserti64x4_high(XMMRegister dst, XMMRegister src) {
1712     Assembler::vinserti64x4(dst, dst, src, 1);
1713   }
1714   void vinsertf64x4_high(XMMRegister dst, XMMRegister src) {
1715     Assembler::vinsertf64x4(dst, dst, src, 1);
1716   }
1717   void vextracti64x4_high(XMMRegister dst, XMMRegister src) {
1718     Assembler::vextracti64x4(dst, src, 1);
1719   }
1720   void vextractf64x4_high(XMMRegister dst, XMMRegister src) {
1721     Assembler::vextractf64x4(dst, src, 1);
1722   }
1723   void vextractf64x4_high(Address dst, XMMRegister src) {
1724     Assembler::vextractf64x4(dst, src, 1);
1725   }
1726   void vinsertf64x4_high(XMMRegister dst, Address src) {
1727     Assembler::vinsertf64x4(dst, dst, src, 1);
1728   }
1729 
1730   // 128bit copy to/from low 128 bits of 256bit (YMM) vector registers
1731   void vinserti128_low(XMMRegister dst, XMMRegister src) {
1732     vinserti128(dst, dst, src, 0);
1733   }
1734   void vinserti128_low(XMMRegister dst, Address src) {
1735     vinserti128(dst, dst, src, 0);
1736   }
1737   void vextracti128_low(XMMRegister dst, XMMRegister src) {
1738     vextracti128(dst, src, 0);
1739   }
1740   void vextracti128_low(Address dst, XMMRegister src) {
1741     vextracti128(dst, src, 0);
1742   }
1743 
1744   void vinsertf128_low(XMMRegister dst, XMMRegister src) {
1745     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1746       Assembler::vinsertf32x4(dst, dst, src, 0);
1747     } else {
1748       Assembler::vinsertf128(dst, dst, src, 0);
1749     }
1750   }
1751 
1752   void vinsertf128_low(XMMRegister dst, Address src) {
1753     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1754       Assembler::vinsertf32x4(dst, dst, src, 0);
1755     } else {
1756       Assembler::vinsertf128(dst, dst, src, 0);
1757     }
1758   }
1759 
1760   void vextractf128_low(XMMRegister dst, XMMRegister src) {
1761     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1762       Assembler::vextractf32x4(dst, src, 0);
1763     } else {
1764       Assembler::vextractf128(dst, src, 0);
1765     }
1766   }
1767 
1768   void vextractf128_low(Address dst, XMMRegister src) {
1769     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1770       Assembler::vextractf32x4(dst, src, 0);
1771     } else {
1772       Assembler::vextractf128(dst, src, 0);
1773     }
1774   }
1775 
1776   // 256bit copy to/from low 256 bits of 512bit (ZMM) vector registers
1777   void vinserti64x4_low(XMMRegister dst, XMMRegister src) {
1778     Assembler::vinserti64x4(dst, dst, src, 0);
1779   }
1780   void vinsertf64x4_low(XMMRegister dst, XMMRegister src) {
1781     Assembler::vinsertf64x4(dst, dst, src, 0);
1782   }
1783   void vextracti64x4_low(XMMRegister dst, XMMRegister src) {
1784     Assembler::vextracti64x4(dst, src, 0);
1785   }
1786   void vextractf64x4_low(XMMRegister dst, XMMRegister src) {
1787     Assembler::vextractf64x4(dst, src, 0);
1788   }
1789   void vextractf64x4_low(Address dst, XMMRegister src) {
1790     Assembler::vextractf64x4(dst, src, 0);
1791   }
1792   void vinsertf64x4_low(XMMRegister dst, Address src) {
1793     Assembler::vinsertf64x4(dst, dst, src, 0);
1794   }
1795 
1796   // Carry-Less Multiplication Quadword
1797   void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1798     // 0x00 - multiply lower 64 bits [0:63]
1799     Assembler::vpclmulqdq(dst, nds, src, 0x00);
1800   }
1801   void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1802     // 0x11 - multiply upper 64 bits [64:127]
1803     Assembler::vpclmulqdq(dst, nds, src, 0x11);
1804   }
1805   void vpclmullqhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1806     // 0x10 - multiply nds[0:63] and src[64:127]
1807     Assembler::vpclmulqdq(dst, nds, src, 0x10);
1808   }
1809   void vpclmulhqlqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1810     //0x01 - multiply nds[64:127] and src[0:63]
1811     Assembler::vpclmulqdq(dst, nds, src, 0x01);
1812   }
1813 
1814   void evpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1815     // 0x00 - multiply lower 64 bits [0:63]
1816     Assembler::evpclmulqdq(dst, nds, src, 0x00, vector_len);
1817   }
1818   void evpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1819     // 0x11 - multiply upper 64 bits [64:127]
1820     Assembler::evpclmulqdq(dst, nds, src, 0x11, vector_len);
1821   }
1822 
1823   // AVX-512 mask operations.
1824   void kand(BasicType etype, KRegister dst, KRegister src1, KRegister src2);
1825   void kor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1826   void knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp = knoreg, Register rtmp = noreg);
1827   void kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1828   void kortest(uint masklen, KRegister src1, KRegister src2);
1829   void ktest(uint masklen, KRegister src1, KRegister src2);
1830 
1831   void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1832   void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1833 
1834   void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1835   void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1836 
1837   void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1838   void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1839 
1840   void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1841   void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1842 
1843   void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1844   void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1845   void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1846   void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1847 
1848   using Assembler::evpandq;
1849   void evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1850 
1851   using Assembler::evpaddq;
1852   void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1853 
1854   using Assembler::evporq;
1855   void evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1856 
1857   using Assembler::vpternlogq;
1858   void vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch = noreg);
1859 
1860   void cmov32( Condition cc, Register dst, Address  src);
1861   void cmov32( Condition cc, Register dst, Register src);
1862 
1863   void cmov(   Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
1864 
1865   void cmovptr(Condition cc, Register dst, Address  src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1866   void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1867 
1868   void movoop(Register dst, jobject obj);
1869   void movoop(Address  dst, jobject obj, Register rscratch);
1870 
1871   void mov_metadata(Register dst, Metadata* obj);
1872   void mov_metadata(Address  dst, Metadata* obj, Register rscratch);
1873 
1874   void movptr(Register     dst, Register       src);
1875   void movptr(Register     dst, Address        src);
1876   void movptr(Register     dst, AddressLiteral src);
1877   void movptr(Register     dst, ArrayAddress   src);
1878   void movptr(Register     dst, intptr_t       src);
1879   void movptr(Address      dst, Register       src);
1880   void movptr(Address      dst, int32_t        imm);
1881   void movptr(Address      dst, intptr_t       src, Register rscratch);
1882   void movptr(ArrayAddress dst, Register       src, Register rscratch);
1883 
1884   void movptr(Register dst, RegisterOrConstant src) {
1885     if (src.is_constant()) movptr(dst, src.as_constant());
1886     else                   movptr(dst, src.as_register());
1887   }
1888 
1889 
1890   // to avoid hiding movl
1891   void mov32(Register       dst, AddressLiteral src);
1892   void mov32(AddressLiteral dst, Register        src, Register rscratch = noreg);
1893 
1894   // Import other mov() methods from the parent class or else
1895   // they will be hidden by the following overriding declaration.
1896   using Assembler::movdl;
1897   void movdl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1898 
1899   using Assembler::movq;
1900   void movq(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1901 
1902   // Can push value or effective address
1903   void pushptr(AddressLiteral src, Register rscratch);
1904 
1905   void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
1906   void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
1907 
1908   void pushoop(jobject obj, Register rscratch);
1909   void pushklass(Metadata* obj, Register rscratch);
1910 
1911   // sign extend as need a l to ptr sized element
1912   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
1913   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
1914 
1915 
1916  public:
1917   // Inline type specific methods
1918   #include "asm/macroAssembler_common.hpp"
1919 
1920   int store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter = true);
1921   bool move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]);
1922   bool unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
1923                             VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
1924                             RegState reg_state[]);
1925   bool pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
1926                           VMRegPair* from, int from_count, int& from_index, VMReg to,
1927                           RegState reg_state[], Register val_array);
1928   int extend_stack_for_inline_args(int args_on_stack);
1929   void remove_frame(int initial_framesize, bool needs_stack_repair);
1930   VMReg spill_reg_for(VMReg reg);
1931 
1932   // clear memory of size 'cnt' qwords, starting at 'base';
1933   // if 'is_large' is set, do not try to produce short loop
1934   void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only, KRegister mask=knoreg);
1935 
1936   // clear memory initialization sequence for constant size;
1937   void clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1938 
1939   // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
1940   void xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1941 
1942   // Fill primitive arrays
1943   void generate_fill(BasicType t, bool aligned,
1944                      Register to, Register value, Register count,
1945                      Register rtmp, XMMRegister xtmp);
1946 
1947   void encode_iso_array(Register src, Register dst, Register len,
1948                         XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1949                         XMMRegister tmp4, Register tmp5, Register result, bool ascii);
1950 
1951 #ifdef _LP64
1952   void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
1953   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1954                              Register y, Register y_idx, Register z,
1955                              Register carry, Register product,
1956                              Register idx, Register kdx);
1957   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
1958                               Register yz_idx, Register idx,
1959                               Register carry, Register product, int offset);
1960   void multiply_128_x_128_bmi2_loop(Register y, Register z,
1961                                     Register carry, Register carry2,
1962                                     Register idx, Register jdx,
1963                                     Register yz_idx1, Register yz_idx2,
1964                                     Register tmp, Register tmp3, Register tmp4);
1965   void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
1966                                Register yz_idx, Register idx, Register jdx,
1967                                Register carry, Register product,
1968                                Register carry2);
1969   void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
1970                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
1971   void square_rshift(Register x, Register len, Register z, Register tmp1, Register tmp3,
1972                      Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1973   void multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry,
1974                             Register tmp2);
1975   void multiply_add_64(Register sum, Register op1, Register op2, Register carry,
1976                        Register rdxReg, Register raxReg);
1977   void add_one_64(Register z, Register zlen, Register carry, Register tmp1);
1978   void lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1979                        Register tmp3, Register tmp4);
1980   void square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1981                      Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1982 
1983   void mul_add_128_x_32_loop(Register out, Register in, Register offset, Register len, Register tmp1,
1984                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1985                Register raxReg);
1986   void mul_add(Register out, Register in, Register offset, Register len, Register k, Register tmp1,
1987                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1988                Register raxReg);
1989   void vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
1990                            Register result, Register tmp1, Register tmp2,
1991                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3);
1992 #endif
1993 
1994   // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
1995   void update_byte_crc32(Register crc, Register val, Register table);
1996   void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
1997 
1998 
1999 #ifdef _LP64
2000   void kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2);
2001   void kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register key, Register pos,
2002                                 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
2003                                 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup);
2004 #endif // _LP64
2005 
2006   // CRC32C code for java.util.zip.CRC32C::updateBytes() intrinsic
2007   // Note on a naming convention:
2008   // Prefix w = register only used on a Westmere+ architecture
2009   // Prefix n = register only used on a Nehalem architecture
2010 #ifdef _LP64
2011   void crc32c_ipl_alg4(Register in_out, uint32_t n,
2012                        Register tmp1, Register tmp2, Register tmp3);
2013 #else
2014   void crc32c_ipl_alg4(Register in_out, uint32_t n,
2015                        Register tmp1, Register tmp2, Register tmp3,
2016                        XMMRegister xtmp1, XMMRegister xtmp2);
2017 #endif
2018   void crc32c_pclmulqdq(XMMRegister w_xtmp1,
2019                         Register in_out,
2020                         uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
2021                         XMMRegister w_xtmp2,
2022                         Register tmp1,
2023                         Register n_tmp2, Register n_tmp3);
2024   void crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
2025                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2026                        Register tmp1, Register tmp2,
2027                        Register n_tmp3);
2028   void crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
2029                          Register in_out1, Register in_out2, Register in_out3,
2030                          Register tmp1, Register tmp2, Register tmp3,
2031                          XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2032                          Register tmp4, Register tmp5,
2033                          Register n_tmp6);
2034   void crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
2035                             Register tmp1, Register tmp2, Register tmp3,
2036                             Register tmp4, Register tmp5, Register tmp6,
2037                             XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2038                             bool is_pclmulqdq_supported);
2039   // Fold 128-bit data chunk
2040   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
2041   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
2042 #ifdef _LP64
2043   // Fold 512-bit data chunk
2044   void fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, Register pos, int offset);
2045 #endif // _LP64
2046   // Fold 8-bit data
2047   void fold_8bit_crc32(Register crc, Register table, Register tmp);
2048   void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
2049 
2050   // Compress char[] array to byte[].
2051   void char_array_compress(Register src, Register dst, Register len,
2052                            XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
2053                            XMMRegister tmp4, Register tmp5, Register result,
2054                            KRegister mask1 = knoreg, KRegister mask2 = knoreg);
2055 
2056   // Inflate byte[] array to char[].
2057   void byte_array_inflate(Register src, Register dst, Register len,
2058                           XMMRegister tmp1, Register tmp2, KRegister mask = knoreg);
2059 
2060   void fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
2061                    Register length, Register temp, int vec_enc);
2062 
2063   void fill64_masked(uint shift, Register dst, int disp,
2064                          XMMRegister xmm, KRegister mask, Register length,
2065                          Register temp, bool use64byteVector = false);
2066 
2067   void fill32_masked(uint shift, Register dst, int disp,
2068                          XMMRegister xmm, KRegister mask, Register length,
2069                          Register temp);
2070 
2071   void fill32(Address dst, XMMRegister xmm);
2072 
2073   void fill32(Register dst, int disp, XMMRegister xmm);
2074 
2075   void fill64(Address dst, XMMRegister xmm, bool use64byteVector = false);
2076 
2077   void fill64(Register dst, int dis, XMMRegister xmm, bool use64byteVector = false);
2078 
2079 #ifdef _LP64
2080   void convert_f2i(Register dst, XMMRegister src);
2081   void convert_d2i(Register dst, XMMRegister src);
2082   void convert_f2l(Register dst, XMMRegister src);
2083   void convert_d2l(Register dst, XMMRegister src);
2084   void round_double(Register dst, XMMRegister src, Register rtmp, Register rcx);
2085   void round_float(Register dst, XMMRegister src, Register rtmp, Register rcx);
2086 
2087   void cache_wb(Address line);
2088   void cache_wbsync(bool is_pre);
2089 
2090 #ifdef COMPILER2_OR_JVMCI
2091   void generate_fill_avx3(BasicType type, Register to, Register value,
2092                           Register count, Register rtmp, XMMRegister xtmp);
2093 #endif // COMPILER2_OR_JVMCI
2094 #endif // _LP64
2095 
2096   void vallones(XMMRegister dst, int vector_len);
2097 
2098   void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
2099 
2100   void fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow);
2101   void fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow);
2102 };
2103 
2104 /**
2105  * class SkipIfEqual:
2106  *
2107  * Instantiating this class will result in assembly code being output that will
2108  * jump around any code emitted between the creation of the instance and it's
2109  * automatic destruction at the end of a scope block, depending on the value of
2110  * the flag passed to the constructor, which will be checked at run-time.
2111  */
2112 class SkipIfEqual {
2113  private:
2114   MacroAssembler* _masm;
2115   Label _label;
2116 
2117  public:
2118    SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value, Register rscratch);
2119    ~SkipIfEqual();
2120 };
2121 
2122 #endif // CPU_X86_MACROASSEMBLER_X86_HPP