1 /*
   2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef CPU_X86_MACROASSEMBLER_X86_HPP
  26 #define CPU_X86_MACROASSEMBLER_X86_HPP
  27 
  28 #include "asm/assembler.hpp"
  29 #include "asm/register.hpp"
  30 #include "code/vmreg.inline.hpp"
  31 #include "compiler/oopMap.hpp"
  32 #include "utilities/macros.hpp"
  33 #include "runtime/rtmLocking.hpp"
  34 #include "runtime/signature.hpp"
  35 #include "runtime/vm_version.hpp"
  36 
  37 class ciInlineKlass;
  38 
  39 // MacroAssembler extends Assembler by frequently used macros.
  40 //
  41 // Instructions for which a 'better' code sequence exists depending
  42 // on arguments should also go in here.
  43 
  44 class MacroAssembler: public Assembler {
  45   friend class LIR_Assembler;
  46   friend class Runtime1;      // as_Address()
  47 
  48  public:
  49   // Support for VM calls
  50   //
  51   // This is the base routine called by the different versions of call_VM_leaf. The interpreter
  52   // may customize this version by overriding it for its purposes (e.g., to save/restore
  53   // additional registers when doing a VM call).
  54 
  55   virtual void call_VM_leaf_base(
  56     address entry_point,               // the entry point
  57     int     number_of_arguments        // the number of arguments to pop after the call
  58   );
  59 
  60  protected:
  61   // This is the base routine called by the different versions of call_VM. The interpreter
  62   // may customize this version by overriding it for its purposes (e.g., to save/restore
  63   // additional registers when doing a VM call).
  64   //
  65   // If no java_thread register is specified (noreg) than rdi will be used instead. call_VM_base
  66   // returns the register which contains the thread upon return. If a thread register has been
  67   // specified, the return value will correspond to that register. If no last_java_sp is specified
  68   // (noreg) than rsp will be used instead.
  69   virtual void call_VM_base(           // returns the register containing the thread upon return
  70     Register oop_result,               // where an oop-result ends up if any; use noreg otherwise
  71     Register java_thread,              // the thread if computed before     ; use noreg otherwise
  72     Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise
  73     address  entry_point,              // the entry point
  74     int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call
  75     bool     check_exceptions          // whether to check for pending exceptions after return
  76   );
  77 
  78   void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
  79 
  80   // helpers for FPU flag access
  81   // tmp is a temporary register, if none is available use noreg
  82   void save_rax   (Register tmp);
  83   void restore_rax(Register tmp);
  84 
  85  public:
  86   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
  87 
  88  // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
  89  // The implementation is only non-empty for the InterpreterMacroAssembler,
  90  // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
  91  virtual void check_and_handle_popframe(Register java_thread);
  92  virtual void check_and_handle_earlyret(Register java_thread);
  93 
  94   Address as_Address(AddressLiteral adr);
  95   Address as_Address(ArrayAddress adr, Register rscratch);
  96 
  97   // Support for NULL-checks
  98   //
  99   // Generates code that causes a NULL OS exception if the content of reg is NULL.
 100   // If the accessed location is M[reg + offset] and the offset is known, provide the
 101   // offset. No explicit code generation is needed if the offset is within a certain
 102   // range (0 <= offset <= page_size).
 103 
 104   void null_check(Register reg, int offset = -1);
 105   static bool needs_explicit_null_check(intptr_t offset);
 106   static bool uses_implicit_null_check(void* address);
 107 
 108   // markWord tests, kills markWord reg
 109   void test_markword_is_inline_type(Register markword, Label& is_inline_type);
 110 
 111   // inlineKlass queries, kills temp_reg
 112   void test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type);
 113   void test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type);
 114   void test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type);
 115 
 116   // Get the default value oop for the given InlineKlass
 117   void get_default_value_oop(Register inline_klass, Register temp_reg, Register obj);
 118   // The empty value oop, for the given InlineKlass ("empty" as in no instance fields)
 119   // get_default_value_oop with extra assertion for empty inline klass
 120   void get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj);
 121 
 122   void test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free);
 123   void test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free);
 124   void test_field_is_inlined(Register flags, Register temp_reg, Label& is_inlined);
 125 
 126   // Check oops for special arrays, i.e. flattened and/or null-free
 127   void test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label);
 128   void test_flattened_array_oop(Register oop, Register temp_reg, Label&is_flattened_array);
 129   void test_non_flattened_array_oop(Register oop, Register temp_reg, Label&is_non_flattened_array);
 130   void test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array);
 131   void test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array);
 132 
 133   // Check array klass layout helper for flatten or null-free arrays...
 134   void test_flattened_array_layout(Register lh, Label& is_flattened_array);
 135   void test_non_flattened_array_layout(Register lh, Label& is_non_flattened_array);
 136   void test_null_free_array_layout(Register lh, Label& is_null_free_array);
 137   void test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array);
 138 
 139   // Required platform-specific helpers for Label::patch_instructions.
 140   // They _shadow_ the declarations in AbstractAssembler, which are undefined.
 141   void pd_patch_instruction(address branch, address target, const char* file, int line) {
 142     unsigned char op = branch[0];
 143     assert(op == 0xE8 /* call */ ||
 144         op == 0xE9 /* jmp */ ||
 145         op == 0xEB /* short jmp */ ||
 146         (op & 0xF0) == 0x70 /* short jcc */ ||
 147         op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
 148         op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
 149         "Invalid opcode at patch point");
 150 
 151     if (op == 0xEB || (op & 0xF0) == 0x70) {
 152       // short offset operators (jmp and jcc)
 153       char* disp = (char*) &branch[1];
 154       int imm8 = target - (address) &disp[1];
 155       guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset at %s:%d",
 156                 file == NULL ? "<NULL>" : file, line);
 157       *disp = imm8;
 158     } else {
 159       int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1];
 160       int imm32 = target - (address) &disp[1];
 161       *disp = imm32;
 162     }
 163   }
 164 
 165   // The following 4 methods return the offset of the appropriate move instruction
 166 
 167   // Support for fast byte/short loading with zero extension (depending on particular CPU)
 168   int load_unsigned_byte(Register dst, Address src);
 169   int load_unsigned_short(Register dst, Address src);
 170 
 171   // Support for fast byte/short loading with sign extension (depending on particular CPU)
 172   int load_signed_byte(Register dst, Address src);
 173   int load_signed_short(Register dst, Address src);
 174 
 175   // Support for sign-extension (hi:lo = extend_sign(lo))
 176   void extend_sign(Register hi, Register lo);
 177 
 178   // Load and store values by size and signed-ness
 179   void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
 180   void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
 181 
 182   // Support for inc/dec with optimal instruction selection depending on value
 183 
 184   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
 185   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
 186 
 187   void decrementl(Address dst, int value = 1);
 188   void decrementl(Register reg, int value = 1);
 189 
 190   void decrementq(Register reg, int value = 1);
 191   void decrementq(Address dst, int value = 1);
 192 
 193   void incrementl(Address dst, int value = 1);
 194   void incrementl(Register reg, int value = 1);
 195 
 196   void incrementq(Register reg, int value = 1);
 197   void incrementq(Address dst, int value = 1);
 198 
 199   // Support optimal SSE move instructions.
 200   void movflt(XMMRegister dst, XMMRegister src) {
 201     if (dst-> encoding() == src->encoding()) return;
 202     if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
 203     else                       { movss (dst, src); return; }
 204   }
 205   void movflt(XMMRegister dst, Address src) { movss(dst, src); }
 206   void movflt(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 207   void movflt(Address dst, XMMRegister src) { movss(dst, src); }
 208 
 209   // Move with zero extension
 210   void movfltz(XMMRegister dst, XMMRegister src) { movss(dst, src); }
 211 
 212   void movdbl(XMMRegister dst, XMMRegister src) {
 213     if (dst-> encoding() == src->encoding()) return;
 214     if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
 215     else                       { movsd (dst, src); return; }
 216   }
 217 
 218   void movdbl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 219 
 220   void movdbl(XMMRegister dst, Address src) {
 221     if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
 222     else                         { movlpd(dst, src); return; }
 223   }
 224   void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
 225 
 226   void incrementl(AddressLiteral dst, Register rscratch = noreg);
 227   void incrementl(ArrayAddress   dst, Register rscratch);
 228 
 229   void incrementq(AddressLiteral dst, Register rscratch = noreg);
 230 
 231   // Alignment
 232   void align32();
 233   void align64();
 234   void align(int modulus);
 235   void align(int modulus, int target);
 236 
 237   void post_call_nop();
 238   // A 5 byte nop that is safe for patching (see patch_verified_entry)
 239   void fat_nop();
 240 
 241   // Stack frame creation/removal
 242   void enter();
 243   void leave();
 244 
 245   // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
 246   // The pointer will be loaded into the thread register.
 247   void get_thread(Register thread);
 248 
 249 #ifdef _LP64
 250   // Support for argument shuffling
 251 
 252   // bias in bytes
 253   void move32_64(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 254   void long_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 255   void float_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 256   void double_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
 257   void move_ptr(VMRegPair src, VMRegPair dst);
 258   void object_move(OopMap* map,
 259                    int oop_handle_offset,
 260                    int framesize_in_slots,
 261                    VMRegPair src,
 262                    VMRegPair dst,
 263                    bool is_receiver,
 264                    int* receiver_offset);
 265 #endif // _LP64
 266 
 267   // Support for VM calls
 268   //
 269   // It is imperative that all calls into the VM are handled via the call_VM macros.
 270   // They make sure that the stack linkage is setup correctly. call_VM's correspond
 271   // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
 272 
 273 
 274   void call_VM(Register oop_result,
 275                address entry_point,
 276                bool check_exceptions = true);
 277   void call_VM(Register oop_result,
 278                address entry_point,
 279                Register arg_1,
 280                bool check_exceptions = true);
 281   void call_VM(Register oop_result,
 282                address entry_point,
 283                Register arg_1, Register arg_2,
 284                bool check_exceptions = true);
 285   void call_VM(Register oop_result,
 286                address entry_point,
 287                Register arg_1, Register arg_2, Register arg_3,
 288                bool check_exceptions = true);
 289 
 290   // Overloadings with last_Java_sp
 291   void call_VM(Register oop_result,
 292                Register last_java_sp,
 293                address entry_point,
 294                int number_of_arguments = 0,
 295                bool check_exceptions = true);
 296   void call_VM(Register oop_result,
 297                Register last_java_sp,
 298                address entry_point,
 299                Register arg_1, bool
 300                check_exceptions = true);
 301   void call_VM(Register oop_result,
 302                Register last_java_sp,
 303                address entry_point,
 304                Register arg_1, Register arg_2,
 305                bool check_exceptions = true);
 306   void call_VM(Register oop_result,
 307                Register last_java_sp,
 308                address entry_point,
 309                Register arg_1, Register arg_2, Register arg_3,
 310                bool check_exceptions = true);
 311 
 312   void get_vm_result  (Register oop_result, Register thread);
 313   void get_vm_result_2(Register metadata_result, Register thread);
 314 
 315   // These always tightly bind to MacroAssembler::call_VM_base
 316   // bypassing the virtual implementation
 317   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
 318   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
 319   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
 320   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
 321   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
 322 
 323   void call_VM_leaf0(address entry_point);
 324   void call_VM_leaf(address entry_point,
 325                     int number_of_arguments = 0);
 326   void call_VM_leaf(address entry_point,
 327                     Register arg_1);
 328   void call_VM_leaf(address entry_point,
 329                     Register arg_1, Register arg_2);
 330   void call_VM_leaf(address entry_point,
 331                     Register arg_1, Register arg_2, Register arg_3);
 332 
 333   void call_VM_leaf(address entry_point,
 334                     Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 335 
 336   // These always tightly bind to MacroAssembler::call_VM_leaf_base
 337   // bypassing the virtual implementation
 338   void super_call_VM_leaf(address entry_point);
 339   void super_call_VM_leaf(address entry_point, Register arg_1);
 340   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
 341   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
 342   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 343 
 344   // last Java Frame (fills frame anchor)
 345   void set_last_Java_frame(Register thread,
 346                            Register last_java_sp,
 347                            Register last_java_fp,
 348                            address  last_java_pc,
 349                            Register rscratch);
 350 
 351   // thread in the default location (r15_thread on 64bit)
 352   void set_last_Java_frame(Register last_java_sp,
 353                            Register last_java_fp,
 354                            address  last_java_pc,
 355                            Register rscratch);
 356 
 357   void reset_last_Java_frame(Register thread, bool clear_fp);
 358 
 359   // thread in the default location (r15_thread on 64bit)
 360   void reset_last_Java_frame(bool clear_fp);
 361 
 362   // jobjects
 363   void clear_jweak_tag(Register possibly_jweak);
 364   void resolve_jobject(Register value, Register thread, Register tmp);
 365 
 366   // C 'boolean' to Java boolean: x == 0 ? 0 : 1
 367   void c2bool(Register x);
 368 
 369   // C++ bool manipulation
 370 
 371   void movbool(Register dst, Address src);
 372   void movbool(Address dst, bool boolconst);
 373   void movbool(Address dst, Register src);
 374   void testbool(Register dst);
 375 
 376   void resolve_oop_handle(Register result, Register tmp);
 377   void resolve_weak_handle(Register result, Register tmp);
 378   void load_mirror(Register mirror, Register method, Register tmp);
 379   void load_method_holder_cld(Register rresult, Register rmethod);
 380 
 381   void load_method_holder(Register holder, Register method);
 382 
 383   // oop manipulations
 384   void load_metadata(Register dst, Register src);
 385   void load_klass(Register dst, Register src, Register tmp);
 386   void store_klass(Register dst, Register src, Register tmp);
 387 
 388   void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 389                       Register tmp1, Register thread_tmp);
 390   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
 391                        Register tmp1, Register tmp2, Register tmp3);
 392 
 393   void access_value_copy(DecoratorSet decorators, Register src, Register dst, Register inline_klass);
 394 
 395   // inline type data payload offsets...
 396   void first_field_offset(Register inline_klass, Register offset);
 397   void data_for_oop(Register oop, Register data, Register inline_klass);
 398   // get data payload ptr a flat value array at index, kills rcx and index
 399   void data_for_value_array_index(Register array, Register array_klass,
 400                                   Register index, Register data);
 401 
 402 
 403   void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
 404                      Register thread_tmp = noreg, DecoratorSet decorators = 0);
 405   void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
 406                               Register thread_tmp = noreg, DecoratorSet decorators = 0);
 407   void store_heap_oop(Address dst, Register src, Register tmp1 = noreg,
 408                       Register tmp2 = noreg, Register tmp3 = noreg, DecoratorSet decorators = 0);
 409 
 410   // Used for storing NULL. All other oop constants should be
 411   // stored using routines that take a jobject.
 412   void store_heap_oop_null(Address dst);
 413 
 414   void load_prototype_header(Register dst, Register src, Register tmp);
 415 
 416 #ifdef _LP64
 417   void store_klass_gap(Register dst, Register src);
 418 
 419   // This dummy is to prevent a call to store_heap_oop from
 420   // converting a zero (like NULL) into a Register by giving
 421   // the compiler two choices it can't resolve
 422 
 423   void store_heap_oop(Address dst, void* dummy);
 424 
 425   void encode_heap_oop(Register r);
 426   void decode_heap_oop(Register r);
 427   void encode_heap_oop_not_null(Register r);
 428   void decode_heap_oop_not_null(Register r);
 429   void encode_heap_oop_not_null(Register dst, Register src);
 430   void decode_heap_oop_not_null(Register dst, Register src);
 431 
 432   void set_narrow_oop(Register dst, jobject obj);
 433   void set_narrow_oop(Address dst, jobject obj);
 434   void cmp_narrow_oop(Register dst, jobject obj);
 435   void cmp_narrow_oop(Address dst, jobject obj);
 436 
 437   void encode_klass_not_null(Register r, Register tmp);
 438   void decode_klass_not_null(Register r, Register tmp);
 439   void encode_and_move_klass_not_null(Register dst, Register src);
 440   void decode_and_move_klass_not_null(Register dst, Register src);
 441   void set_narrow_klass(Register dst, Klass* k);
 442   void set_narrow_klass(Address dst, Klass* k);
 443   void cmp_narrow_klass(Register dst, Klass* k);
 444   void cmp_narrow_klass(Address dst, Klass* k);
 445 
 446   // if heap base register is used - reinit it with the correct value
 447   void reinit_heapbase();
 448 
 449   DEBUG_ONLY(void verify_heapbase(const char* msg);)
 450 
 451 #endif // _LP64
 452 
 453   // Int division/remainder for Java
 454   // (as idivl, but checks for special case as described in JVM spec.)
 455   // returns idivl instruction offset for implicit exception handling
 456   int corrected_idivl(Register reg);
 457 
 458   // Long division/remainder for Java
 459   // (as idivq, but checks for special case as described in JVM spec.)
 460   // returns idivq instruction offset for implicit exception handling
 461   int corrected_idivq(Register reg);
 462 
 463   void int3();
 464 
 465   // Long operation macros for a 32bit cpu
 466   // Long negation for Java
 467   void lneg(Register hi, Register lo);
 468 
 469   // Long multiplication for Java
 470   // (destroys contents of eax, ebx, ecx and edx)
 471   void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
 472 
 473   // Long shifts for Java
 474   // (semantics as described in JVM spec.)
 475   void lshl(Register hi, Register lo);                               // hi:lo << (rcx & 0x3f)
 476   void lshr(Register hi, Register lo, bool sign_extension = false);  // hi:lo >> (rcx & 0x3f)
 477 
 478   // Long compare for Java
 479   // (semantics as described in JVM spec.)
 480   void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
 481 
 482 
 483   // misc
 484 
 485   // Sign extension
 486   void sign_extend_short(Register reg);
 487   void sign_extend_byte(Register reg);
 488 
 489   // Division by power of 2, rounding towards 0
 490   void division_with_shift(Register reg, int shift_value);
 491 
 492 #ifndef _LP64
 493   // Compares the top-most stack entries on the FPU stack and sets the eflags as follows:
 494   //
 495   // CF (corresponds to C0) if x < y
 496   // PF (corresponds to C2) if unordered
 497   // ZF (corresponds to C3) if x = y
 498   //
 499   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 500   // tmp is a temporary register, if none is available use noreg (only matters for non-P6 code)
 501   void fcmp(Register tmp);
 502   // Variant of the above which allows y to be further down the stack
 503   // and which only pops x and y if specified. If pop_right is
 504   // specified then pop_left must also be specified.
 505   void fcmp(Register tmp, int index, bool pop_left, bool pop_right);
 506 
 507   // Floating-point comparison for Java
 508   // Compares the top-most stack entries on the FPU stack and stores the result in dst.
 509   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 510   // (semantics as described in JVM spec.)
 511   void fcmp2int(Register dst, bool unordered_is_less);
 512   // Variant of the above which allows y to be further down the stack
 513   // and which only pops x and y if specified. If pop_right is
 514   // specified then pop_left must also be specified.
 515   void fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right);
 516 
 517   // Floating-point remainder for Java (ST0 = ST0 fremr ST1, ST1 is empty afterwards)
 518   // tmp is a temporary register, if none is available use noreg
 519   void fremr(Register tmp);
 520 
 521   // only if +VerifyFPU
 522   void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
 523 #endif // !LP64
 524 
 525   // dst = c = a * b + c
 526   void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 527   void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 528 
 529   void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 530   void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 531   void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 532   void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 533 
 534 
 535   // same as fcmp2int, but using SSE2
 536   void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 537   void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 538 
 539   // branch to L if FPU flag C2 is set/not set
 540   // tmp is a temporary register, if none is available use noreg
 541   void jC2 (Register tmp, Label& L);
 542   void jnC2(Register tmp, Label& L);
 543 
 544   // Load float value from 'address'. If UseSSE >= 1, the value is loaded into
 545   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 546   void load_float(Address src);
 547 
 548   // Store float value to 'address'. If UseSSE >= 1, the value is stored
 549   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 550   void store_float(Address dst);
 551 
 552   // Load double value from 'address'. If UseSSE >= 2, the value is loaded into
 553   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 554   void load_double(Address src);
 555 
 556   // Store double value to 'address'. If UseSSE >= 2, the value is stored
 557   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 558   void store_double(Address dst);
 559 
 560 #ifndef _LP64
 561   // Pop ST (ffree & fincstp combined)
 562   void fpop();
 563 
 564   void empty_FPU_stack();
 565 #endif // !_LP64
 566 
 567   void push_IU_state();
 568   void pop_IU_state();
 569 
 570   void push_FPU_state();
 571   void pop_FPU_state();
 572 
 573   void push_CPU_state();
 574   void pop_CPU_state();
 575 
 576   void push_cont_fastpath();
 577   void pop_cont_fastpath();
 578 
 579   void inc_held_monitor_count();
 580   void dec_held_monitor_count();
 581 
 582   DEBUG_ONLY(void stop_if_in_cont(Register cont_reg, const char* name);)
 583 
 584   // Round up to a power of two
 585   void round_to(Register reg, int modulus);
 586 
 587 private:
 588   // General purpose and XMM registers potentially clobbered by native code; there
 589   // is no need for FPU or AVX opmask related methods because C1/interpreter
 590   // - we save/restore FPU state as a whole always
 591   // - do not care about AVX-512 opmask
 592   static RegSet call_clobbered_gp_registers();
 593   static XMMRegSet call_clobbered_xmm_registers();
 594 
 595   void push_set(XMMRegSet set, int offset);
 596   void pop_set(XMMRegSet set, int offset);
 597 
 598 public:
 599   void push_set(RegSet set, int offset = -1);
 600   void pop_set(RegSet set, int offset = -1);
 601 
 602   // Push and pop everything that might be clobbered by a native
 603   // runtime call.
 604   // Only save the lower 64 bits of each vector register.
 605   // Additional registers can be excluded in a passed RegSet.
 606   void push_call_clobbered_registers_except(RegSet exclude, bool save_fpu = true);
 607   void pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu = true);
 608 
 609   void push_call_clobbered_registers(bool save_fpu = true) {
 610     push_call_clobbered_registers_except(RegSet(), save_fpu);
 611   }
 612   void pop_call_clobbered_registers(bool restore_fpu = true) {
 613     pop_call_clobbered_registers_except(RegSet(), restore_fpu);
 614   }
 615 
 616   // allocation
 617 
 618   // Object / value buffer allocation...
 619   // Allocate instance of klass, assumes klass initialized by caller
 620   // new_obj prefers to be rax
 621   // Kills t1 and t2, perserves klass, return allocation in new_obj (rsi on LP64)
 622   void allocate_instance(Register klass, Register new_obj,
 623                          Register t1, Register t2,
 624                          bool clear_fields, Label& alloc_failed);
 625 
 626   void tlab_allocate(
 627     Register thread,                   // Current thread
 628     Register obj,                      // result: pointer to object after successful allocation
 629     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 630     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 631     Register t1,                       // temp register
 632     Register t2,                       // temp register
 633     Label&   slow_case                 // continuation point if fast allocation fails
 634   );
 635   void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp);
 636 
 637   // For field "index" within "klass", return inline_klass ...
 638   void get_inline_type_field_klass(Register klass, Register index, Register inline_klass);
 639 
 640   // interface method calling
 641   void lookup_interface_method(Register recv_klass,
 642                                Register intf_klass,
 643                                RegisterOrConstant itable_index,
 644                                Register method_result,
 645                                Register scan_temp,
 646                                Label& no_such_interface,
 647                                bool return_method = true);
 648 
 649   // virtual method calling
 650   void lookup_virtual_method(Register recv_klass,
 651                              RegisterOrConstant vtable_index,
 652                              Register method_result);
 653 
 654   // Test sub_klass against super_klass, with fast and slow paths.
 655 
 656   // The fast path produces a tri-state answer: yes / no / maybe-slow.
 657   // One of the three labels can be NULL, meaning take the fall-through.
 658   // If super_check_offset is -1, the value is loaded up from super_klass.
 659   // No registers are killed, except temp_reg.
 660   void check_klass_subtype_fast_path(Register sub_klass,
 661                                      Register super_klass,
 662                                      Register temp_reg,
 663                                      Label* L_success,
 664                                      Label* L_failure,
 665                                      Label* L_slow_path,
 666                 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
 667 
 668   // The rest of the type check; must be wired to a corresponding fast path.
 669   // It does not repeat the fast path logic, so don't use it standalone.
 670   // The temp_reg and temp2_reg can be noreg, if no temps are available.
 671   // Updates the sub's secondary super cache as necessary.
 672   // If set_cond_codes, condition codes will be Z on success, NZ on failure.
 673   void check_klass_subtype_slow_path(Register sub_klass,
 674                                      Register super_klass,
 675                                      Register temp_reg,
 676                                      Register temp2_reg,
 677                                      Label* L_success,
 678                                      Label* L_failure,
 679                                      bool set_cond_codes = false);
 680 
 681   // Simplified, combined version, good for typical uses.
 682   // Falls through on failure.
 683   void check_klass_subtype(Register sub_klass,
 684                            Register super_klass,
 685                            Register temp_reg,
 686                            Label& L_success);
 687 
 688   void clinit_barrier(Register klass,
 689                       Register thread,
 690                       Label* L_fast_path = NULL,
 691                       Label* L_slow_path = NULL);
 692 
 693   // method handles (JSR 292)
 694   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
 695 
 696   // Debugging
 697 
 698   // only if +VerifyOops
 699   void _verify_oop(Register reg, const char* s, const char* file, int line);
 700   void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
 701 
 702   void _verify_oop_checked(Register reg, const char* s, const char* file, int line) {
 703     if (VerifyOops) {
 704       _verify_oop(reg, s, file, line);
 705     }
 706   }
 707   void _verify_oop_addr_checked(Address reg, const char* s, const char* file, int line) {
 708     if (VerifyOops) {
 709       _verify_oop_addr(reg, s, file, line);
 710     }
 711   }
 712 
 713   // TODO: verify method and klass metadata (compare against vptr?)
 714   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
 715   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
 716 
 717 #define verify_oop(reg) _verify_oop_checked(reg, "broken oop " #reg, __FILE__, __LINE__)
 718 #define verify_oop_msg(reg, msg) _verify_oop_checked(reg, "broken oop " #reg ", " #msg, __FILE__, __LINE__)
 719 #define verify_oop_addr(addr) _verify_oop_addr_checked(addr, "broken oop addr " #addr, __FILE__, __LINE__)
 720 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
 721 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
 722 
 723   // Verify or restore cpu control state after JNI call
 724   void restore_cpu_control_state_after_jni(Register rscratch);
 725 
 726   // prints msg, dumps registers and stops execution
 727   void stop(const char* msg);
 728 
 729   // prints msg and continues
 730   void warn(const char* msg);
 731 
 732   // dumps registers and other state
 733   void print_state();
 734 
 735   static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
 736   static void debug64(char* msg, int64_t pc, int64_t regs[]);
 737   static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
 738   static void print_state64(int64_t pc, int64_t regs[]);
 739 
 740   void os_breakpoint();
 741 
 742   void untested()                                { stop("untested"); }
 743 
 744   void unimplemented(const char* what = "");
 745 
 746   void should_not_reach_here()                   { stop("should not reach here"); }
 747 
 748   void print_CPU_state();
 749 
 750   // Stack overflow checking
 751   void bang_stack_with_offset(int offset) {
 752     // stack grows down, caller passes positive offset
 753     assert(offset > 0, "must bang with negative offset");
 754     movl(Address(rsp, (-offset)), rax);
 755   }
 756 
 757   // Writes to stack successive pages until offset reached to check for
 758   // stack overflow + shadow pages.  Also, clobbers tmp
 759   void bang_stack_size(Register size, Register tmp);
 760 
 761   // Check for reserved stack access in method being exited (for JIT)
 762   void reserved_stack_check();
 763 
 764   void safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod);
 765 
 766   void verify_tlab();
 767 
 768   static Condition negate_condition(Condition cond);
 769 
 770   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
 771   // operands. In general the names are modified to avoid hiding the instruction in Assembler
 772   // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
 773   // here in MacroAssembler. The major exception to this rule is call
 774 
 775   // Arithmetics
 776 
 777 
 778   void addptr(Address dst, int32_t src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)) ; }
 779   void addptr(Address dst, Register src);
 780 
 781   void addptr(Register dst, Address src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); }
 782   void addptr(Register dst, int32_t src);
 783   void addptr(Register dst, Register src);
 784   void addptr(Register dst, RegisterOrConstant src) {
 785     if (src.is_constant()) addptr(dst, src.as_constant());
 786     else                   addptr(dst, src.as_register());
 787   }
 788 
 789   void andptr(Register dst, int32_t src);
 790   void andptr(Register dst, Register src) { LP64_ONLY(andq(dst, src)) NOT_LP64(andl(dst, src)) ; }
 791   void andptr(Register dst, Address src) { LP64_ONLY(andq(dst, src)) NOT_LP64(andl(dst, src)) ; }
 792 
 793   void cmp8(AddressLiteral src1, int imm, Register rscratch = noreg);
 794 
 795   // renamed to drag out the casting of address to int32_t/intptr_t
 796   void cmp32(Register src1, int32_t imm);
 797 
 798   void cmp32(AddressLiteral src1, int32_t imm, Register rscratch = noreg);
 799   // compare reg - mem, or reg - &mem
 800   void cmp32(Register src1, AddressLiteral src2, Register rscratch = noreg);
 801 
 802   void cmp32(Register src1, Address src2);
 803 
 804 #ifndef _LP64
 805   void cmpklass(Address dst, Metadata* obj);
 806   void cmpklass(Register dst, Metadata* obj);
 807   void cmpoop(Address dst, jobject obj);
 808 #endif // _LP64
 809 
 810   void cmpoop(Register src1, Register src2);
 811   void cmpoop(Register src1, Address src2);
 812   void cmpoop(Register dst, jobject obj, Register rscratch);
 813 
 814   // NOTE src2 must be the lval. This is NOT an mem-mem compare
 815   void cmpptr(Address src1, AddressLiteral src2, Register rscratch);
 816 
 817   void cmpptr(Register src1, AddressLiteral src2, Register rscratch = noreg);
 818 
 819   void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 820   void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 821   // void cmpptr(Address src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 822 
 823   void cmpptr(Register src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 824   void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 825 
 826   // cmp64 to avoild hiding cmpq
 827   void cmp64(Register src1, AddressLiteral src, Register rscratch = noreg);
 828 
 829   void cmpxchgptr(Register reg, Address adr);
 830 
 831   void locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch = noreg);
 832 
 833   void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
 834   void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
 835 
 836 
 837   void negptr(Register dst) { LP64_ONLY(negq(dst)) NOT_LP64(negl(dst)); }
 838 
 839   void notptr(Register dst) { LP64_ONLY(notq(dst)) NOT_LP64(notl(dst)); }
 840 
 841   void shlptr(Register dst, int32_t shift);
 842   void shlptr(Register dst) { LP64_ONLY(shlq(dst)) NOT_LP64(shll(dst)); }
 843 
 844   void shrptr(Register dst, int32_t shift);
 845   void shrptr(Register dst) { LP64_ONLY(shrq(dst)) NOT_LP64(shrl(dst)); }
 846 
 847   void sarptr(Register dst) { LP64_ONLY(sarq(dst)) NOT_LP64(sarl(dst)); }
 848   void sarptr(Register dst, int32_t src) { LP64_ONLY(sarq(dst, src)) NOT_LP64(sarl(dst, src)); }
 849 
 850   void subptr(Address dst, int32_t src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 851 
 852   void subptr(Register dst, Address src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 853   void subptr(Register dst, int32_t src);
 854   // Force generation of a 4 byte immediate value even if it fits into 8bit
 855   void subptr_imm32(Register dst, int32_t src);
 856   void subptr(Register dst, Register src);
 857   void subptr(Register dst, RegisterOrConstant src) {
 858     if (src.is_constant()) subptr(dst, (int) src.as_constant());
 859     else                   subptr(dst,       src.as_register());
 860   }
 861 
 862   void sbbptr(Address dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 863   void sbbptr(Register dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 864 
 865   void xchgptr(Register src1, Register src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 866   void xchgptr(Register src1, Address src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 867 
 868   void xaddptr(Address src1, Register src2) { LP64_ONLY(xaddq(src1, src2)) NOT_LP64(xaddl(src1, src2)) ; }
 869 
 870 
 871 
 872   // Helper functions for statistics gathering.
 873   // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
 874   void cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch = noreg);
 875   // Unconditional atomic increment.
 876   void atomic_incl(Address counter_addr);
 877   void atomic_incl(AddressLiteral counter_addr, Register rscratch = noreg);
 878 #ifdef _LP64
 879   void atomic_incq(Address counter_addr);
 880   void atomic_incq(AddressLiteral counter_addr, Register rscratch = noreg);
 881 #endif
 882   void atomic_incptr(AddressLiteral counter_addr, Register rscratch = noreg) { LP64_ONLY(atomic_incq(counter_addr, rscratch)) NOT_LP64(atomic_incl(counter_addr, rscratch)) ; }
 883   void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
 884 
 885   void lea(Register dst, Address        adr) { Assembler::lea(dst, adr); }
 886   void lea(Register dst, AddressLiteral adr);
 887   void lea(Address  dst, AddressLiteral adr, Register rscratch);
 888 
 889   void leal32(Register dst, Address src) { leal(dst, src); }
 890 
 891   // Import other testl() methods from the parent class or else
 892   // they will be hidden by the following overriding declaration.
 893   using Assembler::testl;
 894   void testl(Address dst, int32_t imm32);
 895   void testl(Register dst, int32_t imm32);
 896   void testl(Register dst, AddressLiteral src); // requires reachable address
 897   using Assembler::testq;
 898   void testq(Address dst, int32_t imm32);
 899   void testq(Register dst, int32_t imm32);
 900 
 901   void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 902   void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 903   void orptr(Register dst, int32_t src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 904   void orptr(Address dst, int32_t imm32) { LP64_ONLY(orq(dst, imm32)) NOT_LP64(orl(dst, imm32)); }
 905 
 906   void testptr(Register src, int32_t imm32) {  LP64_ONLY(testq(src, imm32)) NOT_LP64(testl(src, imm32)); }
 907   void testptr(Register src1, Address src2) { LP64_ONLY(testq(src1, src2)) NOT_LP64(testl(src1, src2)); }
 908   void testptr(Register src1, Register src2);
 909 
 910   void xorptr(Register dst, Register src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 911   void xorptr(Register dst, Address src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 912 
 913   // Calls
 914 
 915   void call(Label& L, relocInfo::relocType rtype);
 916   void call(Register entry);
 917   void call(Address addr) { Assembler::call(addr); }
 918 
 919   // NOTE: this call transfers to the effective address of entry NOT
 920   // the address contained by entry. This is because this is more natural
 921   // for jumps/calls.
 922   void call(AddressLiteral entry, Register rscratch = rax);
 923 
 924   // Emit the CompiledIC call idiom
 925   void ic_call(address entry, jint method_index = 0);
 926 
 927   void emit_static_call_stub();
 928 
 929   // Jumps
 930 
 931   // NOTE: these jumps transfer to the effective address of dst NOT
 932   // the address contained by dst. This is because this is more natural
 933   // for jumps/calls.
 934   void jump(AddressLiteral dst, Register rscratch = noreg);
 935 
 936   void jump_cc(Condition cc, AddressLiteral dst, Register rscratch = noreg);
 937 
 938   // 32bit can do a case table jump in one instruction but we no longer allow the base
 939   // to be installed in the Address class. This jump will transfer to the address
 940   // contained in the location described by entry (not the address of entry)
 941   void jump(ArrayAddress entry, Register rscratch);
 942 
 943   // Floating
 944 
 945   void push_f(XMMRegister r);
 946   void pop_f(XMMRegister r);
 947   void push_d(XMMRegister r);
 948   void pop_d(XMMRegister r);
 949 
 950   void andpd(XMMRegister dst, XMMRegister    src) { Assembler::andpd(dst, src); }
 951   void andpd(XMMRegister dst, Address        src) { Assembler::andpd(dst, src); }
 952   void andpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 953 
 954   void andps(XMMRegister dst, XMMRegister    src) { Assembler::andps(dst, src); }
 955   void andps(XMMRegister dst, Address        src) { Assembler::andps(dst, src); }
 956   void andps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 957 
 958   void comiss(XMMRegister dst, XMMRegister    src) { Assembler::comiss(dst, src); }
 959   void comiss(XMMRegister dst, Address        src) { Assembler::comiss(dst, src); }
 960   void comiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 961 
 962   void comisd(XMMRegister dst, XMMRegister    src) { Assembler::comisd(dst, src); }
 963   void comisd(XMMRegister dst, Address        src) { Assembler::comisd(dst, src); }
 964   void comisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
 965 
 966 #ifndef _LP64
 967   void fadd_s(Address        src) { Assembler::fadd_s(src); }
 968   void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
 969 
 970   void fldcw(Address        src) { Assembler::fldcw(src); }
 971   void fldcw(AddressLiteral src);
 972 
 973   void fld_s(int index)          { Assembler::fld_s(index); }
 974   void fld_s(Address        src) { Assembler::fld_s(src); }
 975   void fld_s(AddressLiteral src);
 976 
 977   void fld_d(Address        src) { Assembler::fld_d(src); }
 978   void fld_d(AddressLiteral src);
 979 
 980   void fld_x(Address        src) { Assembler::fld_x(src); }
 981   void fld_x(AddressLiteral src) { Assembler::fld_x(as_Address(src)); }
 982 
 983   void fmul_s(Address        src) { Assembler::fmul_s(src); }
 984   void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
 985 #endif // !_LP64
 986 
 987   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
 988   void ldmxcsr(AddressLiteral src, Register rscratch = noreg);
 989 
 990 #ifdef _LP64
 991  private:
 992   void sha256_AVX2_one_round_compute(
 993     Register  reg_old_h,
 994     Register  reg_a,
 995     Register  reg_b,
 996     Register  reg_c,
 997     Register  reg_d,
 998     Register  reg_e,
 999     Register  reg_f,
1000     Register  reg_g,
1001     Register  reg_h,
1002     int iter);
1003   void sha256_AVX2_four_rounds_compute_first(int start);
1004   void sha256_AVX2_four_rounds_compute_last(int start);
1005   void sha256_AVX2_one_round_and_sched(
1006         XMMRegister xmm_0,     /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */
1007         XMMRegister xmm_1,     /* ymm5 */  /* full cycle is 16 iterations */
1008         XMMRegister xmm_2,     /* ymm6 */
1009         XMMRegister xmm_3,     /* ymm7 */
1010         Register    reg_a,      /* == eax on 0 iteration, then rotate 8 register right on each next iteration */
1011         Register    reg_b,      /* ebx */    /* full cycle is 8 iterations */
1012         Register    reg_c,      /* edi */
1013         Register    reg_d,      /* esi */
1014         Register    reg_e,      /* r8d */
1015         Register    reg_f,      /* r9d */
1016         Register    reg_g,      /* r10d */
1017         Register    reg_h,      /* r11d */
1018         int iter);
1019 
1020   void addm(int disp, Register r1, Register r2);
1021 
1022   void sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, Register d,
1023                                      Register e, Register f, Register g, Register h, int iteration);
1024 
1025   void sha512_AVX2_one_round_and_schedule(XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1026                                           Register a, Register b, Register c, Register d, Register e, Register f,
1027                                           Register g, Register h, int iteration);
1028 
1029   void addmq(int disp, Register r1, Register r2);
1030  public:
1031   void sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1032                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1033                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1034                    bool multi_block, XMMRegister shuf_mask);
1035   void sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1036                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1037                    Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
1038                    XMMRegister shuf_mask);
1039 #endif // _LP64
1040 
1041   void fast_md5(Register buf, Address state, Address ofs, Address limit,
1042                 bool multi_block);
1043 
1044   void fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0,
1045                  XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask,
1046                  Register buf, Register state, Register ofs, Register limit, Register rsp,
1047                  bool multi_block);
1048 
1049 #ifdef _LP64
1050   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1051                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1052                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1053                    bool multi_block, XMMRegister shuf_mask);
1054 #else
1055   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1056                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1057                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1058                    bool multi_block);
1059 #endif
1060 
1061   void fast_exp(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1062                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1063                 Register rax, Register rcx, Register rdx, Register tmp);
1064 
1065 #ifndef _LP64
1066  private:
1067   // Initialized in macroAssembler_x86_constants.cpp
1068   static address ONES;
1069   static address L_2IL0FLOATPACKET_0;
1070   static address PI4_INV;
1071   static address PI4X3;
1072   static address PI4X4;
1073 
1074  public:
1075   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1076                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1077                 Register rax, Register rcx, Register rdx, Register tmp1);
1078 
1079   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1080                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1081                 Register rax, Register rcx, Register rdx, Register tmp);
1082 
1083   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1084                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
1085                 Register rdx, Register tmp);
1086 
1087   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1088                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1089                 Register rax, Register rbx, Register rdx);
1090 
1091   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1092                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1093                 Register rax, Register rcx, Register rdx, Register tmp);
1094 
1095   void libm_sincos_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1096                         Register edx, Register ebx, Register esi, Register edi,
1097                         Register ebp, Register esp);
1098 
1099   void libm_reduce_pi04l(Register eax, Register ecx, Register edx, Register ebx,
1100                          Register esi, Register edi, Register ebp, Register esp);
1101 
1102   void libm_tancot_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1103                         Register edx, Register ebx, Register esi, Register edi,
1104                         Register ebp, Register esp);
1105 
1106   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1107                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1108                 Register rax, Register rcx, Register rdx, Register tmp);
1109 #endif // !_LP64
1110 
1111 private:
1112 
1113   // these are private because users should be doing movflt/movdbl
1114 
1115   void movss(Address     dst, XMMRegister    src) { Assembler::movss(dst, src); }
1116   void movss(XMMRegister dst, XMMRegister    src) { Assembler::movss(dst, src); }
1117   void movss(XMMRegister dst, Address        src) { Assembler::movss(dst, src); }
1118   void movss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1119 
1120   void movlpd(XMMRegister dst, Address        src) {Assembler::movlpd(dst, src); }
1121   void movlpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1122 
1123 public:
1124 
1125   void addsd(XMMRegister dst, XMMRegister    src) { Assembler::addsd(dst, src); }
1126   void addsd(XMMRegister dst, Address        src) { Assembler::addsd(dst, src); }
1127   void addsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1128 
1129   void addss(XMMRegister dst, XMMRegister    src) { Assembler::addss(dst, src); }
1130   void addss(XMMRegister dst, Address        src) { Assembler::addss(dst, src); }
1131   void addss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1132 
1133   void addpd(XMMRegister dst, XMMRegister    src) { Assembler::addpd(dst, src); }
1134   void addpd(XMMRegister dst, Address        src) { Assembler::addpd(dst, src); }
1135   void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1136 
1137   using Assembler::vbroadcastsd;
1138   void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1139 
1140   using Assembler::vbroadcastss;
1141   void vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1142 
1143   void divsd(XMMRegister dst, XMMRegister    src) { Assembler::divsd(dst, src); }
1144   void divsd(XMMRegister dst, Address        src) { Assembler::divsd(dst, src); }
1145   void divsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1146 
1147   void divss(XMMRegister dst, XMMRegister    src) { Assembler::divss(dst, src); }
1148   void divss(XMMRegister dst, Address        src) { Assembler::divss(dst, src); }
1149   void divss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1150 
1151   // Move Unaligned Double Quadword
1152   void movdqu(Address     dst, XMMRegister    src);
1153   void movdqu(XMMRegister dst, XMMRegister    src);
1154   void movdqu(XMMRegister dst, Address        src);
1155   void movdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1156 
1157   void kmovwl(Register  dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1158   void kmovwl(Address   dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1159   void kmovwl(KRegister dst, KRegister      src) { Assembler::kmovwl(dst, src); }
1160   void kmovwl(KRegister dst, Register       src) { Assembler::kmovwl(dst, src); }
1161   void kmovwl(KRegister dst, Address        src) { Assembler::kmovwl(dst, src); }
1162   void kmovwl(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1163 
1164   void kmovql(KRegister dst, KRegister      src) { Assembler::kmovql(dst, src); }
1165   void kmovql(KRegister dst, Register       src) { Assembler::kmovql(dst, src); }
1166   void kmovql(Register  dst, KRegister      src) { Assembler::kmovql(dst, src); }
1167   void kmovql(KRegister dst, Address        src) { Assembler::kmovql(dst, src); }
1168   void kmovql(Address   dst, KRegister      src) { Assembler::kmovql(dst, src); }
1169   void kmovql(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1170 
1171   // Safe move operation, lowers down to 16bit moves for targets supporting
1172   // AVX512F feature and 64bit moves for targets supporting AVX512BW feature.
1173   void kmov(Address  dst, KRegister src);
1174   void kmov(KRegister dst, Address src);
1175   void kmov(KRegister dst, KRegister src);
1176   void kmov(Register dst, KRegister src);
1177   void kmov(KRegister dst, Register src);
1178 
1179   using Assembler::movddup;
1180   void movddup(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1181 
1182   using Assembler::vmovddup;
1183   void vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1184 
1185   // AVX Unaligned forms
1186   void vmovdqu(Address     dst, XMMRegister    src);
1187   void vmovdqu(XMMRegister dst, Address        src);
1188   void vmovdqu(XMMRegister dst, XMMRegister    src);
1189   void vmovdqu(XMMRegister dst, AddressLiteral src,                 Register rscratch = noreg);
1190   void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1191 
1192   // AVX512 Unaligned
1193   void evmovdqu(BasicType type, KRegister kmask, Address     dst, XMMRegister src, bool merge, int vector_len);
1194   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address     src, bool merge, int vector_len);
1195 
1196   void evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1197   void evmovdqub(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1198 
1199   void evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1200     if (dst->encoding() != src->encoding() || mask != k0)  {
1201       Assembler::evmovdqub(dst, mask, src, merge, vector_len);
1202     }
1203   }
1204   void evmovdqub(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1205   void evmovdqub(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1206   void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1207 
1208   void evmovdquw(Address     dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1209   void evmovdquw(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1210 
1211   void evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1212     if (dst->encoding() != src->encoding() || mask != k0) {
1213       Assembler::evmovdquw(dst, mask, src, merge, vector_len);
1214     }
1215   }
1216   void evmovdquw(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1217   void evmovdquw(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1218   void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1219 
1220   void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
1221      if (dst->encoding() != src->encoding()) {
1222        Assembler::evmovdqul(dst, src, vector_len);
1223      }
1224   }
1225   void evmovdqul(Address     dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1226   void evmovdqul(XMMRegister dst, Address     src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1227 
1228   void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1229     if (dst->encoding() != src->encoding() || mask != k0)  {
1230       Assembler::evmovdqul(dst, mask, src, merge, vector_len);
1231     }
1232   }
1233   void evmovdqul(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1234   void evmovdqul(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1235   void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1236 
1237   void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
1238     if (dst->encoding() != src->encoding()) {
1239       Assembler::evmovdquq(dst, src, vector_len);
1240     }
1241   }
1242   void evmovdquq(XMMRegister dst, Address        src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1243   void evmovdquq(Address     dst, XMMRegister    src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1244   void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1245 
1246   void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1247     if (dst->encoding() != src->encoding() || mask != k0) {
1248       Assembler::evmovdquq(dst, mask, src, merge, vector_len);
1249     }
1250   }
1251   void evmovdquq(Address     dst, KRegister mask, XMMRegister    src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1252   void evmovdquq(XMMRegister dst, KRegister mask, Address        src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1253   void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1254 
1255   // Move Aligned Double Quadword
1256   void movdqa(XMMRegister dst, XMMRegister    src) { Assembler::movdqa(dst, src); }
1257   void movdqa(XMMRegister dst, Address        src) { Assembler::movdqa(dst, src); }
1258   void movdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1259 
1260   void movsd(Address     dst, XMMRegister    src) { Assembler::movsd(dst, src); }
1261   void movsd(XMMRegister dst, XMMRegister    src) { Assembler::movsd(dst, src); }
1262   void movsd(XMMRegister dst, Address        src) { Assembler::movsd(dst, src); }
1263   void movsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1264 
1265   void mulpd(XMMRegister dst, XMMRegister    src) { Assembler::mulpd(dst, src); }
1266   void mulpd(XMMRegister dst, Address        src) { Assembler::mulpd(dst, src); }
1267   void mulpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1268 
1269   void mulsd(XMMRegister dst, XMMRegister    src) { Assembler::mulsd(dst, src); }
1270   void mulsd(XMMRegister dst, Address        src) { Assembler::mulsd(dst, src); }
1271   void mulsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1272 
1273   void mulss(XMMRegister dst, XMMRegister    src) { Assembler::mulss(dst, src); }
1274   void mulss(XMMRegister dst, Address        src) { Assembler::mulss(dst, src); }
1275   void mulss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1276 
1277   // Carry-Less Multiplication Quadword
1278   void pclmulldq(XMMRegister dst, XMMRegister src) {
1279     // 0x00 - multiply lower 64 bits [0:63]
1280     Assembler::pclmulqdq(dst, src, 0x00);
1281   }
1282   void pclmulhdq(XMMRegister dst, XMMRegister src) {
1283     // 0x11 - multiply upper 64 bits [64:127]
1284     Assembler::pclmulqdq(dst, src, 0x11);
1285   }
1286 
1287   void pcmpeqb(XMMRegister dst, XMMRegister src);
1288   void pcmpeqw(XMMRegister dst, XMMRegister src);
1289 
1290   void pcmpestri(XMMRegister dst, Address src, int imm8);
1291   void pcmpestri(XMMRegister dst, XMMRegister src, int imm8);
1292 
1293   void pmovzxbw(XMMRegister dst, XMMRegister src);
1294   void pmovzxbw(XMMRegister dst, Address src);
1295 
1296   void pmovmskb(Register dst, XMMRegister src);
1297 
1298   void ptest(XMMRegister dst, XMMRegister src);
1299 
1300   void roundsd(XMMRegister dst, XMMRegister    src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1301   void roundsd(XMMRegister dst, Address        src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1302   void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch = noreg);
1303 
1304   void sqrtss(XMMRegister dst, XMMRegister     src) { Assembler::sqrtss(dst, src); }
1305   void sqrtss(XMMRegister dst, Address         src) { Assembler::sqrtss(dst, src); }
1306   void sqrtss(XMMRegister dst, AddressLiteral  src, Register rscratch = noreg);
1307 
1308   void subsd(XMMRegister dst, XMMRegister    src) { Assembler::subsd(dst, src); }
1309   void subsd(XMMRegister dst, Address        src) { Assembler::subsd(dst, src); }
1310   void subsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1311 
1312   void subss(XMMRegister dst, XMMRegister    src) { Assembler::subss(dst, src); }
1313   void subss(XMMRegister dst, Address        src) { Assembler::subss(dst, src); }
1314   void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1315 
1316   void ucomiss(XMMRegister dst, XMMRegister    src) { Assembler::ucomiss(dst, src); }
1317   void ucomiss(XMMRegister dst, Address        src) { Assembler::ucomiss(dst, src); }
1318   void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1319 
1320   void ucomisd(XMMRegister dst, XMMRegister    src) { Assembler::ucomisd(dst, src); }
1321   void ucomisd(XMMRegister dst, Address        src) { Assembler::ucomisd(dst, src); }
1322   void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1323 
1324   // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
1325   void xorpd(XMMRegister dst, XMMRegister    src);
1326   void xorpd(XMMRegister dst, Address        src) { Assembler::xorpd(dst, src); }
1327   void xorpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1328 
1329   // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
1330   void xorps(XMMRegister dst, XMMRegister    src);
1331   void xorps(XMMRegister dst, Address        src) { Assembler::xorps(dst, src); }
1332   void xorps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1333 
1334   // Shuffle Bytes
1335   void pshufb(XMMRegister dst, XMMRegister    src) { Assembler::pshufb(dst, src); }
1336   void pshufb(XMMRegister dst, Address        src) { Assembler::pshufb(dst, src); }
1337   void pshufb(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1338   // AVX 3-operands instructions
1339 
1340   void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vaddsd(dst, nds, src); }
1341   void vaddsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vaddsd(dst, nds, src); }
1342   void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1343 
1344   void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vaddss(dst, nds, src); }
1345   void vaddss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vaddss(dst, nds, src); }
1346   void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1347 
1348   void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1349   void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1350 
1351   void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len);
1352   void vpaddb(XMMRegister dst, XMMRegister nds, Address        src, int vector_len);
1353   void vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1354 
1355   void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1356   void vpaddw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1357 
1358   void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1359   void vpaddd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1360   void vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1361 
1362   void vpand(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1363   void vpand(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1364   void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1365 
1366   using Assembler::vpbroadcastd;
1367   void vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1368 
1369   using Assembler::vpbroadcastq;
1370   void vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1371 
1372   void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1373 
1374   void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1375   void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1376 
1377   // Vector compares
1378   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1379     Assembler::evpcmpd(kdst, mask, nds, src, comparison, is_signed, vector_len);
1380   }
1381   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1382 
1383   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1384     Assembler::evpcmpq(kdst, mask, nds, src, comparison, is_signed, vector_len);
1385   }
1386   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1387 
1388   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1389     Assembler::evpcmpb(kdst, mask, nds, src, comparison, is_signed, vector_len);
1390   }
1391   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1392 
1393   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister    src, int comparison, bool is_signed, int vector_len) {
1394     Assembler::evpcmpw(kdst, mask, nds, src, comparison, is_signed, vector_len);
1395   }
1396   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1397 
1398   void evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len);
1399 
1400   // Emit comparison instruction for the specified comparison predicate.
1401   void vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len);
1402   void vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len);
1403 
1404   void vpmovzxbw(XMMRegister dst, Address     src, int vector_len);
1405   void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
1406 
1407   void vpmovmskb(Register dst, XMMRegister src, int vector_len = Assembler::AVX_256bit);
1408 
1409   void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1410   void vpmullw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1411 
1412   void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1413   void vpmulld(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1414   void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1415 
1416   void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1417   void vpsubb(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1418 
1419   void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1420   void vpsubw(XMMRegister dst, XMMRegister nds, Address     src, int vector_len);
1421 
1422   void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1423   void vpsraw(XMMRegister dst, XMMRegister nds, int         shift, int vector_len);
1424 
1425   void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1426   void evpsraq(XMMRegister dst, XMMRegister nds, int         shift, int vector_len);
1427 
1428   void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1429     if (!is_varshift) {
1430       Assembler::evpsllw(dst, mask, nds, src, merge, vector_len);
1431     } else {
1432       Assembler::evpsllvw(dst, mask, nds, src, merge, vector_len);
1433     }
1434   }
1435   void evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1436     if (!is_varshift) {
1437       Assembler::evpslld(dst, mask, nds, src, merge, vector_len);
1438     } else {
1439       Assembler::evpsllvd(dst, mask, nds, src, merge, vector_len);
1440     }
1441   }
1442   void evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1443     if (!is_varshift) {
1444       Assembler::evpsllq(dst, mask, nds, src, merge, vector_len);
1445     } else {
1446       Assembler::evpsllvq(dst, mask, nds, src, merge, vector_len);
1447     }
1448   }
1449   void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1450     if (!is_varshift) {
1451       Assembler::evpsrlw(dst, mask, nds, src, merge, vector_len);
1452     } else {
1453       Assembler::evpsrlvw(dst, mask, nds, src, merge, vector_len);
1454     }
1455   }
1456   void evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1457     if (!is_varshift) {
1458       Assembler::evpsrld(dst, mask, nds, src, merge, vector_len);
1459     } else {
1460       Assembler::evpsrlvd(dst, mask, nds, src, merge, vector_len);
1461     }
1462   }
1463   void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1464     if (!is_varshift) {
1465       Assembler::evpsrlq(dst, mask, nds, src, merge, vector_len);
1466     } else {
1467       Assembler::evpsrlvq(dst, mask, nds, src, merge, vector_len);
1468     }
1469   }
1470   void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1471     if (!is_varshift) {
1472       Assembler::evpsraw(dst, mask, nds, src, merge, vector_len);
1473     } else {
1474       Assembler::evpsravw(dst, mask, nds, src, merge, vector_len);
1475     }
1476   }
1477   void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1478     if (!is_varshift) {
1479       Assembler::evpsrad(dst, mask, nds, src, merge, vector_len);
1480     } else {
1481       Assembler::evpsravd(dst, mask, nds, src, merge, vector_len);
1482     }
1483   }
1484   void evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1485     if (!is_varshift) {
1486       Assembler::evpsraq(dst, mask, nds, src, merge, vector_len);
1487     } else {
1488       Assembler::evpsravq(dst, mask, nds, src, merge, vector_len);
1489     }
1490   }
1491 
1492   void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1493   void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1494   void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1495   void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1496 
1497   void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1498   void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1499 
1500   void vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1501   void vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1502 
1503   void vptest(XMMRegister dst, XMMRegister src);
1504   void vptest(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vptest(dst, src, vector_len); }
1505 
1506   void punpcklbw(XMMRegister dst, XMMRegister src);
1507   void punpcklbw(XMMRegister dst, Address src) { Assembler::punpcklbw(dst, src); }
1508 
1509   void pshufd(XMMRegister dst, Address src, int mode);
1510   void pshufd(XMMRegister dst, XMMRegister src, int mode) { Assembler::pshufd(dst, src, mode); }
1511 
1512   void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1513   void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
1514 
1515   void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1516   void vandpd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1517   void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1518 
1519   void vandps(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1520   void vandps(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1521   void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1522 
1523   void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1524 
1525   void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vdivsd(dst, nds, src); }
1526   void vdivsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vdivsd(dst, nds, src); }
1527   void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1528 
1529   void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vdivss(dst, nds, src); }
1530   void vdivss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vdivss(dst, nds, src); }
1531   void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1532 
1533   void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vmulsd(dst, nds, src); }
1534   void vmulsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vmulsd(dst, nds, src); }
1535   void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1536 
1537   void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vmulss(dst, nds, src); }
1538   void vmulss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vmulss(dst, nds, src); }
1539   void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1540 
1541   void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vsubsd(dst, nds, src); }
1542   void vsubsd(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vsubsd(dst, nds, src); }
1543   void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1544 
1545   void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister    src) { Assembler::vsubss(dst, nds, src); }
1546   void vsubss(XMMRegister dst, XMMRegister nds, Address        src) { Assembler::vsubss(dst, nds, src); }
1547   void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1548 
1549   void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1550   void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1551 
1552   // AVX Vector instructions
1553 
1554   void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1555   void vxorpd(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1556   void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1557 
1558   void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1559   void vxorps(XMMRegister dst, XMMRegister nds, Address        src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1560   void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1561 
1562   void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1563     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1564       Assembler::vpxor(dst, nds, src, vector_len);
1565     else
1566       Assembler::vxorpd(dst, nds, src, vector_len);
1567   }
1568   void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
1569     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1570       Assembler::vpxor(dst, nds, src, vector_len);
1571     else
1572       Assembler::vxorpd(dst, nds, src, vector_len);
1573   }
1574   void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1575 
1576   // Simple version for AVX2 256bit vectors
1577   void vpxor(XMMRegister dst, XMMRegister src) {
1578     assert(UseAVX >= 2, "Should be at least AVX2");
1579     Assembler::vpxor(dst, dst, src, AVX_256bit);
1580   }
1581   void vpxor(XMMRegister dst, Address src) {
1582     assert(UseAVX >= 2, "Should be at least AVX2");
1583     Assembler::vpxor(dst, dst, src, AVX_256bit);
1584   }
1585 
1586   void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister    src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
1587   void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1588 
1589   void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
1590     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1591       Assembler::vinserti32x4(dst, nds, src, imm8);
1592     } else if (UseAVX > 1) {
1593       // vinserti128 is available only in AVX2
1594       Assembler::vinserti128(dst, nds, src, imm8);
1595     } else {
1596       Assembler::vinsertf128(dst, nds, src, imm8);
1597     }
1598   }
1599 
1600   void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
1601     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1602       Assembler::vinserti32x4(dst, nds, src, imm8);
1603     } else if (UseAVX > 1) {
1604       // vinserti128 is available only in AVX2
1605       Assembler::vinserti128(dst, nds, src, imm8);
1606     } else {
1607       Assembler::vinsertf128(dst, nds, src, imm8);
1608     }
1609   }
1610 
1611   void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
1612     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1613       Assembler::vextracti32x4(dst, src, imm8);
1614     } else if (UseAVX > 1) {
1615       // vextracti128 is available only in AVX2
1616       Assembler::vextracti128(dst, src, imm8);
1617     } else {
1618       Assembler::vextractf128(dst, src, imm8);
1619     }
1620   }
1621 
1622   void vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
1623     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1624       Assembler::vextracti32x4(dst, src, imm8);
1625     } else if (UseAVX > 1) {
1626       // vextracti128 is available only in AVX2
1627       Assembler::vextracti128(dst, src, imm8);
1628     } else {
1629       Assembler::vextractf128(dst, src, imm8);
1630     }
1631   }
1632 
1633   // 128bit copy to/from high 128 bits of 256bit (YMM) vector registers
1634   void vinserti128_high(XMMRegister dst, XMMRegister src) {
1635     vinserti128(dst, dst, src, 1);
1636   }
1637   void vinserti128_high(XMMRegister dst, Address src) {
1638     vinserti128(dst, dst, src, 1);
1639   }
1640   void vextracti128_high(XMMRegister dst, XMMRegister src) {
1641     vextracti128(dst, src, 1);
1642   }
1643   void vextracti128_high(Address dst, XMMRegister src) {
1644     vextracti128(dst, src, 1);
1645   }
1646 
1647   void vinsertf128_high(XMMRegister dst, XMMRegister src) {
1648     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1649       Assembler::vinsertf32x4(dst, dst, src, 1);
1650     } else {
1651       Assembler::vinsertf128(dst, dst, src, 1);
1652     }
1653   }
1654 
1655   void vinsertf128_high(XMMRegister dst, Address src) {
1656     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1657       Assembler::vinsertf32x4(dst, dst, src, 1);
1658     } else {
1659       Assembler::vinsertf128(dst, dst, src, 1);
1660     }
1661   }
1662 
1663   void vextractf128_high(XMMRegister dst, XMMRegister src) {
1664     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1665       Assembler::vextractf32x4(dst, src, 1);
1666     } else {
1667       Assembler::vextractf128(dst, src, 1);
1668     }
1669   }
1670 
1671   void vextractf128_high(Address dst, XMMRegister src) {
1672     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1673       Assembler::vextractf32x4(dst, src, 1);
1674     } else {
1675       Assembler::vextractf128(dst, src, 1);
1676     }
1677   }
1678 
1679   // 256bit copy to/from high 256 bits of 512bit (ZMM) vector registers
1680   void vinserti64x4_high(XMMRegister dst, XMMRegister src) {
1681     Assembler::vinserti64x4(dst, dst, src, 1);
1682   }
1683   void vinsertf64x4_high(XMMRegister dst, XMMRegister src) {
1684     Assembler::vinsertf64x4(dst, dst, src, 1);
1685   }
1686   void vextracti64x4_high(XMMRegister dst, XMMRegister src) {
1687     Assembler::vextracti64x4(dst, src, 1);
1688   }
1689   void vextractf64x4_high(XMMRegister dst, XMMRegister src) {
1690     Assembler::vextractf64x4(dst, src, 1);
1691   }
1692   void vextractf64x4_high(Address dst, XMMRegister src) {
1693     Assembler::vextractf64x4(dst, src, 1);
1694   }
1695   void vinsertf64x4_high(XMMRegister dst, Address src) {
1696     Assembler::vinsertf64x4(dst, dst, src, 1);
1697   }
1698 
1699   // 128bit copy to/from low 128 bits of 256bit (YMM) vector registers
1700   void vinserti128_low(XMMRegister dst, XMMRegister src) {
1701     vinserti128(dst, dst, src, 0);
1702   }
1703   void vinserti128_low(XMMRegister dst, Address src) {
1704     vinserti128(dst, dst, src, 0);
1705   }
1706   void vextracti128_low(XMMRegister dst, XMMRegister src) {
1707     vextracti128(dst, src, 0);
1708   }
1709   void vextracti128_low(Address dst, XMMRegister src) {
1710     vextracti128(dst, src, 0);
1711   }
1712 
1713   void vinsertf128_low(XMMRegister dst, XMMRegister src) {
1714     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1715       Assembler::vinsertf32x4(dst, dst, src, 0);
1716     } else {
1717       Assembler::vinsertf128(dst, dst, src, 0);
1718     }
1719   }
1720 
1721   void vinsertf128_low(XMMRegister dst, Address src) {
1722     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1723       Assembler::vinsertf32x4(dst, dst, src, 0);
1724     } else {
1725       Assembler::vinsertf128(dst, dst, src, 0);
1726     }
1727   }
1728 
1729   void vextractf128_low(XMMRegister dst, XMMRegister src) {
1730     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1731       Assembler::vextractf32x4(dst, src, 0);
1732     } else {
1733       Assembler::vextractf128(dst, src, 0);
1734     }
1735   }
1736 
1737   void vextractf128_low(Address dst, XMMRegister src) {
1738     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1739       Assembler::vextractf32x4(dst, src, 0);
1740     } else {
1741       Assembler::vextractf128(dst, src, 0);
1742     }
1743   }
1744 
1745   // 256bit copy to/from low 256 bits of 512bit (ZMM) vector registers
1746   void vinserti64x4_low(XMMRegister dst, XMMRegister src) {
1747     Assembler::vinserti64x4(dst, dst, src, 0);
1748   }
1749   void vinsertf64x4_low(XMMRegister dst, XMMRegister src) {
1750     Assembler::vinsertf64x4(dst, dst, src, 0);
1751   }
1752   void vextracti64x4_low(XMMRegister dst, XMMRegister src) {
1753     Assembler::vextracti64x4(dst, src, 0);
1754   }
1755   void vextractf64x4_low(XMMRegister dst, XMMRegister src) {
1756     Assembler::vextractf64x4(dst, src, 0);
1757   }
1758   void vextractf64x4_low(Address dst, XMMRegister src) {
1759     Assembler::vextractf64x4(dst, src, 0);
1760   }
1761   void vinsertf64x4_low(XMMRegister dst, Address src) {
1762     Assembler::vinsertf64x4(dst, dst, src, 0);
1763   }
1764 
1765   // Carry-Less Multiplication Quadword
1766   void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1767     // 0x00 - multiply lower 64 bits [0:63]
1768     Assembler::vpclmulqdq(dst, nds, src, 0x00);
1769   }
1770   void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1771     // 0x11 - multiply upper 64 bits [64:127]
1772     Assembler::vpclmulqdq(dst, nds, src, 0x11);
1773   }
1774   void vpclmullqhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1775     // 0x10 - multiply nds[0:63] and src[64:127]
1776     Assembler::vpclmulqdq(dst, nds, src, 0x10);
1777   }
1778   void vpclmulhqlqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1779     //0x01 - multiply nds[64:127] and src[0:63]
1780     Assembler::vpclmulqdq(dst, nds, src, 0x01);
1781   }
1782 
1783   void evpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1784     // 0x00 - multiply lower 64 bits [0:63]
1785     Assembler::evpclmulqdq(dst, nds, src, 0x00, vector_len);
1786   }
1787   void evpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1788     // 0x11 - multiply upper 64 bits [64:127]
1789     Assembler::evpclmulqdq(dst, nds, src, 0x11, vector_len);
1790   }
1791 
1792   // AVX-512 mask operations.
1793   void kand(BasicType etype, KRegister dst, KRegister src1, KRegister src2);
1794   void kor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1795   void knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp = knoreg, Register rtmp = noreg);
1796   void kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1797   void kortest(uint masklen, KRegister src1, KRegister src2);
1798   void ktest(uint masklen, KRegister src1, KRegister src2);
1799 
1800   void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1801   void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1802 
1803   void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1804   void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1805 
1806   void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1807   void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1808 
1809   void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1810   void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1811 
1812   void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1813   void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1814   void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1815   void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1816 
1817   void alltrue(Register dst, uint masklen, KRegister src1, KRegister src2, KRegister kscratch);
1818   void anytrue(Register dst, uint masklen, KRegister src, KRegister kscratch);
1819 
1820   void cmov32( Condition cc, Register dst, Address  src);
1821   void cmov32( Condition cc, Register dst, Register src);
1822 
1823   void cmov(   Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
1824 
1825   void cmovptr(Condition cc, Register dst, Address  src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1826   void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1827 
1828   void movoop(Register dst, jobject obj);
1829   void movoop(Address  dst, jobject obj, Register rscratch);
1830 
1831   void mov_metadata(Register dst, Metadata* obj);
1832   void mov_metadata(Address  dst, Metadata* obj, Register rscratch);
1833 
1834   void movptr(Register     dst, Register       src);
1835   void movptr(Register     dst, Address        src);
1836   void movptr(Register     dst, AddressLiteral src);
1837   void movptr(Register     dst, ArrayAddress   src);
1838   void movptr(Register     dst, intptr_t       src);
1839   void movptr(Address      dst, Register       src);
1840   void movptr(Address      dst, int32_t        imm);
1841   void movptr(Address      dst, intptr_t       src, Register rscratch);
1842   void movptr(ArrayAddress dst, Register       src, Register rscratch);
1843 
1844   void movptr(Register dst, RegisterOrConstant src) {
1845     if (src.is_constant()) movptr(dst, src.as_constant());
1846     else                   movptr(dst, src.as_register());
1847   }
1848 
1849 
1850   // to avoid hiding movl
1851   void mov32(Register       dst, AddressLiteral src);
1852   void mov32(AddressLiteral dst, Register        src, Register rscratch = noreg);
1853 
1854   // Import other mov() methods from the parent class or else
1855   // they will be hidden by the following overriding declaration.
1856   using Assembler::movdl;
1857   void movdl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1858 
1859   using Assembler::movq;
1860   void movq(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1861 
1862   // Can push value or effective address
1863   void pushptr(AddressLiteral src, Register rscratch);
1864 
1865   void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
1866   void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
1867 
1868   void pushoop(jobject obj, Register rscratch);
1869   void pushklass(Metadata* obj, Register rscratch);
1870 
1871   // sign extend as need a l to ptr sized element
1872   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
1873   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
1874 
1875 
1876  public:
1877   // Inline type specific methods
1878   #include "asm/macroAssembler_common.hpp"
1879 
1880   int store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter = true);
1881   bool move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]);
1882   bool unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
1883                             VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
1884                             RegState reg_state[]);
1885   bool pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
1886                           VMRegPair* from, int from_count, int& from_index, VMReg to,
1887                           RegState reg_state[], Register val_array);
1888   int extend_stack_for_inline_args(int args_on_stack);
1889   void remove_frame(int initial_framesize, bool needs_stack_repair);
1890   VMReg spill_reg_for(VMReg reg);
1891 
1892   // clear memory of size 'cnt' qwords, starting at 'base';
1893   // if 'is_large' is set, do not try to produce short loop
1894   void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only, KRegister mask=knoreg);
1895 
1896   // clear memory initialization sequence for constant size;
1897   void clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1898 
1899   // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
1900   void xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1901 
1902   // Fill primitive arrays
1903   void generate_fill(BasicType t, bool aligned,
1904                      Register to, Register value, Register count,
1905                      Register rtmp, XMMRegister xtmp);
1906 
1907   void encode_iso_array(Register src, Register dst, Register len,
1908                         XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1909                         XMMRegister tmp4, Register tmp5, Register result, bool ascii);
1910 
1911 #ifdef _LP64
1912   void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
1913   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1914                              Register y, Register y_idx, Register z,
1915                              Register carry, Register product,
1916                              Register idx, Register kdx);
1917   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
1918                               Register yz_idx, Register idx,
1919                               Register carry, Register product, int offset);
1920   void multiply_128_x_128_bmi2_loop(Register y, Register z,
1921                                     Register carry, Register carry2,
1922                                     Register idx, Register jdx,
1923                                     Register yz_idx1, Register yz_idx2,
1924                                     Register tmp, Register tmp3, Register tmp4);
1925   void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
1926                                Register yz_idx, Register idx, Register jdx,
1927                                Register carry, Register product,
1928                                Register carry2);
1929   void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
1930                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
1931   void square_rshift(Register x, Register len, Register z, Register tmp1, Register tmp3,
1932                      Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1933   void multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry,
1934                             Register tmp2);
1935   void multiply_add_64(Register sum, Register op1, Register op2, Register carry,
1936                        Register rdxReg, Register raxReg);
1937   void add_one_64(Register z, Register zlen, Register carry, Register tmp1);
1938   void lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1939                        Register tmp3, Register tmp4);
1940   void square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1941                      Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1942 
1943   void mul_add_128_x_32_loop(Register out, Register in, Register offset, Register len, Register tmp1,
1944                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1945                Register raxReg);
1946   void mul_add(Register out, Register in, Register offset, Register len, Register k, Register tmp1,
1947                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1948                Register raxReg);
1949   void vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
1950                            Register result, Register tmp1, Register tmp2,
1951                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3);
1952 #endif
1953 
1954   // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
1955   void update_byte_crc32(Register crc, Register val, Register table);
1956   void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
1957 
1958 
1959 #ifdef _LP64
1960   void kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2);
1961   void kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register key, Register pos,
1962                                 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
1963                                 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup);
1964 #endif // _LP64
1965 
1966   // CRC32C code for java.util.zip.CRC32C::updateBytes() intrinsic
1967   // Note on a naming convention:
1968   // Prefix w = register only used on a Westmere+ architecture
1969   // Prefix n = register only used on a Nehalem architecture
1970 #ifdef _LP64
1971   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1972                        Register tmp1, Register tmp2, Register tmp3);
1973 #else
1974   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1975                        Register tmp1, Register tmp2, Register tmp3,
1976                        XMMRegister xtmp1, XMMRegister xtmp2);
1977 #endif
1978   void crc32c_pclmulqdq(XMMRegister w_xtmp1,
1979                         Register in_out,
1980                         uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
1981                         XMMRegister w_xtmp2,
1982                         Register tmp1,
1983                         Register n_tmp2, Register n_tmp3);
1984   void crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
1985                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1986                        Register tmp1, Register tmp2,
1987                        Register n_tmp3);
1988   void crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
1989                          Register in_out1, Register in_out2, Register in_out3,
1990                          Register tmp1, Register tmp2, Register tmp3,
1991                          XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1992                          Register tmp4, Register tmp5,
1993                          Register n_tmp6);
1994   void crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
1995                             Register tmp1, Register tmp2, Register tmp3,
1996                             Register tmp4, Register tmp5, Register tmp6,
1997                             XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1998                             bool is_pclmulqdq_supported);
1999   // Fold 128-bit data chunk
2000   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
2001   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
2002 #ifdef _LP64
2003   // Fold 512-bit data chunk
2004   void fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, Register pos, int offset);
2005 #endif // _LP64
2006   // Fold 8-bit data
2007   void fold_8bit_crc32(Register crc, Register table, Register tmp);
2008   void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
2009 
2010   // Compress char[] array to byte[].
2011   void char_array_compress(Register src, Register dst, Register len,
2012                            XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
2013                            XMMRegister tmp4, Register tmp5, Register result,
2014                            KRegister mask1 = knoreg, KRegister mask2 = knoreg);
2015 
2016   // Inflate byte[] array to char[].
2017   void byte_array_inflate(Register src, Register dst, Register len,
2018                           XMMRegister tmp1, Register tmp2, KRegister mask = knoreg);
2019 
2020   void fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
2021                    Register length, Register temp, int vec_enc);
2022 
2023   void fill64_masked(uint shift, Register dst, int disp,
2024                          XMMRegister xmm, KRegister mask, Register length,
2025                          Register temp, bool use64byteVector = false);
2026 
2027   void fill32_masked(uint shift, Register dst, int disp,
2028                          XMMRegister xmm, KRegister mask, Register length,
2029                          Register temp);
2030 
2031   void fill32(Address dst, XMMRegister xmm);
2032 
2033   void fill32(Register dst, int disp, XMMRegister xmm);
2034 
2035   void fill64(Address dst, XMMRegister xmm, bool use64byteVector = false);
2036 
2037   void fill64(Register dst, int dis, XMMRegister xmm, bool use64byteVector = false);
2038 
2039 #ifdef _LP64
2040   void convert_f2i(Register dst, XMMRegister src);
2041   void convert_d2i(Register dst, XMMRegister src);
2042   void convert_f2l(Register dst, XMMRegister src);
2043   void convert_d2l(Register dst, XMMRegister src);
2044   void round_double(Register dst, XMMRegister src, Register rtmp, Register rcx);
2045   void round_float(Register dst, XMMRegister src, Register rtmp, Register rcx);
2046 
2047   void cache_wb(Address line);
2048   void cache_wbsync(bool is_pre);
2049 
2050 #ifdef COMPILER2_OR_JVMCI
2051   void generate_fill_avx3(BasicType type, Register to, Register value,
2052                           Register count, Register rtmp, XMMRegister xtmp);
2053 #endif // COMPILER2_OR_JVMCI
2054 
2055   OopMap* continuation_enter_setup(int& stack_slots);
2056   void fill_continuation_entry(Register reg_cont_obj, Register reg_flags);
2057   void continuation_enter_cleanup();
2058 #endif // _LP64
2059 
2060   void vallones(XMMRegister dst, int vector_len);
2061 
2062   void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
2063 
2064 };
2065 
2066 /**
2067  * class SkipIfEqual:
2068  *
2069  * Instantiating this class will result in assembly code being output that will
2070  * jump around any code emitted between the creation of the instance and it's
2071  * automatic destruction at the end of a scope block, depending on the value of
2072  * the flag passed to the constructor, which will be checked at run-time.
2073  */
2074 class SkipIfEqual {
2075  private:
2076   MacroAssembler* _masm;
2077   Label _label;
2078 
2079  public:
2080    SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value, Register rscratch);
2081    ~SkipIfEqual();
2082 };
2083 
2084 #endif // CPU_X86_MACROASSEMBLER_X86_HPP