1 /*
    2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert((op1 & 0x01) == 1, "should be 32bit operation");
  304   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  305   if (is8bit(imm32)) {
  306     emit_int24(op1 | 0x02,        // set sign bit
  307                op2 | encode(dst),
  308                imm32 & 0xFF);
  309   } else {
  310     emit_int16(op1, (op2 | encode(dst)));
  311     emit_int32(imm32);
  312   }
  313 }
  314 
  315 // Force generation of a 4 byte immediate value even if it fits into 8bit
  316 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  317   assert(isByte(op1) && isByte(op2), "wrong opcode");
  318   assert((op1 & 0x01) == 1, "should be 32bit operation");
  319   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  320   emit_int16(op1, (op2 | encode(dst)));
  321   emit_int32(imm32);
  322 }
  323 
  324 // immediate-to-memory forms
  325 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  326   assert((op1 & 0x01) == 1, "should be 32bit operation");
  327   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  328   if (is8bit(imm32)) {
  329     emit_int8(op1 | 0x02); // set sign bit
  330     emit_operand(rm, adr, 1);
  331     emit_int8(imm32 & 0xFF);
  332   } else {
  333     emit_int8(op1);
  334     emit_operand(rm, adr, 4);
  335     emit_int32(imm32);
  336   }
  337 }
  338 
  339 
  340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  341   assert(isByte(op1) && isByte(op2), "wrong opcode");
  342   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  343 }
  344 
  345 
  346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  347                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  348   int mod_idx = 0;
  349   // We will test if the displacement fits the compressed format and if so
  350   // apply the compression to the displacment iff the result is8bit.
  351   if (VM_Version::supports_evex() && is_evex_inst) {
  352     switch (cur_tuple_type) {
  353     case EVEX_FV:
  354       if ((cur_encoding & VEX_W) == VEX_W) {
  355         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  356       } else {
  357         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  358       }
  359       break;
  360 
  361     case EVEX_HV:
  362       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  363       break;
  364 
  365     case EVEX_FVM:
  366       break;
  367 
  368     case EVEX_T1S:
  369       switch (in_size_in_bits) {
  370       case EVEX_8bit:
  371         break;
  372 
  373       case EVEX_16bit:
  374         mod_idx = 1;
  375         break;
  376 
  377       case EVEX_32bit:
  378         mod_idx = 2;
  379         break;
  380 
  381       case EVEX_64bit:
  382         mod_idx = 3;
  383         break;
  384       }
  385       break;
  386 
  387     case EVEX_T1F:
  388     case EVEX_T2:
  389     case EVEX_T4:
  390       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  391       break;
  392 
  393     case EVEX_T8:
  394       break;
  395 
  396     case EVEX_HVM:
  397       break;
  398 
  399     case EVEX_QVM:
  400       break;
  401 
  402     case EVEX_OVM:
  403       break;
  404 
  405     case EVEX_M128:
  406       break;
  407 
  408     case EVEX_DUP:
  409       break;
  410 
  411     default:
  412       assert(0, "no valid evex tuple_table entry");
  413       break;
  414     }
  415 
  416     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  417       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  418       if ((disp % disp_factor) == 0) {
  419         int new_disp = disp / disp_factor;
  420         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  421           disp = new_disp;
  422         }
  423       } else {
  424         return false;
  425       }
  426     }
  427   }
  428   return (-0x80 <= disp && disp < 0x80);
  429 }
  430 
  431 
  432 bool Assembler::emit_compressed_disp_byte(int &disp) {
  433   int mod_idx = 0;
  434   // We will test if the displacement fits the compressed format and if so
  435   // apply the compression to the displacment iff the result is8bit.
  436   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  437     int evex_encoding = _attributes->get_evex_encoding();
  438     int tuple_type = _attributes->get_tuple_type();
  439     switch (tuple_type) {
  440     case EVEX_FV:
  441       if ((evex_encoding & VEX_W) == VEX_W) {
  442         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  443       } else {
  444         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  445       }
  446       break;
  447 
  448     case EVEX_HV:
  449       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  450       break;
  451 
  452     case EVEX_FVM:
  453       break;
  454 
  455     case EVEX_T1S:
  456       switch (_attributes->get_input_size()) {
  457       case EVEX_8bit:
  458         break;
  459 
  460       case EVEX_16bit:
  461         mod_idx = 1;
  462         break;
  463 
  464       case EVEX_32bit:
  465         mod_idx = 2;
  466         break;
  467 
  468       case EVEX_64bit:
  469         mod_idx = 3;
  470         break;
  471       }
  472       break;
  473 
  474     case EVEX_T1F:
  475     case EVEX_T2:
  476     case EVEX_T4:
  477       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  478       break;
  479 
  480     case EVEX_T8:
  481       break;
  482 
  483     case EVEX_HVM:
  484       break;
  485 
  486     case EVEX_QVM:
  487       break;
  488 
  489     case EVEX_OVM:
  490       break;
  491 
  492     case EVEX_M128:
  493       break;
  494 
  495     case EVEX_DUP:
  496       break;
  497 
  498     default:
  499       assert(0, "no valid evex tuple_table entry");
  500       break;
  501     }
  502 
  503     int vector_len = _attributes->get_vector_len();
  504     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  505       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  506       if ((disp % disp_factor) == 0) {
  507         int new_disp = disp / disp_factor;
  508         if (is8bit(new_disp)) {
  509           disp = new_disp;
  510         }
  511       } else {
  512         return false;
  513       }
  514     }
  515   }
  516   return is8bit(disp);
  517 }
  518 
  519 static bool is_valid_encoding(int reg_enc) {
  520   return reg_enc >= 0;
  521 }
  522 
  523 static int raw_encode(Register reg) {
  524   assert(reg == noreg || reg->is_valid(), "sanity");
  525   int reg_enc = (intptr_t)reg;
  526   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  527   return reg_enc;
  528 }
  529 
  530 static int raw_encode(XMMRegister xmmreg) {
  531   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  532   int xmmreg_enc = (intptr_t)xmmreg;
  533   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  534   return xmmreg_enc;
  535 }
  536 
  537 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  538   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  539 }
  540 
  541 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  542   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  543 }
  544 
  545 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  546   assert((mod & 3) != 0b11, "forbidden");
  547   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  548   emit_int8(modrm);
  549 }
  550 
  551 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  552                                         int disp) {
  553   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  554   emit_int16(modrm, disp & 0xFF);
  555 }
  556 
  557 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  558                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   int sib = sib_encoding(scale, index_enc, base_enc);
  561   emit_int16(modrm, sib);
  562 }
  563 
  564 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  565                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  566                                             int disp) {
  567   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  568   int sib = sib_encoding(scale, index_enc, base_enc);
  569   emit_int24(modrm, sib, disp & 0xFF);
  570 }
  571 
  572 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  573                                     Address::ScaleFactor scale, int disp,
  574                                     RelocationHolder const& rspec,
  575                                     int rip_relative_correction) {
  576   bool no_relocation = (rspec.type() == relocInfo::none);
  577 
  578   if (is_valid_encoding(base_enc)) {
  579     if (is_valid_encoding(index_enc)) {
  580       assert(scale != Address::no_scale, "inconsistent address");
  581       // [base + index*scale + disp]
  582       if (disp == 0 && no_relocation &&
  583           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  584         // [base + index*scale]
  585         // [00 reg 100][ss index base]
  586         emit_modrm_sib(0b00, reg_enc, 0b100,
  587                        scale, index_enc, base_enc);
  588       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  589         // [base + index*scale + imm8]
  590         // [01 reg 100][ss index base] imm8
  591         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  592                              scale, index_enc, base_enc,
  593                              disp);
  594       } else {
  595         // [base + index*scale + disp32]
  596         // [10 reg 100][ss index base] disp32
  597         emit_modrm_sib(0b10, reg_enc, 0b100,
  598                        scale, index_enc, base_enc);
  599         emit_data(disp, rspec, disp32_operand);
  600       }
  601     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  602       // [rsp + disp]
  603       if (disp == 0 && no_relocation) {
  604         // [rsp]
  605         // [00 reg 100][00 100 100]
  606         emit_modrm_sib(0b00, reg_enc, 0b100,
  607                        Address::times_1, 0b100, 0b100);
  608       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  609         // [rsp + imm8]
  610         // [01 reg 100][00 100 100] disp8
  611         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  612                              Address::times_1, 0b100, 0b100,
  613                              disp);
  614       } else {
  615         // [rsp + imm32]
  616         // [10 reg 100][00 100 100] disp32
  617         emit_modrm_sib(0b10, reg_enc, 0b100,
  618                        Address::times_1, 0b100, 0b100);
  619         emit_data(disp, rspec, disp32_operand);
  620       }
  621     } else {
  622       // [base + disp]
  623       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  624       if (disp == 0 && no_relocation &&
  625           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  626         // [base]
  627         // [00 reg base]
  628         emit_modrm(0, reg_enc, base_enc);
  629       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  630         // [base + disp8]
  631         // [01 reg base] disp8
  632         emit_modrm_disp8(0b01, reg_enc, base_enc,
  633                          disp);
  634       } else {
  635         // [base + disp32]
  636         // [10 reg base] disp32
  637         emit_modrm(0b10, reg_enc, base_enc);
  638         emit_data(disp, rspec, disp32_operand);
  639       }
  640     }
  641   } else {
  642     if (is_valid_encoding(index_enc)) {
  643       assert(scale != Address::no_scale, "inconsistent address");
  644       // base == noreg
  645       // [index*scale + disp]
  646       // [00 reg 100][ss index 101] disp32
  647       emit_modrm_sib(0b00, reg_enc, 0b100,
  648                      scale, index_enc, 0b101 /* no base */);
  649       emit_data(disp, rspec, disp32_operand);
  650     } else if (!no_relocation) {
  651       // base == noreg, index == noreg
  652       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  653       // [00 reg 101] disp32
  654 
  655       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  656       // Note that the RIP-rel. correction applies to the generated
  657       // disp field, but _not_ to the target address in the rspec.
  658 
  659       // disp was created by converting the target address minus the pc
  660       // at the start of the instruction. That needs more correction here.
  661       // intptr_t disp = target - next_ip;
  662       assert(inst_mark() != NULL, "must be inside InstructionMark");
  663       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  664       int64_t adjusted = disp;
  665       // Do rip-rel adjustment for 64bit
  666       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  667       assert(is_simm32(adjusted),
  668              "must be 32bit offset (RIP relative address)");
  669       emit_data((int32_t) adjusted, rspec, disp32_operand);
  670 
  671     } else {
  672       // base == noreg, index == noreg, no_relocation == true
  673       // 32bit never did this, did everything as the rip-rel/disp code above
  674       // [disp] ABSOLUTE
  675       // [00 reg 100][00 100 101] disp32
  676       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  677                      Address::times_1, 0b100, 0b101);
  678       emit_data(disp, rspec, disp32_operand);
  679     }
  680   }
  681 }
  682 
  683 void Assembler::emit_operand(Register reg, Register base, Register index,
  684                              Address::ScaleFactor scale, int disp,
  685                              RelocationHolder const& rspec,
  686                              int rip_relative_correction) {
  687   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  688   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  689                       scale, disp, rspec, rip_relative_correction);
  690 
  691 }
  692 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  693                              Address::ScaleFactor scale, int disp,
  694                              RelocationHolder const& rspec) {
  695   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  696   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  697   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  698                       scale, disp, rspec);
  699 }
  700 
  701 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  702                              Address::ScaleFactor scale, int disp,
  703                              RelocationHolder const& rspec) {
  704   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  705   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  706   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  707                       scale, disp, rspec, /* rip_relative_correction */ 0);
  708 }
  709 
  710 // Secret local extension to Assembler::WhichOperand:
  711 #define end_pc_operand (_WhichOperand_limit)
  712 
  713 address Assembler::locate_operand(address inst, WhichOperand which) {
  714   // Decode the given instruction, and return the address of
  715   // an embedded 32-bit operand word.
  716 
  717   // If "which" is disp32_operand, selects the displacement portion
  718   // of an effective address specifier.
  719   // If "which" is imm64_operand, selects the trailing immediate constant.
  720   // If "which" is call32_operand, selects the displacement of a call or jump.
  721   // Caller is responsible for ensuring that there is such an operand,
  722   // and that it is 32/64 bits wide.
  723 
  724   // If "which" is end_pc_operand, find the end of the instruction.
  725 
  726   address ip = inst;
  727   bool is_64bit = false;
  728 
  729   debug_only(bool has_disp32 = false);
  730   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  731 
  732   again_after_prefix:
  733   switch (0xFF & *ip++) {
  734 
  735   // These convenience macros generate groups of "case" labels for the switch.
  736 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  737 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  738              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  739 #define REP16(x) REP8((x)+0): \
  740               case REP8((x)+8)
  741 
  742   case CS_segment:
  743   case SS_segment:
  744   case DS_segment:
  745   case ES_segment:
  746   case FS_segment:
  747   case GS_segment:
  748     // Seems dubious
  749     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  750     assert(ip == inst+1, "only one prefix allowed");
  751     goto again_after_prefix;
  752 
  753   case 0x67:
  754   case REX:
  755   case REX_B:
  756   case REX_X:
  757   case REX_XB:
  758   case REX_R:
  759   case REX_RB:
  760   case REX_RX:
  761   case REX_RXB:
  762     NOT_LP64(assert(false, "64bit prefixes"));
  763     goto again_after_prefix;
  764 
  765   case REX_W:
  766   case REX_WB:
  767   case REX_WX:
  768   case REX_WXB:
  769   case REX_WR:
  770   case REX_WRB:
  771   case REX_WRX:
  772   case REX_WRXB:
  773     NOT_LP64(assert(false, "64bit prefixes"));
  774     is_64bit = true;
  775     goto again_after_prefix;
  776 
  777   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  778   case 0x88: // movb a, r
  779   case 0x89: // movl a, r
  780   case 0x8A: // movb r, a
  781   case 0x8B: // movl r, a
  782   case 0x8F: // popl a
  783     debug_only(has_disp32 = true);
  784     break;
  785 
  786   case 0x68: // pushq #32
  787     if (which == end_pc_operand) {
  788       return ip + 4;
  789     }
  790     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  791     return ip;                  // not produced by emit_operand
  792 
  793   case 0x66: // movw ... (size prefix)
  794     again_after_size_prefix2:
  795     switch (0xFF & *ip++) {
  796     case REX:
  797     case REX_B:
  798     case REX_X:
  799     case REX_XB:
  800     case REX_R:
  801     case REX_RB:
  802     case REX_RX:
  803     case REX_RXB:
  804     case REX_W:
  805     case REX_WB:
  806     case REX_WX:
  807     case REX_WXB:
  808     case REX_WR:
  809     case REX_WRB:
  810     case REX_WRX:
  811     case REX_WRXB:
  812       NOT_LP64(assert(false, "64bit prefix found"));
  813       goto again_after_size_prefix2;
  814     case 0x8B: // movw r, a
  815     case 0x89: // movw a, r
  816       debug_only(has_disp32 = true);
  817       break;
  818     case 0xC7: // movw a, #16
  819       debug_only(has_disp32 = true);
  820       tail_size = 2;  // the imm16
  821       break;
  822     case 0x0F: // several SSE/SSE2 variants
  823       ip--;    // reparse the 0x0F
  824       goto again_after_prefix;
  825     default:
  826       ShouldNotReachHere();
  827     }
  828     break;
  829 
  830   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  831     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  832     // these asserts are somewhat nonsensical
  833 #ifndef _LP64
  834     assert(which == imm_operand || which == disp32_operand,
  835            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  836 #else
  837     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  838            which == narrow_oop_operand && !is_64bit,
  839            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  840 #endif // _LP64
  841     return ip;
  842 
  843   case 0x69: // imul r, a, #32
  844   case 0xC7: // movl a, #32(oop?)
  845     tail_size = 4;
  846     debug_only(has_disp32 = true); // has both kinds of operands!
  847     break;
  848 
  849   case 0x0F: // movx..., etc.
  850     switch (0xFF & *ip++) {
  851     case 0x3A: // pcmpestri
  852       tail_size = 1;
  853     case 0x38: // ptest, pmovzxbw
  854       ip++; // skip opcode
  855       debug_only(has_disp32 = true); // has both kinds of operands!
  856       break;
  857 
  858     case 0x70: // pshufd r, r/a, #8
  859       debug_only(has_disp32 = true); // has both kinds of operands!
  860     case 0x73: // psrldq r, #8
  861       tail_size = 1;
  862       break;
  863 
  864     case 0x12: // movlps
  865     case 0x28: // movaps
  866     case 0x2E: // ucomiss
  867     case 0x2F: // comiss
  868     case 0x54: // andps
  869     case 0x55: // andnps
  870     case 0x56: // orps
  871     case 0x57: // xorps
  872     case 0x58: // addpd
  873     case 0x59: // mulpd
  874     case 0x6E: // movd
  875     case 0x7E: // movd
  876     case 0x6F: // movdq
  877     case 0x7F: // movdq
  878     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  879     case 0xFE: // paddd
  880       debug_only(has_disp32 = true);
  881       break;
  882 
  883     case 0xAD: // shrd r, a, %cl
  884     case 0xAF: // imul r, a
  885     case 0xBE: // movsbl r, a (movsxb)
  886     case 0xBF: // movswl r, a (movsxw)
  887     case 0xB6: // movzbl r, a (movzxb)
  888     case 0xB7: // movzwl r, a (movzxw)
  889     case REP16(0x40): // cmovl cc, r, a
  890     case 0xB0: // cmpxchgb
  891     case 0xB1: // cmpxchg
  892     case 0xC1: // xaddl
  893     case 0xC7: // cmpxchg8
  894     case REP16(0x90): // setcc a
  895       debug_only(has_disp32 = true);
  896       // fall out of the switch to decode the address
  897       break;
  898 
  899     case 0xC4: // pinsrw r, a, #8
  900       debug_only(has_disp32 = true);
  901     case 0xC5: // pextrw r, r, #8
  902       tail_size = 1;  // the imm8
  903       break;
  904 
  905     case 0xAC: // shrd r, a, #8
  906       debug_only(has_disp32 = true);
  907       tail_size = 1;  // the imm8
  908       break;
  909 
  910     case REP16(0x80): // jcc rdisp32
  911       if (which == end_pc_operand)  return ip + 4;
  912       assert(which == call32_operand, "jcc has no disp32 or imm");
  913       return ip;
  914     default:
  915       ShouldNotReachHere();
  916     }
  917     break;
  918 
  919   case 0x81: // addl a, #32; addl r, #32
  920     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  921     // on 32bit in the case of cmpl, the imm might be an oop
  922     tail_size = 4;
  923     debug_only(has_disp32 = true); // has both kinds of operands!
  924     break;
  925 
  926   case 0x83: // addl a, #8; addl r, #8
  927     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  928     debug_only(has_disp32 = true); // has both kinds of operands!
  929     tail_size = 1;
  930     break;
  931 
  932   case 0x9B:
  933     switch (0xFF & *ip++) {
  934     case 0xD9: // fnstcw a
  935       debug_only(has_disp32 = true);
  936       break;
  937     default:
  938       ShouldNotReachHere();
  939     }
  940     break;
  941 
  942   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  943   case REP4(0x10): // adc...
  944   case REP4(0x20): // and...
  945   case REP4(0x30): // xor...
  946   case REP4(0x08): // or...
  947   case REP4(0x18): // sbb...
  948   case REP4(0x28): // sub...
  949   case 0xF7: // mull a
  950   case 0x8D: // lea r, a
  951   case 0x87: // xchg r, a
  952   case REP4(0x38): // cmp...
  953   case 0x85: // test r, a
  954     debug_only(has_disp32 = true); // has both kinds of operands!
  955     break;
  956 
  957   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  958   case 0xC6: // movb a, #8
  959   case 0x80: // cmpb a, #8
  960   case 0x6B: // imul r, a, #8
  961     debug_only(has_disp32 = true); // has both kinds of operands!
  962     tail_size = 1; // the imm8
  963     break;
  964 
  965   case 0xC4: // VEX_3bytes
  966   case 0xC5: // VEX_2bytes
  967     assert((UseAVX > 0), "shouldn't have VEX prefix");
  968     assert(ip == inst+1, "no prefixes allowed");
  969     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  970     // but they have prefix 0x0F and processed when 0x0F processed above.
  971     //
  972     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
  973     // instructions (these instructions are not supported in 64-bit mode).
  974     // To distinguish them bits [7:6] are set in the VEX second byte since
  975     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
  976     // those VEX bits REX and vvvv bits are inverted.
  977     //
  978     // Fortunately C2 doesn't generate these instructions so we don't need
  979     // to check for them in product version.
  980 
  981     // Check second byte
  982     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
  983 
  984     int vex_opcode;
  985     // First byte
  986     if ((0xFF & *inst) == VEX_3bytes) {
  987       vex_opcode = VEX_OPCODE_MASK & *ip;
  988       ip++; // third byte
  989       is_64bit = ((VEX_W & *ip) == VEX_W);
  990     } else {
  991       vex_opcode = VEX_OPCODE_0F;
  992     }
  993     ip++; // opcode
  994     // To find the end of instruction (which == end_pc_operand).
  995     switch (vex_opcode) {
  996       case VEX_OPCODE_0F:
  997         switch (0xFF & *ip) {
  998         case 0x70: // pshufd r, r/a, #8
  999         case 0x71: // ps[rl|ra|ll]w r, #8
 1000         case 0x72: // ps[rl|ra|ll]d r, #8
 1001         case 0x73: // ps[rl|ra|ll]q r, #8
 1002         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1003         case 0xC4: // pinsrw r, r, r/a, #8
 1004         case 0xC5: // pextrw r/a, r, #8
 1005         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1006           tail_size = 1;  // the imm8
 1007           break;
 1008         }
 1009         break;
 1010       case VEX_OPCODE_0F_3A:
 1011         tail_size = 1;
 1012         break;
 1013     }
 1014     ip++; // skip opcode
 1015     debug_only(has_disp32 = true); // has both kinds of operands!
 1016     break;
 1017 
 1018   case 0x62: // EVEX_4bytes
 1019     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1020     assert(ip == inst+1, "no prefixes allowed");
 1021     // no EVEX collisions, all instructions that have 0x62 opcodes
 1022     // have EVEX versions and are subopcodes of 0x66
 1023     ip++; // skip P0 and exmaine W in P1
 1024     is_64bit = ((VEX_W & *ip) == VEX_W);
 1025     ip++; // move to P2
 1026     ip++; // skip P2, move to opcode
 1027     // To find the end of instruction (which == end_pc_operand).
 1028     switch (0xFF & *ip) {
 1029     case 0x22: // pinsrd r, r/a, #8
 1030     case 0x61: // pcmpestri r, r/a, #8
 1031     case 0x70: // pshufd r, r/a, #8
 1032     case 0x73: // psrldq r, #8
 1033     case 0x1f: // evpcmpd/evpcmpq
 1034     case 0x3f: // evpcmpb/evpcmpw
 1035       tail_size = 1;  // the imm8
 1036       break;
 1037     default:
 1038       break;
 1039     }
 1040     ip++; // skip opcode
 1041     debug_only(has_disp32 = true); // has both kinds of operands!
 1042     break;
 1043 
 1044   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1045   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1046   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1047   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1048   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1049   case 0xDF: // fild_d a; fistp_d a
 1050   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1051   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1052   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1053     debug_only(has_disp32 = true);
 1054     break;
 1055 
 1056   case 0xE8: // call rdisp32
 1057   case 0xE9: // jmp  rdisp32
 1058     if (which == end_pc_operand)  return ip + 4;
 1059     assert(which == call32_operand, "call has no disp32 or imm");
 1060     return ip;
 1061 
 1062   case 0xF0:                    // Lock
 1063     goto again_after_prefix;
 1064 
 1065   case 0xF3:                    // For SSE
 1066   case 0xF2:                    // For SSE2
 1067     switch (0xFF & *ip++) {
 1068     case REX:
 1069     case REX_B:
 1070     case REX_X:
 1071     case REX_XB:
 1072     case REX_R:
 1073     case REX_RB:
 1074     case REX_RX:
 1075     case REX_RXB:
 1076     case REX_W:
 1077     case REX_WB:
 1078     case REX_WX:
 1079     case REX_WXB:
 1080     case REX_WR:
 1081     case REX_WRB:
 1082     case REX_WRX:
 1083     case REX_WRXB:
 1084       NOT_LP64(assert(false, "found 64bit prefix"));
 1085       ip++;
 1086     default:
 1087       ip++;
 1088     }
 1089     debug_only(has_disp32 = true); // has both kinds of operands!
 1090     break;
 1091 
 1092   default:
 1093     ShouldNotReachHere();
 1094 
 1095 #undef REP8
 1096 #undef REP16
 1097   }
 1098 
 1099   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1100 #ifdef _LP64
 1101   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1102 #else
 1103   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1104   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1105 #endif // LP64
 1106   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1107 
 1108   // parse the output of emit_operand
 1109   int op2 = 0xFF & *ip++;
 1110   int base = op2 & 0x07;
 1111   int op3 = -1;
 1112   const int b100 = 4;
 1113   const int b101 = 5;
 1114   if (base == b100 && (op2 >> 6) != 3) {
 1115     op3 = 0xFF & *ip++;
 1116     base = op3 & 0x07;   // refetch the base
 1117   }
 1118   // now ip points at the disp (if any)
 1119 
 1120   switch (op2 >> 6) {
 1121   case 0:
 1122     // [00 reg  100][ss index base]
 1123     // [00 reg  100][00   100  esp]
 1124     // [00 reg base]
 1125     // [00 reg  100][ss index  101][disp32]
 1126     // [00 reg  101]               [disp32]
 1127 
 1128     if (base == b101) {
 1129       if (which == disp32_operand)
 1130         return ip;              // caller wants the disp32
 1131       ip += 4;                  // skip the disp32
 1132     }
 1133     break;
 1134 
 1135   case 1:
 1136     // [01 reg  100][ss index base][disp8]
 1137     // [01 reg  100][00   100  esp][disp8]
 1138     // [01 reg base]               [disp8]
 1139     ip += 1;                    // skip the disp8
 1140     break;
 1141 
 1142   case 2:
 1143     // [10 reg  100][ss index base][disp32]
 1144     // [10 reg  100][00   100  esp][disp32]
 1145     // [10 reg base]               [disp32]
 1146     if (which == disp32_operand)
 1147       return ip;                // caller wants the disp32
 1148     ip += 4;                    // skip the disp32
 1149     break;
 1150 
 1151   case 3:
 1152     // [11 reg base]  (not a memory addressing mode)
 1153     break;
 1154   }
 1155 
 1156   if (which == end_pc_operand) {
 1157     return ip + tail_size;
 1158   }
 1159 
 1160 #ifdef _LP64
 1161   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1162 #else
 1163   assert(which == imm_operand, "instruction has only an imm field");
 1164 #endif // LP64
 1165   return ip;
 1166 }
 1167 
 1168 address Assembler::locate_next_instruction(address inst) {
 1169   // Secretly share code with locate_operand:
 1170   return locate_operand(inst, end_pc_operand);
 1171 }
 1172 
 1173 
 1174 #ifdef ASSERT
 1175 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1176   address inst = inst_mark();
 1177   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1178   address opnd;
 1179 
 1180   Relocation* r = rspec.reloc();
 1181   if (r->type() == relocInfo::none) {
 1182     return;
 1183   } else if (r->is_call() || format == call32_operand) {
 1184     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1185     opnd = locate_operand(inst, call32_operand);
 1186   } else if (r->is_data()) {
 1187     assert(format == imm_operand || format == disp32_operand
 1188            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1189     opnd = locate_operand(inst, (WhichOperand)format);
 1190   } else {
 1191     assert(format == imm_operand, "cannot specify a format");
 1192     return;
 1193   }
 1194   assert(opnd == pc(), "must put operand where relocs can find it");
 1195 }
 1196 #endif // ASSERT
 1197 
 1198 void Assembler::emit_operand(Register reg, Address adr,
 1199                              int rip_relative_correction) {
 1200   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1201                adr._rspec,
 1202                rip_relative_correction);
 1203 }
 1204 
 1205 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1206     if (adr.isxmmindex()) {
 1207        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1208     } else {
 1209        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1210        adr._rspec);
 1211     }
 1212 }
 1213 
 1214 // Now the Assembler instructions (identical for 32/64 bits)
 1215 
 1216 void Assembler::adcl(Address dst, int32_t imm32) {
 1217   InstructionMark im(this);
 1218   prefix(dst);
 1219   emit_arith_operand(0x81, rdx, dst, imm32);
 1220 }
 1221 
 1222 void Assembler::adcl(Address dst, Register src) {
 1223   InstructionMark im(this);
 1224   prefix(dst, src);
 1225   emit_int8(0x11);
 1226   emit_operand(src, dst);
 1227 }
 1228 
 1229 void Assembler::adcl(Register dst, int32_t imm32) {
 1230   prefix(dst);
 1231   emit_arith(0x81, 0xD0, dst, imm32);
 1232 }
 1233 
 1234 void Assembler::adcl(Register dst, Address src) {
 1235   InstructionMark im(this);
 1236   prefix(src, dst);
 1237   emit_int8(0x13);
 1238   emit_operand(dst, src);
 1239 }
 1240 
 1241 void Assembler::adcl(Register dst, Register src) {
 1242   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1243   emit_arith(0x13, 0xC0, dst, src);
 1244 }
 1245 
 1246 void Assembler::addl(Address dst, int32_t imm32) {
 1247   InstructionMark im(this);
 1248   prefix(dst);
 1249   emit_arith_operand(0x81, rax, dst, imm32);
 1250 }
 1251 
 1252 void Assembler::addb(Address dst, int imm8) {
 1253   InstructionMark im(this);
 1254   prefix(dst);
 1255   emit_int8((unsigned char)0x80);
 1256   emit_operand(rax, dst, 1);
 1257   emit_int8(imm8);
 1258 }
 1259 
 1260 void Assembler::addw(Register dst, Register src) {
 1261   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1262   emit_arith(0x03, 0xC0, dst, src);
 1263 }
 1264 
 1265 void Assembler::addw(Address dst, int imm16) {
 1266   InstructionMark im(this);
 1267   emit_int8(0x66);
 1268   prefix(dst);
 1269   emit_int8((unsigned char)0x81);
 1270   emit_operand(rax, dst, 2);
 1271   emit_int16(imm16);
 1272 }
 1273 
 1274 void Assembler::addl(Address dst, Register src) {
 1275   InstructionMark im(this);
 1276   prefix(dst, src);
 1277   emit_int8(0x01);
 1278   emit_operand(src, dst);
 1279 }
 1280 
 1281 void Assembler::addl(Register dst, int32_t imm32) {
 1282   prefix(dst);
 1283   emit_arith(0x81, 0xC0, dst, imm32);
 1284 }
 1285 
 1286 void Assembler::addl(Register dst, Address src) {
 1287   InstructionMark im(this);
 1288   prefix(src, dst);
 1289   emit_int8(0x03);
 1290   emit_operand(dst, src);
 1291 }
 1292 
 1293 void Assembler::addl(Register dst, Register src) {
 1294   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1295   emit_arith(0x03, 0xC0, dst, src);
 1296 }
 1297 
 1298 void Assembler::addr_nop_4() {
 1299   assert(UseAddressNop, "no CPU support");
 1300   // 4 bytes: NOP DWORD PTR [EAX+0]
 1301   emit_int32(0x0F,
 1302              0x1F,
 1303              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1304              0);   // 8-bits offset (1 byte)
 1305 }
 1306 
 1307 void Assembler::addr_nop_5() {
 1308   assert(UseAddressNop, "no CPU support");
 1309   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1310   emit_int32(0x0F,
 1311              0x1F,
 1312              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1313              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1314   emit_int8(0);     // 8-bits offset (1 byte)
 1315 }
 1316 
 1317 void Assembler::addr_nop_7() {
 1318   assert(UseAddressNop, "no CPU support");
 1319   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1320   emit_int24(0x0F,
 1321              0x1F,
 1322              (unsigned char)0x80);
 1323                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1324   emit_int32(0);   // 32-bits offset (4 bytes)
 1325 }
 1326 
 1327 void Assembler::addr_nop_8() {
 1328   assert(UseAddressNop, "no CPU support");
 1329   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1330   emit_int32(0x0F,
 1331              0x1F,
 1332              (unsigned char)0x84,
 1333                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1334              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1335   emit_int32(0);    // 32-bits offset (4 bytes)
 1336 }
 1337 
 1338 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1341   attributes.set_rex_vex_w_reverted();
 1342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1343   emit_int16(0x58, (0xC0 | encode));
 1344 }
 1345 
 1346 void Assembler::addsd(XMMRegister dst, Address src) {
 1347   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1348   InstructionMark im(this);
 1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1350   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1351   attributes.set_rex_vex_w_reverted();
 1352   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1353   emit_int8(0x58);
 1354   emit_operand(dst, src);
 1355 }
 1356 
 1357 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1358   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1360   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1361   emit_int16(0x58, (0xC0 | encode));
 1362 }
 1363 
 1364 void Assembler::addss(XMMRegister dst, Address src) {
 1365   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1366   InstructionMark im(this);
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1369   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1370   emit_int8(0x58);
 1371   emit_operand(dst, src);
 1372 }
 1373 
 1374 void Assembler::aesdec(XMMRegister dst, Address src) {
 1375   assert(VM_Version::supports_aes(), "");
 1376   InstructionMark im(this);
 1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1378   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1379   emit_int8((unsigned char)0xDE);
 1380   emit_operand(dst, src);
 1381 }
 1382 
 1383 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1384   assert(VM_Version::supports_aes(), "");
 1385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1386   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1387   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1388 }
 1389 
 1390 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1391   assert(VM_Version::supports_avx512_vaes(), "");
 1392   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1393   attributes.set_is_evex_instruction();
 1394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1395   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1396 }
 1397 
 1398 
 1399 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1400   assert(VM_Version::supports_aes(), "");
 1401   InstructionMark im(this);
 1402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1403   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1404   emit_int8((unsigned char)0xDF);
 1405   emit_operand(dst, src);
 1406 }
 1407 
 1408 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1409   assert(VM_Version::supports_aes(), "");
 1410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1411   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1412   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1413 }
 1414 
 1415 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1416   assert(VM_Version::supports_avx512_vaes(), "");
 1417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1418   attributes.set_is_evex_instruction();
 1419   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1420   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1421 }
 1422 
 1423 void Assembler::aesenc(XMMRegister dst, Address src) {
 1424   assert(VM_Version::supports_aes(), "");
 1425   InstructionMark im(this);
 1426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1427   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1428   emit_int8((unsigned char)0xDC);
 1429   emit_operand(dst, src);
 1430 }
 1431 
 1432 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1433   assert(VM_Version::supports_aes(), "");
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1437 }
 1438 
 1439 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1440   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1442   attributes.set_is_evex_instruction();
 1443   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1448   assert(VM_Version::supports_aes(), "");
 1449   InstructionMark im(this);
 1450   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1451   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int8((unsigned char)0xDD);
 1453   emit_operand(dst, src);
 1454 }
 1455 
 1456 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1460   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1461 }
 1462 
 1463 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1464   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1465   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1466   attributes.set_is_evex_instruction();
 1467   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1468   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1469 }
 1470 
 1471 void Assembler::andb(Address dst, Register src) {
 1472   InstructionMark im(this);
 1473   prefix(dst, src, true);
 1474   emit_int8(0x20);
 1475   emit_operand(src, dst);
 1476 }
 1477 
 1478 void Assembler::andw(Register dst, Register src) {
 1479   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1480   emit_arith(0x23, 0xC0, dst, src);
 1481 }
 1482 
 1483 void Assembler::andl(Address dst, int32_t imm32) {
 1484   InstructionMark im(this);
 1485   prefix(dst);
 1486   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1487 }
 1488 
 1489 void Assembler::andl(Register dst, int32_t imm32) {
 1490   prefix(dst);
 1491   emit_arith(0x81, 0xE0, dst, imm32);
 1492 }
 1493 
 1494 void Assembler::andl(Address dst, Register src) {
 1495   InstructionMark im(this);
 1496   prefix(dst, src);
 1497   emit_int8(0x21);
 1498   emit_operand(src, dst);
 1499 }
 1500 
 1501 void Assembler::andl(Register dst, Address src) {
 1502   InstructionMark im(this);
 1503   prefix(src, dst);
 1504   emit_int8(0x23);
 1505   emit_operand(dst, src);
 1506 }
 1507 
 1508 void Assembler::andl(Register dst, Register src) {
 1509   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1510   emit_arith(0x23, 0xC0, dst, src);
 1511 }
 1512 
 1513 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1514   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1521   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1522   InstructionMark im(this);
 1523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1524   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int8((unsigned char)0xF2);
 1526   emit_operand(dst, src2);
 1527 }
 1528 
 1529 void Assembler::bsfl(Register dst, Register src) {
 1530   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1531   emit_int24(0x0F,
 1532              (unsigned char)0xBC,
 1533              0xC0 | encode);
 1534 }
 1535 
 1536 void Assembler::bsrl(Register dst, Register src) {
 1537   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1538   emit_int24(0x0F,
 1539              (unsigned char)0xBD,
 1540              0xC0 | encode);
 1541 }
 1542 
 1543 void Assembler::bswapl(Register reg) { // bswap
 1544   int encode = prefix_and_encode(reg->encoding());
 1545   emit_int16(0x0F, (0xC8 | encode));
 1546 }
 1547 
 1548 void Assembler::blsil(Register dst, Register src) {
 1549   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1550   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1553 }
 1554 
 1555 void Assembler::blsil(Register dst, Address src) {
 1556   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1557   InstructionMark im(this);
 1558   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1559   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1560   emit_int8((unsigned char)0xF3);
 1561   emit_operand(rbx, src);
 1562 }
 1563 
 1564 void Assembler::blsmskl(Register dst, Register src) {
 1565   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1566   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1567   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1568   emit_int16((unsigned char)0xF3,
 1569              0xC0 | encode);
 1570 }
 1571 
 1572 void Assembler::blsmskl(Register dst, Address src) {
 1573   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1574   InstructionMark im(this);
 1575   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1576   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1577   emit_int8((unsigned char)0xF3);
 1578   emit_operand(rdx, src);
 1579 }
 1580 
 1581 void Assembler::blsrl(Register dst, Register src) {
 1582   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1583   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1584   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1585   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1586 }
 1587 
 1588 void Assembler::blsrl(Register dst, Address src) {
 1589   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1590   InstructionMark im(this);
 1591   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1592   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1593   emit_int8((unsigned char)0xF3);
 1594   emit_operand(rcx, src);
 1595 }
 1596 
 1597 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1598   // suspect disp32 is always good
 1599   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1600 
 1601   if (L.is_bound()) {
 1602     const int long_size = 5;
 1603     int offs = (int)( target(L) - pc() );
 1604     assert(offs <= 0, "assembler error");
 1605     InstructionMark im(this);
 1606     // 1110 1000 #32-bit disp
 1607     emit_int8((unsigned char)0xE8);
 1608     emit_data(offs - long_size, rtype, operand);
 1609   } else {
 1610     InstructionMark im(this);
 1611     // 1110 1000 #32-bit disp
 1612     L.add_patch_at(code(), locator());
 1613 
 1614     emit_int8((unsigned char)0xE8);
 1615     emit_data(int(0), rtype, operand);
 1616   }
 1617 }
 1618 
 1619 void Assembler::call(Register dst) {
 1620   int encode = prefix_and_encode(dst->encoding());
 1621   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1622 }
 1623 
 1624 
 1625 void Assembler::call(Address adr) {
 1626   InstructionMark im(this);
 1627   prefix(adr);
 1628   emit_int8((unsigned char)0xFF);
 1629   emit_operand(rdx, adr);
 1630 }
 1631 
 1632 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1633   InstructionMark im(this);
 1634   emit_int8((unsigned char)0xE8);
 1635   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1636   // Entry is NULL in case of a scratch emit.
 1637   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1638   // Technically, should use call32_operand, but this format is
 1639   // implied by the fact that we're emitting a call instruction.
 1640 
 1641   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1642   emit_data((int) disp, rspec, operand);
 1643 }
 1644 
 1645 void Assembler::cdql() {
 1646   emit_int8((unsigned char)0x99);
 1647 }
 1648 
 1649 void Assembler::cld() {
 1650   emit_int8((unsigned char)0xFC);
 1651 }
 1652 
 1653 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1654   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1655   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1656   emit_int24(0x0F,
 1657              0x40 | cc,
 1658              0xC0 | encode);
 1659 }
 1660 
 1661 
 1662 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1663   InstructionMark im(this);
 1664   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1665   prefix(src, dst);
 1666   emit_int16(0x0F, (0x40 | cc));
 1667   emit_operand(dst, src);
 1668 }
 1669 
 1670 void Assembler::cmpb(Address dst, int imm8) {
 1671   InstructionMark im(this);
 1672   prefix(dst);
 1673   emit_int8((unsigned char)0x80);
 1674   emit_operand(rdi, dst, 1);
 1675   emit_int8(imm8);
 1676 }
 1677 
 1678 void Assembler::cmpl(Address dst, int32_t imm32) {
 1679   InstructionMark im(this);
 1680   prefix(dst);
 1681   emit_int8((unsigned char)0x81);
 1682   emit_operand(rdi, dst, 4);
 1683   emit_int32(imm32);
 1684 }
 1685 
 1686 void Assembler::cmp(Register dst, int32_t imm32) {
 1687   prefix(dst);
 1688   emit_int8((unsigned char)0x3D);
 1689   emit_int32(imm32);
 1690 }
 1691 
 1692 void Assembler::cmpl(Register dst, int32_t imm32) {
 1693   prefix(dst);
 1694   emit_arith(0x81, 0xF8, dst, imm32);
 1695 }
 1696 
 1697 void Assembler::cmpl(Register dst, Register src) {
 1698   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1699   emit_arith(0x3B, 0xC0, dst, src);
 1700 }
 1701 
 1702 void Assembler::cmpl(Register dst, Address  src) {
 1703   InstructionMark im(this);
 1704   prefix(src, dst);
 1705   emit_int8(0x3B);
 1706   emit_operand(dst, src);
 1707 }
 1708 
 1709 void Assembler::cmpw(Address dst, int imm16) {
 1710   InstructionMark im(this);
 1711   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1712   emit_int16(0x66, (unsigned char)0x81);
 1713   emit_operand(rdi, dst, 2);
 1714   emit_int16(imm16);
 1715 }
 1716 
 1717 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1718 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1719 // The ZF is set if the compared values were equal, and cleared otherwise.
 1720 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1721   InstructionMark im(this);
 1722   prefix(adr, reg);
 1723   emit_int16(0x0F, (unsigned char)0xB1);
 1724   emit_operand(reg, adr);
 1725 }
 1726 
 1727 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1728   InstructionMark im(this);
 1729   size_prefix();
 1730   prefix(adr, reg);
 1731   emit_int16(0x0F, (unsigned char)0xB1);
 1732   emit_operand(reg, adr);
 1733 }
 1734 
 1735 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1736 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1737 // The ZF is set if the compared values were equal, and cleared otherwise.
 1738 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1739   InstructionMark im(this);
 1740   prefix(adr, reg, true);
 1741   emit_int16(0x0F, (unsigned char)0xB0);
 1742   emit_operand(reg, adr);
 1743 }
 1744 
 1745 void Assembler::comisd(XMMRegister dst, Address src) {
 1746   // NOTE: dbx seems to decode this as comiss even though the
 1747   // 0x66 is there. Strangly ucomisd comes out correct
 1748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1749   InstructionMark im(this);
 1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1751   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1752   attributes.set_rex_vex_w_reverted();
 1753   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1754   emit_int8(0x2F);
 1755   emit_operand(dst, src);
 1756 }
 1757 
 1758 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1760   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1761   attributes.set_rex_vex_w_reverted();
 1762   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1763   emit_int16(0x2F, (0xC0 | encode));
 1764 }
 1765 
 1766 void Assembler::comiss(XMMRegister dst, Address src) {
 1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1768   InstructionMark im(this);
 1769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1770   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1771   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1772   emit_int8(0x2F);
 1773   emit_operand(dst, src);
 1774 }
 1775 
 1776 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1777   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1779   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1780   emit_int16(0x2F, (0xC0 | encode));
 1781 }
 1782 
 1783 void Assembler::cpuid() {
 1784   emit_int16(0x0F, (unsigned char)0xA2);
 1785 }
 1786 
 1787 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1788 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1789 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1790 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1791 //
 1792 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1793 //
 1794 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1795 //
 1796 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1797 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1798   assert(VM_Version::supports_sse4_2(), "");
 1799   int8_t w = 0x01;
 1800   Prefix p = Prefix_EMPTY;
 1801 
 1802   emit_int8((unsigned char)0xF2);
 1803   switch (sizeInBytes) {
 1804   case 1:
 1805     w = 0;
 1806     break;
 1807   case 2:
 1808   case 4:
 1809     break;
 1810   LP64_ONLY(case 8:)
 1811     // This instruction is not valid in 32 bits
 1812     // Note:
 1813     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1814     //
 1815     // Page B - 72   Vol. 2C says
 1816     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1817     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1818     //                                                                            F0!!!
 1819     // while 3 - 208 Vol. 2A
 1820     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1821     //
 1822     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1823     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1824     p = REX_W;
 1825     break;
 1826   default:
 1827     assert(0, "Unsupported value for a sizeInBytes argument");
 1828     break;
 1829   }
 1830   LP64_ONLY(prefix(crc, v, p);)
 1831   emit_int32(0x0F,
 1832              0x38,
 1833              0xF0 | w,
 1834              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1835 }
 1836 
 1837 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1838   assert(VM_Version::supports_sse4_2(), "");
 1839   InstructionMark im(this);
 1840   int8_t w = 0x01;
 1841   Prefix p = Prefix_EMPTY;
 1842 
 1843   emit_int8((int8_t)0xF2);
 1844   switch (sizeInBytes) {
 1845   case 1:
 1846     w = 0;
 1847     break;
 1848   case 2:
 1849   case 4:
 1850     break;
 1851   LP64_ONLY(case 8:)
 1852     // This instruction is not valid in 32 bits
 1853     p = REX_W;
 1854     break;
 1855   default:
 1856     assert(0, "Unsupported value for a sizeInBytes argument");
 1857     break;
 1858   }
 1859   LP64_ONLY(prefix(crc, adr, p);)
 1860   emit_int24(0x0F, 0x38, (0xF0 | w));
 1861   emit_operand(crc, adr);
 1862 }
 1863 
 1864 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1868   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1869 }
 1870 
 1871 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1872   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1875   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1876 }
 1877 
 1878 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1882   emit_int16(0x5B, (0xC0 | encode));
 1883 }
 1884 
 1885 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1886   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1887   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1889   emit_int16(0x5B, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1893   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1894   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1895   attributes.set_rex_vex_w_reverted();
 1896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1897   emit_int16(0x5A, (0xC0 | encode));
 1898 }
 1899 
 1900 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1901   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1902   InstructionMark im(this);
 1903   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1904   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1905   attributes.set_rex_vex_w_reverted();
 1906   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1907   emit_int8(0x5A);
 1908   emit_operand(dst, src);
 1909 }
 1910 
 1911 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1914   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1915   emit_int16(0x2A, (0xC0 | encode));
 1916 }
 1917 
 1918 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1919   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1920   InstructionMark im(this);
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1922   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1923   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1924   emit_int8(0x2A);
 1925   emit_operand(dst, src);
 1926 }
 1927 
 1928 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1929   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1931   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1932   emit_int16(0x2A, (0xC0 | encode));
 1933 }
 1934 
 1935 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1937   InstructionMark im(this);
 1938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1939   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1940   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1941   emit_int8(0x2A);
 1942   emit_operand(dst, src);
 1943 }
 1944 
 1945 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1946   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1947   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1948   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1949   emit_int16(0x2A, (0xC0 | encode));
 1950 }
 1951 
 1952 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1953   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1955   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1956   emit_int16(0x5A, (0xC0 | encode));
 1957 }
 1958 
 1959 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1961   InstructionMark im(this);
 1962   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1963   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1964   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1965   emit_int8(0x5A);
 1966   emit_operand(dst, src);
 1967 }
 1968 
 1969 
 1970 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1971   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1973   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1974   emit_int16(0x2C, (0xC0 | encode));
 1975 }
 1976 
 1977 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 1978   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1980   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1981   emit_int16(0x2C, (0xC0 | encode));
 1982 }
 1983 
 1984 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 1985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1986   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 1987   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1988   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1989   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1990 }
 1991 
 1992 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 1993   assert(VM_Version::supports_ssse3(), "");
 1994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 1995   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1996   emit_int16(0x1C, (0xC0 | encode));
 1997 }
 1998 
 1999 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2000   assert(VM_Version::supports_ssse3(), "");
 2001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2002   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2003   emit_int16(0x1D, (0xC0 | encode));
 2004 }
 2005 
 2006 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2007   assert(VM_Version::supports_ssse3(), "");
 2008   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2009   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2010   emit_int16(0x1E, (0xC0 | encode));
 2011 }
 2012 
 2013 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2014   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2015          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2016          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2017   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2018   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2019   emit_int16(0x1C, (0xC0 | encode));
 2020 }
 2021 
 2022 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2023   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2024          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2025          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2026   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2027   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2028   emit_int16(0x1D, (0xC0 | encode));
 2029 }
 2030 
 2031 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2032   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2033   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2034   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2035   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2037   emit_int16(0x1E, (0xC0 | encode));
 2038 }
 2039 
 2040 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2041   assert(UseAVX > 2, "");
 2042   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2043   attributes.set_is_evex_instruction();
 2044   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2045   emit_int16(0x1F, (0xC0 | encode));
 2046 }
 2047 
 2048 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2049   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2050   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2052   emit_int16(0x5A, (0xC0 | encode));
 2053 }
 2054 
 2055 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2056   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2057   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2058   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2059   attributes.set_rex_vex_w_reverted();
 2060   emit_int16(0x5A, (0xC0 | encode));
 2061 }
 2062 
 2063 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2064   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2065   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2066   attributes.set_is_evex_instruction();
 2067   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2068   emit_int16(0x5B, (0xC0 | encode));
 2069 }
 2070 
 2071 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2072   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2073   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2074   attributes.set_is_evex_instruction();
 2075   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2076   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2077 }
 2078 
 2079 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2080   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2082   attributes.set_is_evex_instruction();
 2083   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2084   emit_int16(0x30, (0xC0 | encode));
 2085 }
 2086 
 2087 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2088   assert(UseAVX > 2, "");
 2089   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2090   attributes.set_is_evex_instruction();
 2091   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2092   emit_int16(0x33, (0xC0 | encode));
 2093 }
 2094 
 2095 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2096   assert(UseAVX > 2, "");
 2097   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2098   attributes.set_is_evex_instruction();
 2099   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x31, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(UseAVX > 2, "");
 2105   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2106   attributes.set_is_evex_instruction();
 2107   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2108   emit_int16(0x35, (0xC0 | encode));
 2109 }
 2110 
 2111 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2112   assert(UseAVX > 2, "");
 2113   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2114   attributes.set_is_evex_instruction();
 2115   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2116   emit_int16(0x32, (0xC0 | encode));
 2117 }
 2118 
 2119 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2120   assert(UseAVX > 2, "");
 2121   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2122   attributes.set_is_evex_instruction();
 2123   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2124   emit_int16(0x34, (0xC0 | encode));
 2125 }
 2126 
 2127 void Assembler::decl(Address dst) {
 2128   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2129   InstructionMark im(this);
 2130   prefix(dst);
 2131   emit_int8((unsigned char)0xFF);
 2132   emit_operand(rcx, dst);
 2133 }
 2134 
 2135 void Assembler::divsd(XMMRegister dst, Address src) {
 2136   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2137   InstructionMark im(this);
 2138   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2139   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2140   attributes.set_rex_vex_w_reverted();
 2141   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2142   emit_int8(0x5E);
 2143   emit_operand(dst, src);
 2144 }
 2145 
 2146 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2147   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2148   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2149   attributes.set_rex_vex_w_reverted();
 2150   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2151   emit_int16(0x5E, (0xC0 | encode));
 2152 }
 2153 
 2154 void Assembler::divss(XMMRegister dst, Address src) {
 2155   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2156   InstructionMark im(this);
 2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2158   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2159   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2160   emit_int8(0x5E);
 2161   emit_operand(dst, src);
 2162 }
 2163 
 2164 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2168   emit_int16(0x5E, (0xC0 | encode));
 2169 }
 2170 
 2171 void Assembler::hlt() {
 2172   emit_int8((unsigned char)0xF4);
 2173 }
 2174 
 2175 void Assembler::idivl(Register src) {
 2176   int encode = prefix_and_encode(src->encoding());
 2177   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2178 }
 2179 
 2180 void Assembler::divl(Register src) { // Unsigned
 2181   int encode = prefix_and_encode(src->encoding());
 2182   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2183 }
 2184 
 2185 void Assembler::imull(Register src) {
 2186   int encode = prefix_and_encode(src->encoding());
 2187   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2188 }
 2189 
 2190 void Assembler::imull(Register dst, Register src) {
 2191   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2192   emit_int24(0x0F,
 2193              (unsigned char)0xAF,
 2194              (0xC0 | encode));
 2195 }
 2196 
 2197 void Assembler::imull(Register dst, Address src, int32_t value) {
 2198   InstructionMark im(this);
 2199   prefix(src, dst);
 2200   if (is8bit(value)) {
 2201     emit_int8((unsigned char)0x6B);
 2202     emit_operand(dst, src);
 2203     emit_int8(value);
 2204   } else {
 2205     emit_int8((unsigned char)0x69);
 2206     emit_operand(dst, src);
 2207     emit_int32(value);
 2208   }
 2209 }
 2210 
 2211 void Assembler::imull(Register dst, Register src, int value) {
 2212   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2213   if (is8bit(value)) {
 2214     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2215   } else {
 2216     emit_int16(0x69, (0xC0 | encode));
 2217     emit_int32(value);
 2218   }
 2219 }
 2220 
 2221 void Assembler::imull(Register dst, Address src) {
 2222   InstructionMark im(this);
 2223   prefix(src, dst);
 2224   emit_int16(0x0F, (unsigned char)0xAF);
 2225   emit_operand(dst, src);
 2226 }
 2227 
 2228 
 2229 void Assembler::incl(Address dst) {
 2230   // Don't use it directly. Use MacroAssembler::increment() instead.
 2231   InstructionMark im(this);
 2232   prefix(dst);
 2233   emit_int8((unsigned char)0xFF);
 2234   emit_operand(rax, dst);
 2235 }
 2236 
 2237 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2238   InstructionMark im(this);
 2239   assert((0 <= cc) && (cc < 16), "illegal cc");
 2240   if (L.is_bound()) {
 2241     address dst = target(L);
 2242     assert(dst != NULL, "jcc most probably wrong");
 2243 
 2244     const int short_size = 2;
 2245     const int long_size = 6;
 2246     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2247     if (maybe_short && is8bit(offs - short_size)) {
 2248       // 0111 tttn #8-bit disp
 2249       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2250     } else {
 2251       // 0000 1111 1000 tttn #32-bit disp
 2252       assert(is_simm32(offs - long_size),
 2253              "must be 32bit offset (call4)");
 2254       emit_int16(0x0F, (0x80 | cc));
 2255       emit_int32(offs - long_size);
 2256     }
 2257   } else {
 2258     // Note: could eliminate cond. jumps to this jump if condition
 2259     //       is the same however, seems to be rather unlikely case.
 2260     // Note: use jccb() if label to be bound is very close to get
 2261     //       an 8-bit displacement
 2262     L.add_patch_at(code(), locator());
 2263     emit_int16(0x0F, (0x80 | cc));
 2264     emit_int32(0);
 2265   }
 2266 }
 2267 
 2268 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2269   if (L.is_bound()) {
 2270     const int short_size = 2;
 2271     address entry = target(L);
 2272 #ifdef ASSERT
 2273     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2274     intptr_t delta = short_branch_delta();
 2275     if (delta != 0) {
 2276       dist += (dist < 0 ? (-delta) :delta);
 2277     }
 2278     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2279 #endif
 2280     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2281     // 0111 tttn #8-bit disp
 2282     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2283   } else {
 2284     InstructionMark im(this);
 2285     L.add_patch_at(code(), locator(), file, line);
 2286     emit_int16(0x70 | cc, 0);
 2287   }
 2288 }
 2289 
 2290 void Assembler::jmp(Address adr) {
 2291   InstructionMark im(this);
 2292   prefix(adr);
 2293   emit_int8((unsigned char)0xFF);
 2294   emit_operand(rsp, adr);
 2295 }
 2296 
 2297 void Assembler::jmp(Label& L, bool maybe_short) {
 2298   if (L.is_bound()) {
 2299     address entry = target(L);
 2300     assert(entry != NULL, "jmp most probably wrong");
 2301     InstructionMark im(this);
 2302     const int short_size = 2;
 2303     const int long_size = 5;
 2304     intptr_t offs = entry - pc();
 2305     if (maybe_short && is8bit(offs - short_size)) {
 2306       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2307     } else {
 2308       emit_int8((unsigned char)0xE9);
 2309       emit_int32(offs - long_size);
 2310     }
 2311   } else {
 2312     // By default, forward jumps are always 32-bit displacements, since
 2313     // we can't yet know where the label will be bound.  If you're sure that
 2314     // the forward jump will not run beyond 256 bytes, use jmpb to
 2315     // force an 8-bit displacement.
 2316     InstructionMark im(this);
 2317     L.add_patch_at(code(), locator());
 2318     emit_int8((unsigned char)0xE9);
 2319     emit_int32(0);
 2320   }
 2321 }
 2322 
 2323 void Assembler::jmp(Register entry) {
 2324   int encode = prefix_and_encode(entry->encoding());
 2325   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2326 }
 2327 
 2328 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2329   InstructionMark im(this);
 2330   emit_int8((unsigned char)0xE9);
 2331   assert(dest != NULL, "must have a target");
 2332   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2333   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2334   emit_data(disp, rspec.reloc(), call32_operand);
 2335 }
 2336 
 2337 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2338   if (L.is_bound()) {
 2339     const int short_size = 2;
 2340     address entry = target(L);
 2341     assert(entry != NULL, "jmp most probably wrong");
 2342 #ifdef ASSERT
 2343     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2344     intptr_t delta = short_branch_delta();
 2345     if (delta != 0) {
 2346       dist += (dist < 0 ? (-delta) :delta);
 2347     }
 2348     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2349 #endif
 2350     intptr_t offs = entry - pc();
 2351     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2352   } else {
 2353     InstructionMark im(this);
 2354     L.add_patch_at(code(), locator(), file, line);
 2355     emit_int16((unsigned char)0xEB, 0);
 2356   }
 2357 }
 2358 
 2359 void Assembler::ldmxcsr( Address src) {
 2360   if (UseAVX > 0 ) {
 2361     InstructionMark im(this);
 2362     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2363     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2364     emit_int8((unsigned char)0xAE);
 2365     emit_operand(as_Register(2), src);
 2366   } else {
 2367     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2368     InstructionMark im(this);
 2369     prefix(src);
 2370     emit_int16(0x0F, (unsigned char)0xAE);
 2371     emit_operand(as_Register(2), src);
 2372   }
 2373 }
 2374 
 2375 void Assembler::leal(Register dst, Address src) {
 2376   InstructionMark im(this);
 2377 #ifdef _LP64
 2378   emit_int8(0x67); // addr32
 2379   prefix(src, dst);
 2380 #endif // LP64
 2381   emit_int8((unsigned char)0x8D);
 2382   emit_operand(dst, src);
 2383 }
 2384 
 2385 void Assembler::lfence() {
 2386   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2387 }
 2388 
 2389 void Assembler::lock() {
 2390   emit_int8((unsigned char)0xF0);
 2391 }
 2392 
 2393 void Assembler::size_prefix() {
 2394   emit_int8(0x66);
 2395 }
 2396 
 2397 void Assembler::lzcntl(Register dst, Register src) {
 2398   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2399   emit_int8((unsigned char)0xF3);
 2400   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2401   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2402 }
 2403 
 2404 // Emit mfence instruction
 2405 void Assembler::mfence() {
 2406   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2407   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2408 }
 2409 
 2410 // Emit sfence instruction
 2411 void Assembler::sfence() {
 2412   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2413   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2414 }
 2415 
 2416 void Assembler::mov(Register dst, Register src) {
 2417   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2418 }
 2419 
 2420 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2422   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2423   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2424   attributes.set_rex_vex_w_reverted();
 2425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2426   emit_int16(0x28, (0xC0 | encode));
 2427 }
 2428 
 2429 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2430   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2431   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2432   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2433   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2434   emit_int16(0x28, (0xC0 | encode));
 2435 }
 2436 
 2437 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2438   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2440   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2441   emit_int16(0x16, (0xC0 | encode));
 2442 }
 2443 
 2444 void Assembler::movb(Register dst, Address src) {
 2445   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2446   InstructionMark im(this);
 2447   prefix(src, dst, true);
 2448   emit_int8((unsigned char)0x8A);
 2449   emit_operand(dst, src);
 2450 }
 2451 
 2452 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2453   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2454   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2455   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2456   attributes.set_rex_vex_w_reverted();
 2457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2458   emit_int16(0x12, 0xC0 | encode);
 2459 }
 2460 
 2461 void Assembler::kmovbl(KRegister dst, Register src) {
 2462   assert(VM_Version::supports_avx512dq(), "");
 2463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2465   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2466 }
 2467 
 2468 void Assembler::kmovbl(Register dst, KRegister src) {
 2469   assert(VM_Version::supports_avx512dq(), "");
 2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2472   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2473 }
 2474 
 2475 void Assembler::kmovwl(KRegister dst, Register src) {
 2476   assert(VM_Version::supports_evex(), "");
 2477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2478   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2479   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2480 }
 2481 
 2482 void Assembler::kmovwl(Register dst, KRegister src) {
 2483   assert(VM_Version::supports_evex(), "");
 2484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2485   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2486   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2487 }
 2488 
 2489 void Assembler::kmovwl(KRegister dst, Address src) {
 2490   assert(VM_Version::supports_evex(), "");
 2491   InstructionMark im(this);
 2492   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2493   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2494   emit_int8((unsigned char)0x90);
 2495   emit_operand((Register)dst, src);
 2496 }
 2497 
 2498 void Assembler::kmovwl(Address dst, KRegister src) {
 2499   assert(VM_Version::supports_evex(), "");
 2500   InstructionMark im(this);
 2501   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2502   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2503   emit_int8((unsigned char)0x91);
 2504   emit_operand((Register)src, dst);
 2505 }
 2506 
 2507 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2508   assert(VM_Version::supports_avx512bw(), "");
 2509   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2510   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2511   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2512 }
 2513 
 2514 void Assembler::kmovdl(KRegister dst, Register src) {
 2515   assert(VM_Version::supports_avx512bw(), "");
 2516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2517   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2518   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2519 }
 2520 
 2521 void Assembler::kmovdl(Register dst, KRegister src) {
 2522   assert(VM_Version::supports_avx512bw(), "");
 2523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2524   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2525   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2526 }
 2527 
 2528 void Assembler::kmovql(KRegister dst, KRegister src) {
 2529   assert(VM_Version::supports_avx512bw(), "");
 2530   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2531   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2532   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2533 }
 2534 
 2535 void Assembler::kmovql(KRegister dst, Address src) {
 2536   assert(VM_Version::supports_avx512bw(), "");
 2537   InstructionMark im(this);
 2538   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2539   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2540   emit_int8((unsigned char)0x90);
 2541   emit_operand((Register)dst, src);
 2542 }
 2543 
 2544 void Assembler::kmovql(Address dst, KRegister src) {
 2545   assert(VM_Version::supports_avx512bw(), "");
 2546   InstructionMark im(this);
 2547   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2548   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2549   emit_int8((unsigned char)0x91);
 2550   emit_operand((Register)src, dst);
 2551 }
 2552 
 2553 void Assembler::kmovql(KRegister dst, Register src) {
 2554   assert(VM_Version::supports_avx512bw(), "");
 2555   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2556   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2557   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2558 }
 2559 
 2560 void Assembler::kmovql(Register dst, KRegister src) {
 2561   assert(VM_Version::supports_avx512bw(), "");
 2562   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2563   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2564   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2565 }
 2566 
 2567 void Assembler::knotwl(KRegister dst, KRegister src) {
 2568   assert(VM_Version::supports_evex(), "");
 2569   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2570   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2571   emit_int16(0x44, (0xC0 | encode));
 2572 }
 2573 
 2574 void Assembler::knotql(KRegister dst, KRegister src) {
 2575   assert(VM_Version::supports_avx512bw(), "");
 2576   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2578   emit_int16(0x44, (0xC0 | encode));
 2579 }
 2580 
 2581 // This instruction produces ZF or CF flags
 2582 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2583   assert(VM_Version::supports_avx512dq(), "");
 2584   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2585   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2586   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2587 }
 2588 
 2589 // This instruction produces ZF or CF flags
 2590 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2591   assert(VM_Version::supports_evex(), "");
 2592   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2593   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2594   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2595 }
 2596 
 2597 // This instruction produces ZF or CF flags
 2598 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2599   assert(VM_Version::supports_avx512bw(), "");
 2600   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2601   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2602   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2603 }
 2604 
 2605 // This instruction produces ZF or CF flags
 2606 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2607   assert(VM_Version::supports_avx512bw(), "");
 2608   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2609   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2610   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2611 }
 2612 
 2613 // This instruction produces ZF or CF flags
 2614 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2615   assert(VM_Version::supports_avx512bw(), "");
 2616   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2617   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2618   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2619 }
 2620 
 2621 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2622   assert(VM_Version::supports_avx512bw(), "");
 2623   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2624   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2625   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2626 }
 2627 
 2628 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2629   assert(VM_Version::supports_avx512bw(), "");
 2630   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2631   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2632   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2633 }
 2634 
 2635 void Assembler::movb(Address dst, int imm8) {
 2636   InstructionMark im(this);
 2637    prefix(dst);
 2638   emit_int8((unsigned char)0xC6);
 2639   emit_operand(rax, dst, 1);
 2640   emit_int8(imm8);
 2641 }
 2642 
 2643 
 2644 void Assembler::movb(Address dst, Register src) {
 2645   assert(src->has_byte_register(), "must have byte register");
 2646   InstructionMark im(this);
 2647   prefix(dst, src, true);
 2648   emit_int8((unsigned char)0x88);
 2649   emit_operand(src, dst);
 2650 }
 2651 
 2652 void Assembler::movdl(XMMRegister dst, Register src) {
 2653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2655   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2656   emit_int16(0x6E, (0xC0 | encode));
 2657 }
 2658 
 2659 void Assembler::movdl(Register dst, XMMRegister src) {
 2660   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2661   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2662   // swap src/dst to get correct prefix
 2663   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2664   emit_int16(0x7E, (0xC0 | encode));
 2665 }
 2666 
 2667 void Assembler::movdl(XMMRegister dst, Address src) {
 2668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2669   InstructionMark im(this);
 2670   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2671   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2672   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2673   emit_int8(0x6E);
 2674   emit_operand(dst, src);
 2675 }
 2676 
 2677 void Assembler::movdl(Address dst, XMMRegister src) {
 2678   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2679   InstructionMark im(this);
 2680   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2681   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2682   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2683   emit_int8(0x7E);
 2684   emit_operand(src, dst);
 2685 }
 2686 
 2687 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2688   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2690   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2691   emit_int16(0x6F, (0xC0 | encode));
 2692 }
 2693 
 2694 void Assembler::movdqa(XMMRegister dst, Address src) {
 2695   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2696   InstructionMark im(this);
 2697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2698   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2699   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2700   emit_int8(0x6F);
 2701   emit_operand(dst, src);
 2702 }
 2703 
 2704 void Assembler::movdqu(XMMRegister dst, Address src) {
 2705   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2706   InstructionMark im(this);
 2707   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2708   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2709   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2710   emit_int8(0x6F);
 2711   emit_operand(dst, src);
 2712 }
 2713 
 2714 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2718   emit_int16(0x6F, (0xC0 | encode));
 2719 }
 2720 
 2721 void Assembler::movdqu(Address dst, XMMRegister src) {
 2722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2723   InstructionMark im(this);
 2724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2725   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2726   attributes.reset_is_clear_context();
 2727   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2728   emit_int8(0x7F);
 2729   emit_operand(src, dst);
 2730 }
 2731 
 2732 // Move Unaligned 256bit Vector
 2733 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2734   assert(UseAVX > 0, "");
 2735   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2736   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2737   emit_int16(0x6F, (0xC0 | encode));
 2738 }
 2739 
 2740 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 2741   assert(UseAVX > 0, "");
 2742   InstructionMark im(this);
 2743   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2744   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2745   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2746   emit_int8(0x6F);
 2747   emit_operand(dst, src);
 2748 }
 2749 
 2750 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 2751   assert(UseAVX > 0, "");
 2752   InstructionMark im(this);
 2753   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2754   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2755   attributes.reset_is_clear_context();
 2756   // swap src<->dst for encoding
 2757   assert(src != xnoreg, "sanity");
 2758   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2759   emit_int8(0x7F);
 2760   emit_operand(src, dst);
 2761 }
 2762 
 2763 // Move Aligned 256bit Vector
 2764 
 2765 void Assembler::vmovdqa(XMMRegister dst, Address src) {
 2766   assert(UseAVX > 0, "");
 2767   InstructionMark im(this);
 2768   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2769   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2770   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2771   emit_int8(0x6F);
 2772   emit_operand(dst, src);
 2773 }
 2774 
 2775 void Assembler::vmovdqa(Address dst, XMMRegister src) {
 2776   assert(UseAVX > 0, "");
 2777   InstructionMark im(this);
 2778   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2779   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2780   attributes.reset_is_clear_context();
 2781   // swap src<->dst for encoding
 2782   assert(src != xnoreg, "sanity");
 2783   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2784   emit_int8(0x7F);
 2785   emit_operand(src, dst);
 2786 }
 2787 
 2788 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 2789 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 2790   assert(VM_Version::supports_evex(), "");
 2791   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2792   attributes.set_is_evex_instruction();
 2793   if (merge) {
 2794     attributes.reset_is_clear_context();
 2795   }
 2796   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2797   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2798   emit_int16(0x6F, (0xC0 | encode));
 2799 }
 2800 
 2801 void Assembler::evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) {
 2802   assert(VM_Version::supports_evex(), "");
 2803   InstructionMark im(this);
 2804   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2805   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2806   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2807   attributes.set_is_evex_instruction();
 2808   if (merge) {
 2809     attributes.reset_is_clear_context();
 2810   }
 2811   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2812   emit_int8(0x6F);
 2813   emit_operand(dst, src);
 2814 }
 2815 
 2816 void Assembler::evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) {
 2817   assert(VM_Version::supports_evex(), "");
 2818   assert(src != xnoreg, "sanity");
 2819   InstructionMark im(this);
 2820   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2821   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2822   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2823   attributes.set_is_evex_instruction();
 2824   if (merge) {
 2825     attributes.reset_is_clear_context();
 2826   }
 2827   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2828   emit_int8(0x7F);
 2829   emit_operand(src, dst);
 2830 }
 2831 
 2832 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2833   assert(VM_Version::supports_avx512vlbw(), "");
 2834   InstructionMark im(this);
 2835   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2836   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2837   attributes.set_embedded_opmask_register_specifier(mask);
 2838   attributes.set_is_evex_instruction();
 2839   if (merge) {
 2840     attributes.reset_is_clear_context();
 2841   }
 2842   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2843   emit_int8(0x6F);
 2844   emit_operand(dst, src);
 2845 }
 2846 
 2847 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2848   assert(VM_Version::supports_avx512vlbw(), "");
 2849   assert(src != xnoreg, "sanity");
 2850   InstructionMark im(this);
 2851   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2852   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2853   attributes.set_embedded_opmask_register_specifier(mask);
 2854   attributes.set_is_evex_instruction();
 2855   if (merge) {
 2856     attributes.reset_is_clear_context();
 2857   }
 2858   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2859   emit_int8(0x7F);
 2860   emit_operand(src, dst);
 2861 }
 2862 
 2863 void Assembler::evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) {
 2864   assert(VM_Version::supports_evex(), "");
 2865   InstructionMark im(this);
 2866   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2867   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2868   attributes.set_is_evex_instruction();
 2869   if (merge) {
 2870     attributes.reset_is_clear_context();
 2871   }
 2872   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2873   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2874   emit_int8(0x6F);
 2875   emit_operand(dst, src);
 2876 }
 2877 
 2878 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2879   assert(VM_Version::supports_avx512vlbw(), "");
 2880   InstructionMark im(this);
 2881   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2882   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2883   attributes.set_embedded_opmask_register_specifier(mask);
 2884   attributes.set_is_evex_instruction();
 2885   if (merge) {
 2886     attributes.reset_is_clear_context();
 2887   }
 2888   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2889   emit_int8(0x6F);
 2890   emit_operand(dst, src);
 2891 }
 2892 
 2893 void Assembler::evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) {
 2894   assert(VM_Version::supports_evex(), "");
 2895   assert(src != xnoreg, "sanity");
 2896   InstructionMark im(this);
 2897   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2898   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2899   attributes.set_is_evex_instruction();
 2900   if (merge) {
 2901     attributes.reset_is_clear_context();
 2902   }
 2903   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2904   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2905   emit_int8(0x7F);
 2906   emit_operand(src, dst);
 2907 }
 2908 
 2909 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2910   assert(VM_Version::supports_avx512vlbw(), "");
 2911   assert(src != xnoreg, "sanity");
 2912   InstructionMark im(this);
 2913   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2914   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2915   attributes.set_embedded_opmask_register_specifier(mask);
 2916   attributes.set_is_evex_instruction();
 2917   if (merge) {
 2918     attributes.reset_is_clear_context();
 2919   }
 2920   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2921   emit_int8(0x7F);
 2922   emit_operand(src, dst);
 2923 }
 2924 
 2925 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 2926   // Unmasked instruction
 2927   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 2928 }
 2929 
 2930 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2931   assert(VM_Version::supports_evex(), "");
 2932   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2933   attributes.set_embedded_opmask_register_specifier(mask);
 2934   attributes.set_is_evex_instruction();
 2935   if (merge) {
 2936     attributes.reset_is_clear_context();
 2937   }
 2938   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2939   emit_int16(0x6F, (0xC0 | encode));
 2940 }
 2941 
 2942 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 2943   // Unmasked instruction
 2944   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 2945 }
 2946 
 2947 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2948   assert(VM_Version::supports_evex(), "");
 2949   InstructionMark im(this);
 2950   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 2951   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2952   attributes.set_embedded_opmask_register_specifier(mask);
 2953   attributes.set_is_evex_instruction();
 2954   if (merge) {
 2955     attributes.reset_is_clear_context();
 2956   }
 2957   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2958   emit_int8(0x6F);
 2959   emit_operand(dst, src);
 2960 }
 2961 
 2962 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 2963   // Unmasked isntruction
 2964   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 2965 }
 2966 
 2967 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2968   assert(VM_Version::supports_evex(), "");
 2969   assert(src != xnoreg, "sanity");
 2970   InstructionMark im(this);
 2971   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2972   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2973   attributes.set_embedded_opmask_register_specifier(mask);
 2974   attributes.set_is_evex_instruction();
 2975   if (merge) {
 2976     attributes.reset_is_clear_context();
 2977   }
 2978   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2979   emit_int8(0x7F);
 2980   emit_operand(src, dst);
 2981 }
 2982 
 2983 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 2984   // Unmasked instruction
 2985   if (dst->encoding() == src->encoding()) return;
 2986   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 2987 }
 2988 
 2989 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2990   assert(VM_Version::supports_evex(), "");
 2991   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2992   attributes.set_embedded_opmask_register_specifier(mask);
 2993   attributes.set_is_evex_instruction();
 2994   if (merge) {
 2995     attributes.reset_is_clear_context();
 2996   }
 2997   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2998   emit_int16(0x6F, (0xC0 | encode));
 2999 }
 3000 
 3001 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3002   // Unmasked instruction
 3003   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3004 }
 3005 
 3006 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3007   assert(VM_Version::supports_evex(), "");
 3008   InstructionMark im(this);
 3009   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3010   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3011   attributes.set_embedded_opmask_register_specifier(mask);
 3012   attributes.set_is_evex_instruction();
 3013   if (merge) {
 3014     attributes.reset_is_clear_context();
 3015   }
 3016   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3017   emit_int8(0x6F);
 3018   emit_operand(dst, src);
 3019 }
 3020 
 3021 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3022   // Unmasked instruction
 3023   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3024 }
 3025 
 3026 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3027   assert(VM_Version::supports_evex(), "");
 3028   assert(src != xnoreg, "sanity");
 3029   InstructionMark im(this);
 3030   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3031   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3032   attributes.set_embedded_opmask_register_specifier(mask);
 3033   if (merge) {
 3034     attributes.reset_is_clear_context();
 3035   }
 3036   attributes.set_is_evex_instruction();
 3037   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3038   emit_int8(0x7F);
 3039   emit_operand(src, dst);
 3040 }
 3041 
 3042 // Move Aligned EVEX enabled Vector (programmable : 8,16,32,64)
 3043 void Assembler::evmovdqab(XMMRegister dst, Address src, int vector_len) {
 3044   assert(VM_Version::supports_evex(), "");
 3045   InstructionMark im(this);
 3046   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3047   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3048   attributes.set_is_evex_instruction();
 3049   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3050   emit_int8(0x6F);
 3051   emit_operand(dst, src);
 3052 }
 3053 
 3054 void Assembler::evmovdqab(Address dst, XMMRegister src, int vector_len) {
 3055   assert(VM_Version::supports_evex(), "");
 3056   assert(src != xnoreg, "sanity");
 3057   InstructionMark im(this);
 3058   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3059   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3060   attributes.set_is_evex_instruction();
 3061   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3062   emit_int8(0x7F);
 3063   emit_operand(src, dst);
 3064 }
 3065 
 3066 void Assembler::evmovdqab(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 3067   assert(VM_Version::supports_avx512vlbw(), "");
 3068   InstructionMark im(this);
 3069   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3070   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3071   attributes.set_embedded_opmask_register_specifier(mask);
 3072   attributes.set_is_evex_instruction();
 3073   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3074   emit_int8(0x6F);
 3075   emit_operand(dst, src);
 3076 }
 3077 
 3078 void Assembler::evmovdqaw(XMMRegister dst, Address src, int vector_len) {
 3079   assert(VM_Version::supports_evex(), "");
 3080   InstructionMark im(this);
 3081   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3082   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3083   attributes.set_is_evex_instruction();
 3084   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3085   emit_int8(0x6F);
 3086   emit_operand(dst, src);
 3087 }
 3088 
 3089 void Assembler::evmovdqaw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 3090   assert(VM_Version::supports_avx512vlbw(), "");
 3091   InstructionMark im(this);
 3092   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3093   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3094   attributes.set_embedded_opmask_register_specifier(mask);
 3095   attributes.set_is_evex_instruction();
 3096   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3097   emit_int8(0x6F);
 3098   emit_operand(dst, src);
 3099 }
 3100 
 3101 void Assembler::evmovdqaw(Address dst, XMMRegister src, int vector_len) {
 3102   assert(VM_Version::supports_evex(), "");
 3103   assert(src != xnoreg, "sanity");
 3104   InstructionMark im(this);
 3105   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3106   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3107   attributes.set_is_evex_instruction();
 3108   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3109   emit_int8(0x7F);
 3110   emit_operand(src, dst);
 3111 }
 3112 
 3113 void Assembler::evmovdqaw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 3114   assert(VM_Version::supports_avx512vlbw(), "");
 3115   assert(src != xnoreg, "sanity");
 3116   InstructionMark im(this);
 3117   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3118   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3119   attributes.reset_is_clear_context();
 3120   attributes.set_embedded_opmask_register_specifier(mask);
 3121   attributes.set_is_evex_instruction();
 3122   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3123   emit_int8(0x7F);
 3124   emit_operand(src, dst);
 3125 }
 3126 
 3127 void Assembler::evmovdqal(XMMRegister dst, Address src, int vector_len) {
 3128   assert(VM_Version::supports_evex(), "");
 3129   InstructionMark im(this);
 3130   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ true);
 3131   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3132   attributes.set_is_evex_instruction();
 3133   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3134   emit_int8(0x6F);
 3135   emit_operand(dst, src);
 3136 }
 3137 
 3138 void Assembler::evmovdqal(Address dst, XMMRegister src, int vector_len) {
 3139   assert(VM_Version::supports_evex(), "");
 3140   assert(src != xnoreg, "sanity");
 3141   InstructionMark im(this);
 3142   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3143   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3144   attributes.reset_is_clear_context();
 3145   attributes.set_is_evex_instruction();
 3146   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3147   emit_int8(0x7F);
 3148   emit_operand(src, dst);
 3149 }
 3150 
 3151 void Assembler::evmovdqaq(XMMRegister dst, Address src, int vector_len) {
 3152   assert(VM_Version::supports_evex(), "");
 3153   InstructionMark im(this);
 3154   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3155   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3156   attributes.set_is_evex_instruction();
 3157   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3158   emit_int8(0x6F);
 3159   emit_operand(dst, src);
 3160 }
 3161 
 3162 void Assembler::evmovdqaq(Address dst, XMMRegister src, int vector_len) {
 3163   assert(VM_Version::supports_evex(), "");
 3164   assert(src != xnoreg, "sanity");
 3165   InstructionMark im(this);
 3166   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3167   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3168   attributes.reset_is_clear_context();
 3169   attributes.set_is_evex_instruction();
 3170   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3171   emit_int8(0x7F);
 3172   emit_operand(src, dst);
 3173 }
 3174 
 3175 // Uses zero extension on 64bit
 3176 
 3177 void Assembler::movl(Register dst, int32_t imm32) {
 3178   int encode = prefix_and_encode(dst->encoding());
 3179   emit_int8(0xB8 | encode);
 3180   emit_int32(imm32);
 3181 }
 3182 
 3183 void Assembler::movl(Register dst, Register src) {
 3184   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3185   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3186 }
 3187 
 3188 void Assembler::movl(Register dst, Address src) {
 3189   InstructionMark im(this);
 3190   prefix(src, dst);
 3191   emit_int8((unsigned char)0x8B);
 3192   emit_operand(dst, src);
 3193 }
 3194 
 3195 void Assembler::movl(Address dst, int32_t imm32) {
 3196   InstructionMark im(this);
 3197   prefix(dst);
 3198   emit_int8((unsigned char)0xC7);
 3199   emit_operand(rax, dst, 4);
 3200   emit_int32(imm32);
 3201 }
 3202 
 3203 void Assembler::movl(Address dst, Register src) {
 3204   InstructionMark im(this);
 3205   prefix(dst, src);
 3206   emit_int8((unsigned char)0x89);
 3207   emit_operand(src, dst);
 3208 }
 3209 
 3210 // New cpus require to use movsd and movss to avoid partial register stall
 3211 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3212 // The selection is done in MacroAssembler::movdbl() and movflt().
 3213 void Assembler::movlpd(XMMRegister dst, Address src) {
 3214   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3215   InstructionMark im(this);
 3216   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3217   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3218   attributes.set_rex_vex_w_reverted();
 3219   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3220   emit_int8(0x12);
 3221   emit_operand(dst, src);
 3222 }
 3223 
 3224 void Assembler::movntq(Address dst, Register src) { // uses the MOVNTI operation
 3225   InstructionMark im(this);
 3226   prefixq(dst);
 3227   emit_int8(0x0F);
 3228   emit_int8((unsigned char)0xC3);
 3229   emit_operand(src, dst);
 3230 }
 3231 
 3232 void Assembler::movntdq(Address dst, XMMRegister src) {
 3233   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3234   InstructionMark im(this);
 3235   emit_int8((unsigned char)0x66);
 3236   emit_int8(0x0F);
 3237   emit_int8((unsigned char)0xE7);
 3238   emit_operand(src, dst);
 3239 }
 3240 
 3241 void Assembler::movntdqa(XMMRegister dst, Address src) {
 3242   NOT_LP64(assert(VM_Version::supports_sse4_1(), ""));
 3243   emit_int8((unsigned char)0x66);
 3244   emit_int8(0x0F);
 3245   emit_int8(0x38);
 3246   emit_int8((unsigned char)0x2A);
 3247   emit_operand(dst, src);
 3248 
 3249   // Same thing, with AVX:
 3250   // assert(UseAVX > 0, "");
 3251   // InstructionMark im(this);
 3252   // InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3253   // simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3254   // emit_int8((unsigned char)0x2A);
 3255   // emit_operand(dst, src);
 3256 }
 3257 
 3258 void Assembler::vmovntdqa(XMMRegister dst, Address src) {
 3259   assert(UseAVX > 0, "");
 3260   InstructionMark im(this);
 3261   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3262   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3263   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3264   emit_int8(0x2A);
 3265   emit_operand(dst, src);
 3266 }
 3267 
 3268 void Assembler::evmovntdqa(XMMRegister dst, Address src, int vector_len) {
 3269   assert(VM_Version::supports_evex(), "");
 3270   InstructionMark im(this);
 3271   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3272   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3273   attributes.set_is_evex_instruction();
 3274   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3275   emit_int8(0x2A);
 3276   emit_operand(dst, src);
 3277 }
 3278 
 3279 void Assembler::vmovntdq(Address dst, XMMRegister src) {
 3280   assert(UseAVX > 0, "");
 3281   InstructionMark im(this);
 3282   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3283   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3284   attributes.reset_is_clear_context();
 3285   // swap src<->dst for encoding
 3286   assert(src != xnoreg, "sanity");
 3287   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3288   emit_int8((unsigned char)0xE7);
 3289   emit_operand(src, dst);
 3290 }
 3291 
 3292 void Assembler::evmovntdq(Address dst, XMMRegister src, int vector_len) {
 3293   assert(VM_Version::supports_evex(), "");
 3294   assert(src != xnoreg, "sanity");
 3295   InstructionMark im(this);
 3296   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3297   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3298   attributes.reset_is_clear_context();
 3299   attributes.set_is_evex_instruction();
 3300   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3301   emit_int8((unsigned char)0xE7);
 3302   emit_operand(src, dst);
 3303 }
 3304 
 3305 void Assembler::movq(XMMRegister dst, Address src) {
 3306   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3307   InstructionMark im(this);
 3308   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3309   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3310   attributes.set_rex_vex_w_reverted();
 3311   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3312   emit_int8(0x7E);
 3313   emit_operand(dst, src);
 3314 }
 3315 
 3316 void Assembler::movq(Address dst, XMMRegister src) {
 3317   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3318   InstructionMark im(this);
 3319   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3320   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3321   attributes.set_rex_vex_w_reverted();
 3322   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3323   emit_int8((unsigned char)0xD6);
 3324   emit_operand(src, dst);
 3325 }
 3326 
 3327 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3329   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3330   attributes.set_rex_vex_w_reverted();
 3331   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3332   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3333 }
 3334 
 3335 void Assembler::movq(Register dst, XMMRegister src) {
 3336   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3337   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3338   // swap src/dst to get correct prefix
 3339   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3340   emit_int16(0x7E, (0xC0 | encode));
 3341 }
 3342 
 3343 void Assembler::movq(XMMRegister dst, Register src) {
 3344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3345   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3346   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3347   emit_int16(0x6E, (0xC0 | encode));
 3348 }
 3349 
 3350 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3351   InstructionMark im(this);
 3352   prefix(src, dst);
 3353   emit_int16(0x0F, (unsigned char)0xBE);
 3354   emit_operand(dst, src);
 3355 }
 3356 
 3357 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3358   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3359   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3360   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3361 }
 3362 
 3363 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3364   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3365   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3366   attributes.set_rex_vex_w_reverted();
 3367   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3368   emit_int16(0x10, (0xC0 | encode));
 3369 }
 3370 
 3371 void Assembler::movsd(XMMRegister dst, Address src) {
 3372   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3373   InstructionMark im(this);
 3374   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3375   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3376   attributes.set_rex_vex_w_reverted();
 3377   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3378   emit_int8(0x10);
 3379   emit_operand(dst, src);
 3380 }
 3381 
 3382 void Assembler::movsd(Address dst, XMMRegister src) {
 3383   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3384   InstructionMark im(this);
 3385   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3386   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3387   attributes.reset_is_clear_context();
 3388   attributes.set_rex_vex_w_reverted();
 3389   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3390   emit_int8(0x11);
 3391   emit_operand(src, dst);
 3392 }
 3393 
 3394 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3395   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3397   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3398   emit_int16(0x10, (0xC0 | encode));
 3399 }
 3400 
 3401 void Assembler::movss(XMMRegister dst, Address src) {
 3402   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3403   InstructionMark im(this);
 3404   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3405   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3406   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3407   emit_int8(0x10);
 3408   emit_operand(dst, src);
 3409 }
 3410 
 3411 void Assembler::movss(Address dst, XMMRegister src) {
 3412   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3413   InstructionMark im(this);
 3414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3415   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3416   attributes.reset_is_clear_context();
 3417   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3418   emit_int8(0x11);
 3419   emit_operand(src, dst);
 3420 }
 3421 
 3422 void Assembler::movswl(Register dst, Address src) { // movsxw
 3423   InstructionMark im(this);
 3424   prefix(src, dst);
 3425   emit_int16(0x0F, (unsigned char)0xBF);
 3426   emit_operand(dst, src);
 3427 }
 3428 
 3429 void Assembler::movswl(Register dst, Register src) { // movsxw
 3430   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3431   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3432 }
 3433 
 3434 void Assembler::movw(Address dst, int imm16) {
 3435   InstructionMark im(this);
 3436 
 3437   emit_int8(0x66); // switch to 16-bit mode
 3438   prefix(dst);
 3439   emit_int8((unsigned char)0xC7);
 3440   emit_operand(rax, dst, 2);
 3441   emit_int16(imm16);
 3442 }
 3443 
 3444 void Assembler::movw(Register dst, Address src) {
 3445   InstructionMark im(this);
 3446   emit_int8(0x66);
 3447   prefix(src, dst);
 3448   emit_int8((unsigned char)0x8B);
 3449   emit_operand(dst, src);
 3450 }
 3451 
 3452 void Assembler::movw(Address dst, Register src) {
 3453   InstructionMark im(this);
 3454   emit_int8(0x66);
 3455   prefix(dst, src);
 3456   emit_int8((unsigned char)0x89);
 3457   emit_operand(src, dst);
 3458 }
 3459 
 3460 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3461   InstructionMark im(this);
 3462   prefix(src, dst);
 3463   emit_int16(0x0F, (unsigned char)0xB6);
 3464   emit_operand(dst, src);
 3465 }
 3466 
 3467 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3468   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3469   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3470   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3471 }
 3472 
 3473 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3474   InstructionMark im(this);
 3475   prefix(src, dst);
 3476   emit_int16(0x0F, (unsigned char)0xB7);
 3477   emit_operand(dst, src);
 3478 }
 3479 
 3480 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3481   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3482   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3483 }
 3484 
 3485 void Assembler::mull(Address src) {
 3486   InstructionMark im(this);
 3487   prefix(src);
 3488   emit_int8((unsigned char)0xF7);
 3489   emit_operand(rsp, src);
 3490 }
 3491 
 3492 void Assembler::mull(Register src) {
 3493   int encode = prefix_and_encode(src->encoding());
 3494   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3495 }
 3496 
 3497 void Assembler::mulsd(XMMRegister dst, Address src) {
 3498   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3499   InstructionMark im(this);
 3500   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3501   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3502   attributes.set_rex_vex_w_reverted();
 3503   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3504   emit_int8(0x59);
 3505   emit_operand(dst, src);
 3506 }
 3507 
 3508 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3509   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3510   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3511   attributes.set_rex_vex_w_reverted();
 3512   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3513   emit_int16(0x59, (0xC0 | encode));
 3514 }
 3515 
 3516 void Assembler::mulss(XMMRegister dst, Address src) {
 3517   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3518   InstructionMark im(this);
 3519   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3520   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3521   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3522   emit_int8(0x59);
 3523   emit_operand(dst, src);
 3524 }
 3525 
 3526 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3527   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3528   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3529   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3530   emit_int16(0x59, (0xC0 | encode));
 3531 }
 3532 
 3533 void Assembler::negl(Register dst) {
 3534   int encode = prefix_and_encode(dst->encoding());
 3535   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3536 }
 3537 
 3538 void Assembler::negl(Address dst) {
 3539   InstructionMark im(this);
 3540   prefix(dst);
 3541   emit_int8((unsigned char)0xF7);
 3542   emit_operand(as_Register(3), dst);
 3543 }
 3544 
 3545 void Assembler::nop(int i) {
 3546 #ifdef ASSERT
 3547   assert(i > 0, " ");
 3548   // The fancy nops aren't currently recognized by debuggers making it a
 3549   // pain to disassemble code while debugging. If asserts are on clearly
 3550   // speed is not an issue so simply use the single byte traditional nop
 3551   // to do alignment.
 3552 
 3553   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3554   return;
 3555 
 3556 #endif // ASSERT
 3557 
 3558   if (UseAddressNop && VM_Version::is_intel()) {
 3559     //
 3560     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3561     //  1: 0x90
 3562     //  2: 0x66 0x90
 3563     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3564     //  4: 0x0F 0x1F 0x40 0x00
 3565     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3566     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3567     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3568     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3569     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3570     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3571     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3572 
 3573     // The rest coding is Intel specific - don't use consecutive address nops
 3574 
 3575     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3576     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3577     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3578     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3579 
 3580     while(i >= 15) {
 3581       // For Intel don't generate consecutive addess nops (mix with regular nops)
 3582       i -= 15;
 3583       emit_int24(0x66, 0x66, 0x66);
 3584       addr_nop_8();
 3585       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3586     }
 3587     switch (i) {
 3588       case 14:
 3589         emit_int8(0x66); // size prefix
 3590       case 13:
 3591         emit_int8(0x66); // size prefix
 3592       case 12:
 3593         addr_nop_8();
 3594         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3595         break;
 3596       case 11:
 3597         emit_int8(0x66); // size prefix
 3598       case 10:
 3599         emit_int8(0x66); // size prefix
 3600       case 9:
 3601         emit_int8(0x66); // size prefix
 3602       case 8:
 3603         addr_nop_8();
 3604         break;
 3605       case 7:
 3606         addr_nop_7();
 3607         break;
 3608       case 6:
 3609         emit_int8(0x66); // size prefix
 3610       case 5:
 3611         addr_nop_5();
 3612         break;
 3613       case 4:
 3614         addr_nop_4();
 3615         break;
 3616       case 3:
 3617         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3618         emit_int8(0x66); // size prefix
 3619       case 2:
 3620         emit_int8(0x66); // size prefix
 3621       case 1:
 3622         emit_int8((unsigned char)0x90);
 3623                          // nop
 3624         break;
 3625       default:
 3626         assert(i == 0, " ");
 3627     }
 3628     return;
 3629   }
 3630   if (UseAddressNop && VM_Version::is_amd_family()) {
 3631     //
 3632     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3633     //  1: 0x90
 3634     //  2: 0x66 0x90
 3635     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3636     //  4: 0x0F 0x1F 0x40 0x00
 3637     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3638     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3639     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3640     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3641     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3642     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3643     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3644 
 3645     // The rest coding is AMD specific - use consecutive address nops
 3646 
 3647     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3648     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3649     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3650     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3651     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3652     //     Size prefixes (0x66) are added for larger sizes
 3653 
 3654     while(i >= 22) {
 3655       i -= 11;
 3656       emit_int24(0x66, 0x66, 0x66);
 3657       addr_nop_8();
 3658     }
 3659     // Generate first nop for size between 21-12
 3660     switch (i) {
 3661       case 21:
 3662         i -= 1;
 3663         emit_int8(0x66); // size prefix
 3664       case 20:
 3665       case 19:
 3666         i -= 1;
 3667         emit_int8(0x66); // size prefix
 3668       case 18:
 3669       case 17:
 3670         i -= 1;
 3671         emit_int8(0x66); // size prefix
 3672       case 16:
 3673       case 15:
 3674         i -= 8;
 3675         addr_nop_8();
 3676         break;
 3677       case 14:
 3678       case 13:
 3679         i -= 7;
 3680         addr_nop_7();
 3681         break;
 3682       case 12:
 3683         i -= 6;
 3684         emit_int8(0x66); // size prefix
 3685         addr_nop_5();
 3686         break;
 3687       default:
 3688         assert(i < 12, " ");
 3689     }
 3690 
 3691     // Generate second nop for size between 11-1
 3692     switch (i) {
 3693       case 11:
 3694         emit_int8(0x66); // size prefix
 3695       case 10:
 3696         emit_int8(0x66); // size prefix
 3697       case 9:
 3698         emit_int8(0x66); // size prefix
 3699       case 8:
 3700         addr_nop_8();
 3701         break;
 3702       case 7:
 3703         addr_nop_7();
 3704         break;
 3705       case 6:
 3706         emit_int8(0x66); // size prefix
 3707       case 5:
 3708         addr_nop_5();
 3709         break;
 3710       case 4:
 3711         addr_nop_4();
 3712         break;
 3713       case 3:
 3714         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3715         emit_int8(0x66); // size prefix
 3716       case 2:
 3717         emit_int8(0x66); // size prefix
 3718       case 1:
 3719         emit_int8((unsigned char)0x90);
 3720                          // nop
 3721         break;
 3722       default:
 3723         assert(i == 0, " ");
 3724     }
 3725     return;
 3726   }
 3727 
 3728   if (UseAddressNop && VM_Version::is_zx()) {
 3729     //
 3730     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3731     //  1: 0x90
 3732     //  2: 0x66 0x90
 3733     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3734     //  4: 0x0F 0x1F 0x40 0x00
 3735     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3736     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3737     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3738     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3739     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3740     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3741     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3742 
 3743     // The rest coding is ZX specific - don't use consecutive address nops
 3744 
 3745     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3746     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3747     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3748     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3749 
 3750     while (i >= 15) {
 3751       // For ZX don't generate consecutive addess nops (mix with regular nops)
 3752       i -= 15;
 3753       emit_int24(0x66, 0x66, 0x66);
 3754       addr_nop_8();
 3755       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3756     }
 3757     switch (i) {
 3758       case 14:
 3759         emit_int8(0x66); // size prefix
 3760       case 13:
 3761         emit_int8(0x66); // size prefix
 3762       case 12:
 3763         addr_nop_8();
 3764         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3765         break;
 3766       case 11:
 3767         emit_int8(0x66); // size prefix
 3768       case 10:
 3769         emit_int8(0x66); // size prefix
 3770       case 9:
 3771         emit_int8(0x66); // size prefix
 3772       case 8:
 3773         addr_nop_8();
 3774         break;
 3775       case 7:
 3776         addr_nop_7();
 3777         break;
 3778       case 6:
 3779         emit_int8(0x66); // size prefix
 3780       case 5:
 3781         addr_nop_5();
 3782         break;
 3783       case 4:
 3784         addr_nop_4();
 3785         break;
 3786       case 3:
 3787         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3788         emit_int8(0x66); // size prefix
 3789       case 2:
 3790         emit_int8(0x66); // size prefix
 3791       case 1:
 3792         emit_int8((unsigned char)0x90);
 3793                          // nop
 3794         break;
 3795       default:
 3796         assert(i == 0, " ");
 3797     }
 3798     return;
 3799   }
 3800 
 3801   // Using nops with size prefixes "0x66 0x90".
 3802   // From AMD Optimization Guide:
 3803   //  1: 0x90
 3804   //  2: 0x66 0x90
 3805   //  3: 0x66 0x66 0x90
 3806   //  4: 0x66 0x66 0x66 0x90
 3807   //  5: 0x66 0x66 0x90 0x66 0x90
 3808   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3809   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3810   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3811   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3812   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3813   //
 3814   while (i > 12) {
 3815     i -= 4;
 3816     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3817   }
 3818   // 1 - 12 nops
 3819   if (i > 8) {
 3820     if (i > 9) {
 3821       i -= 1;
 3822       emit_int8(0x66);
 3823     }
 3824     i -= 3;
 3825     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3826   }
 3827   // 1 - 8 nops
 3828   if (i > 4) {
 3829     if (i > 6) {
 3830       i -= 1;
 3831       emit_int8(0x66);
 3832     }
 3833     i -= 3;
 3834     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3835   }
 3836   switch (i) {
 3837     case 4:
 3838       emit_int8(0x66);
 3839     case 3:
 3840       emit_int8(0x66);
 3841     case 2:
 3842       emit_int8(0x66);
 3843     case 1:
 3844       emit_int8((unsigned char)0x90);
 3845       break;
 3846     default:
 3847       assert(i == 0, " ");
 3848   }
 3849 }
 3850 
 3851 void Assembler::notl(Register dst) {
 3852   int encode = prefix_and_encode(dst->encoding());
 3853   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 3854 }
 3855 
 3856 void Assembler::orw(Register dst, Register src) {
 3857   (void)prefix_and_encode(dst->encoding(), src->encoding());
 3858   emit_arith(0x0B, 0xC0, dst, src);
 3859 }
 3860 
 3861 void Assembler::orl(Address dst, int32_t imm32) {
 3862   InstructionMark im(this);
 3863   prefix(dst);
 3864   emit_arith_operand(0x81, rcx, dst, imm32);
 3865 }
 3866 
 3867 void Assembler::orl(Register dst, int32_t imm32) {
 3868   prefix(dst);
 3869   emit_arith(0x81, 0xC8, dst, imm32);
 3870 }
 3871 
 3872 void Assembler::orl(Register dst, Address src) {
 3873   InstructionMark im(this);
 3874   prefix(src, dst);
 3875   emit_int8(0x0B);
 3876   emit_operand(dst, src);
 3877 }
 3878 
 3879 void Assembler::orl(Register dst, Register src) {
 3880   (void) prefix_and_encode(dst->encoding(), src->encoding());
 3881   emit_arith(0x0B, 0xC0, dst, src);
 3882 }
 3883 
 3884 void Assembler::orl(Address dst, Register src) {
 3885   InstructionMark im(this);
 3886   prefix(dst, src);
 3887   emit_int8(0x09);
 3888   emit_operand(src, dst);
 3889 }
 3890 
 3891 void Assembler::orb(Address dst, int imm8) {
 3892   InstructionMark im(this);
 3893   prefix(dst);
 3894   emit_int8((unsigned char)0x80);
 3895   emit_operand(rcx, dst, 1);
 3896   emit_int8(imm8);
 3897 }
 3898 
 3899 void Assembler::orb(Address dst, Register src) {
 3900   InstructionMark im(this);
 3901   prefix(dst, src, true);
 3902   emit_int8(0x08);
 3903   emit_operand(src, dst);
 3904 }
 3905 
 3906 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 3907   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3908   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3909   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3910   emit_int16(0x63, (0xC0 | encode));
 3911 }
 3912 
 3913 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3914   assert(UseAVX > 0, "some form of AVX must be enabled");
 3915   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3916   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3917   emit_int16(0x63, (0xC0 | encode));
 3918 }
 3919 
 3920 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 3921   assert(VM_Version::supports_sse2(), "");
 3922   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3923   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3924   emit_int16(0x6B, (0xC0 | encode));
 3925 }
 3926 
 3927 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3928   assert(UseAVX > 0, "some form of AVX must be enabled");
 3929   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3930   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3931   emit_int16(0x6B, (0xC0 | encode));
 3932 }
 3933 
 3934 void Assembler::packuswb(XMMRegister dst, Address src) {
 3935   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3936   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 3937   InstructionMark im(this);
 3938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3939   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 3940   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3941   emit_int8(0x67);
 3942   emit_operand(dst, src);
 3943 }
 3944 
 3945 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 3946   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3947   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3948   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3949   emit_int16(0x67, (0xC0 | encode));
 3950 }
 3951 
 3952 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3953   assert(UseAVX > 0, "some form of AVX must be enabled");
 3954   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3955   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3956   emit_int16(0x67, (0xC0 | encode));
 3957 }
 3958 
 3959 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 3960   assert(VM_Version::supports_sse4_1(), "");
 3961   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3962   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3963   emit_int16(0x2B, (0xC0 | encode));
 3964 }
 3965 
 3966 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3967   assert(UseAVX > 0, "some form of AVX must be enabled");
 3968   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3969   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3970   emit_int16(0x2B, (0xC0 | encode));
 3971 }
 3972 
 3973 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3974   assert(VM_Version::supports_avx2(), "");
 3975   assert(vector_len != AVX_128bit, "");
 3976   // VEX.256.66.0F3A.W1 00 /r ib
 3977   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3978   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3979   emit_int24(0x00, (0xC0 | encode), imm8);
 3980 }
 3981 
 3982 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3983   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 3984          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 3985   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3986   attributes.set_is_evex_instruction();
 3987   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3988   emit_int16(0x36, (0xC0 | encode));
 3989 }
 3990 
 3991 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3992   assert(VM_Version::supports_avx512_vbmi(), "");
 3993   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3994   attributes.set_is_evex_instruction();
 3995   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3996   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 3997 }
 3998 
 3999 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4000   assert(VM_Version::supports_avx512_vbmi(), "");
 4001   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4002   attributes.set_is_evex_instruction();
 4003   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4004   emit_int8((unsigned char)0x8D);
 4005   emit_operand(dst, src);
 4006 }
 4007 
 4008 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4009   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4010          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4011          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4012   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4013   attributes.set_is_evex_instruction();
 4014   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4015   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4016 }
 4017 
 4018 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4019   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4020   // VEX.NDS.256.66.0F38.W0 36 /r
 4021   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4022   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4023   emit_int16(0x36, (0xC0 | encode));
 4024 }
 4025 
 4026 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4027   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4028   // VEX.NDS.256.66.0F38.W0 36 /r
 4029   InstructionMark im(this);
 4030   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4031   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4032   emit_int8(0x36);
 4033   emit_operand(dst, src);
 4034 }
 4035 
 4036 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4037   assert(VM_Version::supports_avx2(), "");
 4038   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4039   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4040   emit_int24(0x46, (0xC0 | encode), imm8);
 4041 }
 4042 
 4043 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4044   assert(VM_Version::supports_avx(), "");
 4045   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4046   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4047   emit_int24(0x06, (0xC0 | encode), imm8);
 4048 }
 4049 
 4050 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4051   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4052   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4053   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4054   emit_int24(0x04, (0xC0 | encode), imm8);
 4055 }
 4056 
 4057 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4058   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4059   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4060   attributes.set_rex_vex_w_reverted();
 4061   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4062   emit_int24(0x05, (0xC0 | encode), imm8);
 4063 }
 4064 
 4065 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4066   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4067   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4068   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4069   emit_int24(0x01, (0xC0 | encode), imm8);
 4070 }
 4071 
 4072 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4073   assert(VM_Version::supports_evex(), "");
 4074   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4075   attributes.set_is_evex_instruction();
 4076   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4077   emit_int16(0x76, (0xC0 | encode));
 4078 }
 4079 
 4080 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4081   assert(VM_Version::supports_avx512_vbmi(), "");
 4082   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4083   attributes.set_is_evex_instruction();
 4084   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4085   emit_int16(0x7D, (0xC0 | encode));
 4086 }
 4087 
 4088 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4089   assert(VM_Version::supports_avx512_vbmi(), "");
 4090   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4091   attributes.set_is_evex_instruction();
 4092   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4093   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4094 }
 4095 
 4096 void Assembler::pause() {
 4097   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4098 }
 4099 
 4100 void Assembler::ud2() {
 4101   emit_int16(0x0F, 0x0B);
 4102 }
 4103 
 4104 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4105   assert(VM_Version::supports_sse4_2(), "");
 4106   InstructionMark im(this);
 4107   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4108   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4109   emit_int8(0x61);
 4110   emit_operand(dst, src);
 4111   emit_int8(imm8);
 4112 }
 4113 
 4114 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4115   assert(VM_Version::supports_sse4_2(), "");
 4116   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4117   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4118   emit_int24(0x61, (0xC0 | encode), imm8);
 4119 }
 4120 
 4121 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4122 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4123   assert(VM_Version::supports_sse2(), "");
 4124   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4125   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4126   emit_int16(0x74, (0xC0 | encode));
 4127 }
 4128 
 4129 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4130   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4131   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4132   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4133   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4134   emit_int16(cond_encoding, (0xC0 | encode));
 4135 }
 4136 
 4137 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4138 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4139   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4140   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4141   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4142   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4143   emit_int16(0x74, (0xC0 | encode));
 4144 }
 4145 
 4146 // In this context, kdst is written the mask used to process the equal components
 4147 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4148   assert(VM_Version::supports_avx512bw(), "");
 4149   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4150   attributes.set_is_evex_instruction();
 4151   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4152   emit_int16(0x74, (0xC0 | encode));
 4153 }
 4154 
 4155 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4156   assert(VM_Version::supports_avx512vlbw(), "");
 4157   InstructionMark im(this);
 4158   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4159   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4160   attributes.set_is_evex_instruction();
 4161   int dst_enc = kdst->encoding();
 4162   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4163   emit_int8(0x64);
 4164   emit_operand(as_Register(dst_enc), src);
 4165 }
 4166 
 4167 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4168   assert(VM_Version::supports_avx512vlbw(), "");
 4169   InstructionMark im(this);
 4170   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4171   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4172   attributes.reset_is_clear_context();
 4173   attributes.set_embedded_opmask_register_specifier(mask);
 4174   attributes.set_is_evex_instruction();
 4175   int dst_enc = kdst->encoding();
 4176   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4177   emit_int8(0x64);
 4178   emit_operand(as_Register(dst_enc), src);
 4179 }
 4180 
 4181 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4182   assert(VM_Version::supports_avx512vlbw(), "");
 4183   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4184   attributes.set_is_evex_instruction();
 4185   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4186   emit_int24(0x3E, (0xC0 | encode), vcc);
 4187 }
 4188 
 4189 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4190   assert(VM_Version::supports_avx512vlbw(), "");
 4191   InstructionMark im(this);
 4192   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4193   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4194   attributes.set_is_evex_instruction();
 4195   int dst_enc = kdst->encoding();
 4196   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4197   emit_int8(0x3E);
 4198   emit_operand(as_Register(dst_enc), src);
 4199   emit_int8(vcc);
 4200 }
 4201 
 4202 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4203   assert(VM_Version::supports_avx512bw(), "");
 4204   InstructionMark im(this);
 4205   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4206   attributes.set_is_evex_instruction();
 4207   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4208   int dst_enc = kdst->encoding();
 4209   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4210   emit_int8(0x74);
 4211   emit_operand(as_Register(dst_enc), src);
 4212 }
 4213 
 4214 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4215   assert(VM_Version::supports_avx512vlbw(), "");
 4216   InstructionMark im(this);
 4217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4218   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4219   attributes.reset_is_clear_context();
 4220   attributes.set_embedded_opmask_register_specifier(mask);
 4221   attributes.set_is_evex_instruction();
 4222   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4223   emit_int8(0x74);
 4224   emit_operand(as_Register(kdst->encoding()), src);
 4225 }
 4226 
 4227 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4228 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4229   assert(VM_Version::supports_sse2(), "");
 4230   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4231   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4232   emit_int16(0x75, (0xC0 | encode));
 4233 }
 4234 
 4235 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4236 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4237   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4238   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4239   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4240   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4241   emit_int16(0x75, (0xC0 | encode));
 4242 }
 4243 
 4244 // In this context, kdst is written the mask used to process the equal components
 4245 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4246   assert(VM_Version::supports_avx512bw(), "");
 4247   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4248   attributes.set_is_evex_instruction();
 4249   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4250   emit_int16(0x75, (0xC0 | encode));
 4251 }
 4252 
 4253 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4254   assert(VM_Version::supports_avx512bw(), "");
 4255   InstructionMark im(this);
 4256   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4257   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4258   attributes.set_is_evex_instruction();
 4259   int dst_enc = kdst->encoding();
 4260   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4261   emit_int8(0x75);
 4262   emit_operand(as_Register(dst_enc), src);
 4263 }
 4264 
 4265 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4266 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4267   assert(VM_Version::supports_sse2(), "");
 4268   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4269   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4270   emit_int16(0x76, (0xC0 | encode));
 4271 }
 4272 
 4273 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4274 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4275   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4276   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4277   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4278   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4279   emit_int16(0x76, (0xC0 | encode));
 4280 }
 4281 
 4282 // In this context, kdst is written the mask used to process the equal components
 4283 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4284   assert(VM_Version::supports_evex(), "");
 4285   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4286   attributes.set_is_evex_instruction();
 4287   attributes.reset_is_clear_context();
 4288   attributes.set_embedded_opmask_register_specifier(mask);
 4289   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4290   emit_int16(0x76, (0xC0 | encode));
 4291 }
 4292 
 4293 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4294   assert(VM_Version::supports_evex(), "");
 4295   InstructionMark im(this);
 4296   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4297   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4298   attributes.set_is_evex_instruction();
 4299   attributes.reset_is_clear_context();
 4300   attributes.set_embedded_opmask_register_specifier(mask);
 4301   int dst_enc = kdst->encoding();
 4302   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4303   emit_int8(0x76);
 4304   emit_operand(as_Register(dst_enc), src);
 4305 }
 4306 
 4307 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4308 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4309   assert(VM_Version::supports_sse4_1(), "");
 4310   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4311   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4312   emit_int16(0x29, (0xC0 | encode));
 4313 }
 4314 
 4315 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4316   assert(VM_Version::supports_avx(), "");
 4317   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4319   emit_int16(cond_encoding, (0xC0 | encode));
 4320 }
 4321 
 4322 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4323 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4324   assert(VM_Version::supports_avx(), "");
 4325   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4326   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4327   emit_int16(0x29, (0xC0 | encode));
 4328 }
 4329 
 4330 // In this context, kdst is written the mask used to process the equal components
 4331 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4332   assert(VM_Version::supports_evex(), "");
 4333   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4334   attributes.reset_is_clear_context();
 4335   attributes.set_is_evex_instruction();
 4336   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4337   emit_int16(0x29, (0xC0 | encode));
 4338 }
 4339 
 4340 // In this context, kdst is written the mask used to process the equal components
 4341 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4342   assert(VM_Version::supports_evex(), "");
 4343   InstructionMark im(this);
 4344   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4345   attributes.reset_is_clear_context();
 4346   attributes.set_is_evex_instruction();
 4347   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4348   int dst_enc = kdst->encoding();
 4349   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4350   emit_int8(0x29);
 4351   emit_operand(as_Register(dst_enc), src);
 4352 }
 4353 
 4354 void Assembler::evpmovd2m(KRegister kdst, XMMRegister src, int vector_len) {
 4355   assert(UseAVX > 2  && VM_Version::supports_avx512dq(), "");
 4356   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4357   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4358   attributes.set_is_evex_instruction();
 4359   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4360   emit_int16(0x39, (0xC0 | encode));
 4361 }
 4362 
 4363 void Assembler::evpmovq2m(KRegister kdst, XMMRegister src, int vector_len) {
 4364   assert(UseAVX > 2  && VM_Version::supports_avx512dq(), "");
 4365   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4366   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4367   attributes.set_is_evex_instruction();
 4368   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4369   emit_int16(0x39, (0xC0 | encode));
 4370 }
 4371 
 4372 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4373   assert(VM_Version::supports_sse4_1(), "");
 4374   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4375   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4376   emit_int16(0x37, (0xC0 | encode));
 4377 }
 4378 
 4379 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4380   assert(VM_Version::supports_sse2(), "");
 4381   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4382   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4383   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4384 }
 4385 
 4386 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4387   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4388          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4389   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4390   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4391   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4392 }
 4393 
 4394 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4395   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 4396   InstructionMark im(this);
 4397   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 4398   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4399   emit_int8((unsigned char)0x8C);
 4400   emit_operand(dst, src);
 4401 }
 4402 
 4403 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4404   assert(VM_Version::supports_sse4_1(), "");
 4405   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4406   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4407   emit_int24(0x16, (0xC0 | encode), imm8);
 4408 }
 4409 
 4410 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4411   assert(VM_Version::supports_sse4_1(), "");
 4412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4413   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4414   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4415   emit_int8(0x16);
 4416   emit_operand(src, dst);
 4417   emit_int8(imm8);
 4418 }
 4419 
 4420 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4421   assert(VM_Version::supports_sse4_1(), "");
 4422   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4423   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4424   emit_int24(0x16, (0xC0 | encode), imm8);
 4425 }
 4426 
 4427 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4428   assert(VM_Version::supports_sse4_1(), "");
 4429   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4430   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4431   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4432   emit_int8(0x16);
 4433   emit_operand(src, dst);
 4434   emit_int8(imm8);
 4435 }
 4436 
 4437 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4438   assert(VM_Version::supports_sse2(), "");
 4439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4440   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4441   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4442 }
 4443 
 4444 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4445   assert(VM_Version::supports_sse4_1(), "");
 4446   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4447   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4448   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4449   emit_int8(0x15);
 4450   emit_operand(src, dst);
 4451   emit_int8(imm8);
 4452 }
 4453 
 4454 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4455   assert(VM_Version::supports_sse4_1(), "");
 4456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4457   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4458   emit_int24(0x14, (0xC0 | encode), imm8);
 4459 }
 4460 
 4461 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4462   assert(VM_Version::supports_sse4_1(), "");
 4463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4464   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4465   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4466   emit_int8(0x14);
 4467   emit_operand(src, dst);
 4468   emit_int8(imm8);
 4469 }
 4470 
 4471 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4472   assert(VM_Version::supports_sse4_1(), "");
 4473   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4474   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4475   emit_int24(0x22, (0xC0 | encode), imm8);
 4476 }
 4477 
 4478 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4479   assert(VM_Version::supports_sse4_1(), "");
 4480   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4482   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4483   emit_int8(0x22);
 4484   emit_operand(dst,src);
 4485   emit_int8(imm8);
 4486 }
 4487 
 4488 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4489   assert(VM_Version::supports_avx(), "");
 4490   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4491   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4492   emit_int24(0x22, (0xC0 | encode), imm8);
 4493 }
 4494 
 4495 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4496   assert(VM_Version::supports_sse4_1(), "");
 4497   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4498   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4499   emit_int24(0x22, (0xC0 | encode), imm8);
 4500 }
 4501 
 4502 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4503   assert(VM_Version::supports_sse4_1(), "");
 4504   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4505   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4506   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4507   emit_int8(0x22);
 4508   emit_operand(dst, src);
 4509   emit_int8(imm8);
 4510 }
 4511 
 4512 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4513   assert(VM_Version::supports_avx(), "");
 4514   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4515   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4516   emit_int24(0x22, (0xC0 | encode), imm8);
 4517 }
 4518 
 4519 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4520   assert(VM_Version::supports_sse2(), "");
 4521   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4522   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4523   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4524 }
 4525 
 4526 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4527   assert(VM_Version::supports_sse2(), "");
 4528   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4529   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4530   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4531   emit_int8((unsigned char)0xC4);
 4532   emit_operand(dst, src);
 4533   emit_int8(imm8);
 4534 }
 4535 
 4536 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4537   assert(VM_Version::supports_avx(), "");
 4538   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4539   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4540   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4541 }
 4542 
 4543 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4544   assert(VM_Version::supports_sse4_1(), "");
 4545   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4547   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4548   emit_int8(0x20);
 4549   emit_operand(dst, src);
 4550   emit_int8(imm8);
 4551 }
 4552 
 4553 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4554   assert(VM_Version::supports_sse4_1(), "");
 4555   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4556   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4557   emit_int24(0x20, (0xC0 | encode), imm8);
 4558 }
 4559 
 4560 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4561   assert(VM_Version::supports_avx(), "");
 4562   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4564   emit_int24(0x20, (0xC0 | encode), imm8);
 4565 }
 4566 
 4567 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4568   assert(VM_Version::supports_sse4_1(), "");
 4569   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4570   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4571   emit_int24(0x21, (0xC0 | encode), imm8);
 4572 }
 4573 
 4574 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4575   assert(VM_Version::supports_avx(), "");
 4576   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4578   emit_int24(0x21, (0xC0 | encode), imm8);
 4579 }
 4580 
 4581 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4582   assert(VM_Version::supports_sse4_1(), "");
 4583   InstructionMark im(this);
 4584   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4585   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4586   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4587   emit_int8(0x30);
 4588   emit_operand(dst, src);
 4589 }
 4590 
 4591 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4592   assert(VM_Version::supports_sse4_1(), "");
 4593   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4594   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4595   emit_int16(0x30, (0xC0 | encode));
 4596 }
 4597 
 4598 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4599   assert(VM_Version::supports_sse4_1(), "");
 4600   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4601   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4602   emit_int16(0x20, (0xC0 | encode));
 4603 }
 4604 
 4605 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4606   assert(VM_Version::supports_sse4_1(), "");
 4607   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4608   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4609   emit_int16(0x35, (0xC0 | encode));
 4610 }
 4611 
 4612 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4613   assert(VM_Version::supports_sse4_1(), "");
 4614   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4615   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4616   emit_int16(0x21, (0xC0 | encode));
 4617 }
 4618 
 4619 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4620   assert(VM_Version::supports_sse4_1(), "");
 4621   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4622   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4623   emit_int16(0x31, (0xC0 | encode));
 4624 }
 4625 
 4626 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4627   assert(VM_Version::supports_sse4_1(), "");
 4628   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4629   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4630   emit_int16(0x22, (0xC0 | encode));
 4631 }
 4632 
 4633 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4634   assert(VM_Version::supports_sse4_1(), "");
 4635   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4636   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4637   emit_int16(0x23, (0xC0 | encode));
 4638 }
 4639 
 4640 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4641   assert(VM_Version::supports_avx(), "");
 4642   InstructionMark im(this);
 4643   assert(dst != xnoreg, "sanity");
 4644   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4645   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4646   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4647   emit_int8(0x30);
 4648   emit_operand(dst, src);
 4649 }
 4650 
 4651 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4652   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4653   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4654   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4655   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4656   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4657   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4658 }
 4659 
 4660 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4661   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4662   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4663   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4664   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4665   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4666   emit_int16(0x20, (0xC0 | encode));
 4667 }
 4668 
 4669 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4670   assert(VM_Version::supports_avx512vlbw(), "");
 4671   assert(dst != xnoreg, "sanity");
 4672   InstructionMark im(this);
 4673   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4674   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4675   attributes.set_embedded_opmask_register_specifier(mask);
 4676   attributes.set_is_evex_instruction();
 4677   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4678   emit_int8(0x30);
 4679   emit_operand(dst, src);
 4680 }
 4681 
 4682 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4683   assert(VM_Version::supports_evex(), "");
 4684   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4685   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4686   attributes.set_is_evex_instruction();
 4687   attributes.set_embedded_opmask_register_specifier(mask);
 4688   if (merge) {
 4689     attributes.reset_is_clear_context();
 4690   }
 4691   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4692   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4693 }
 4694 
 4695 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4696   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4697   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4698   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4699   emit_int16(0x35, (0xC0 | encode));
 4700 }
 4701 
 4702 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4703   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4704   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4705   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4706   emit_int16(0x31, (0xC0 | encode));
 4707 }
 4708 
 4709 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4710   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4711   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4712   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4713   emit_int16(0x32, (0xC0 | encode));
 4714 }
 4715 
 4716 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4717   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4718          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4719              VM_Version::supports_evex(), "");
 4720   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4721   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4722   emit_int16(0x21, (0xC0 | encode));
 4723 }
 4724 
 4725 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4726   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4727          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4728              VM_Version::supports_evex(), "");
 4729   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4730   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4731   emit_int16(0x22, (0xC0 | encode));
 4732 }
 4733 
 4734 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4735   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4736          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4737              VM_Version::supports_evex(), "");
 4738   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4739   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4740   emit_int16(0x23, (0xC0 | encode));
 4741 }
 4742 
 4743 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4744   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4745          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4746              VM_Version::supports_evex(), "");
 4747   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4748   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4749   emit_int16(0x24, (0xC0 | encode));
 4750 }
 4751 
 4752 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4753   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4754          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4755              VM_Version::supports_evex(), "");
 4756   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4757   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4758   emit_int16(0x25, (0xC0 | encode));
 4759 }
 4760 
 4761 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4762   assert(VM_Version::supports_avx512vlbw(), "");
 4763   assert(src != xnoreg, "sanity");
 4764   InstructionMark im(this);
 4765   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4766   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4767   attributes.set_is_evex_instruction();
 4768   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4769   emit_int8(0x30);
 4770   emit_operand(src, dst);
 4771 }
 4772 
 4773 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4774   assert(VM_Version::supports_avx512vlbw(), "");
 4775   assert(src != xnoreg, "sanity");
 4776   InstructionMark im(this);
 4777   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4778   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4779   attributes.reset_is_clear_context();
 4780   attributes.set_embedded_opmask_register_specifier(mask);
 4781   attributes.set_is_evex_instruction();
 4782   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4783   emit_int8(0x30);
 4784   emit_operand(src, dst);
 4785 }
 4786 
 4787 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4788   assert(VM_Version::supports_evex(), "");
 4789   assert(src != xnoreg, "sanity");
 4790   InstructionMark im(this);
 4791   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4792   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4793   attributes.set_is_evex_instruction();
 4794   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4795   emit_int8(0x31);
 4796   emit_operand(src, dst);
 4797 }
 4798 
 4799 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4800   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4801   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4802   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4803   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4804   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4805   emit_int16(0x33, (0xC0 | encode));
 4806 }
 4807 
 4808 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4809   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4810   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4811   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4812   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4813 }
 4814 
 4815 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4816   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4817     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4818     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4819   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4820   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4821   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4822 }
 4823 
 4824 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4825 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4826        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4827        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4828   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4829   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4830   emit_int16(0x04, (0xC0 | encode));
 4831 }
 4832 
 4833 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4834   assert(VM_Version::supports_evex(), "");
 4835   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4836   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4837   attributes.set_is_evex_instruction();
 4838   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4839   emit_int16(0x52, (0xC0 | encode));
 4840 }
 4841 
 4842 // generic
 4843 void Assembler::pop(Register dst) {
 4844   int encode = prefix_and_encode(dst->encoding());
 4845   emit_int8(0x58 | encode);
 4846 }
 4847 
 4848 void Assembler::popcntl(Register dst, Address src) {
 4849   assert(VM_Version::supports_popcnt(), "must support");
 4850   InstructionMark im(this);
 4851   emit_int8((unsigned char)0xF3);
 4852   prefix(src, dst);
 4853   emit_int16(0x0F, (unsigned char)0xB8);
 4854   emit_operand(dst, src);
 4855 }
 4856 
 4857 void Assembler::popcntl(Register dst, Register src) {
 4858   assert(VM_Version::supports_popcnt(), "must support");
 4859   emit_int8((unsigned char)0xF3);
 4860   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 4861   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 4862 }
 4863 
 4864 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
 4865   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4866   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4867   attributes.set_is_evex_instruction();
 4868   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4869   emit_int16(0x55, (0xC0 | encode));
 4870 }
 4871 
 4872 void Assembler::popf() {
 4873   emit_int8((unsigned char)0x9D);
 4874 }
 4875 
 4876 #ifndef _LP64 // no 32bit push/pop on amd64
 4877 void Assembler::popl(Address dst) {
 4878   // NOTE: this will adjust stack by 8byte on 64bits
 4879   InstructionMark im(this);
 4880   prefix(dst);
 4881   emit_int8((unsigned char)0x8F);
 4882   emit_operand(rax, dst);
 4883 }
 4884 #endif
 4885 
 4886 void Assembler::prefetchnta(Address src) {
 4887   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4888   InstructionMark im(this);
 4889   prefix(src);
 4890   emit_int16(0x0F, 0x18);
 4891   emit_operand(rax, src); // 0, src
 4892 }
 4893 
 4894 void Assembler::prefetchr(Address src) {
 4895   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4896   InstructionMark im(this);
 4897   prefix(src);
 4898   emit_int16(0x0F, 0x0D);
 4899   emit_operand(rax, src); // 0, src
 4900 }
 4901 
 4902 void Assembler::prefetcht0(Address src) {
 4903   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4904   InstructionMark im(this);
 4905   prefix(src);
 4906   emit_int16(0x0F, 0x18);
 4907   emit_operand(rcx, src); // 1, src
 4908 }
 4909 
 4910 void Assembler::prefetcht1(Address src) {
 4911   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4912   InstructionMark im(this);
 4913   prefix(src);
 4914   emit_int16(0x0F, 0x18);
 4915   emit_operand(rdx, src); // 2, src
 4916 }
 4917 
 4918 void Assembler::prefetcht2(Address src) {
 4919   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4920   InstructionMark im(this);
 4921   prefix(src);
 4922   emit_int16(0x0F, 0x18);
 4923   emit_operand(rbx, src); // 3, src
 4924 }
 4925 
 4926 void Assembler::prefetchw(Address src) {
 4927   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4928   InstructionMark im(this);
 4929   prefix(src);
 4930   emit_int16(0x0F, 0x0D);
 4931   emit_operand(rcx, src); // 1, src
 4932 }
 4933 
 4934 void Assembler::prefix(Prefix p) {
 4935   emit_int8(p);
 4936 }
 4937 
 4938 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 4939   assert(VM_Version::supports_ssse3(), "");
 4940   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4941   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4942   emit_int16(0x00, (0xC0 | encode));
 4943 }
 4944 
 4945 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4946   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4947          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4948          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4949   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4950   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4951   emit_int16(0x00, (0xC0 | encode));
 4952 }
 4953 
 4954 void Assembler::pshufb(XMMRegister dst, Address src) {
 4955   assert(VM_Version::supports_ssse3(), "");
 4956   InstructionMark im(this);
 4957   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4958   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4959   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4960   emit_int8(0x00);
 4961   emit_operand(dst, src);
 4962 }
 4963 
 4964 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 4965   assert(isByte(mode), "invalid value");
 4966   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4967   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 4968   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4969   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4970   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4971 }
 4972 
 4973 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 4974   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4975          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4976          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 4977   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4978   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4979   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4980   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4981 }
 4982 
 4983 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 4984   assert(isByte(mode), "invalid value");
 4985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4986   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4987   InstructionMark im(this);
 4988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4989   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4990   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4991   emit_int8(0x70);
 4992   emit_operand(dst, src);
 4993   emit_int8(mode & 0xFF);
 4994 }
 4995 
 4996 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 4997   assert(isByte(mode), "invalid value");
 4998   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4999   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5000   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5001   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5002 }
 5003 
 5004 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5005   assert(isByte(mode), "invalid value");
 5006   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5007   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5008   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5009   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5010 }
 5011 
 5012 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5013   assert(isByte(mode), "invalid value");
 5014   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5015   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5016   InstructionMark im(this);
 5017   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5018   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5019   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5020   emit_int8(0x70);
 5021   emit_operand(dst, src);
 5022   emit_int8(mode & 0xFF);
 5023 }
 5024 
 5025 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5026   assert(VM_Version::supports_evex(), "requires EVEX support");
 5027   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5028   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5029   attributes.set_is_evex_instruction();
 5030   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5031   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5032 }
 5033 
 5034 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5035   assert(isByte(imm8), "invalid value");
 5036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5038   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5039   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5040 }
 5041 
 5042 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5043   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5044   attributes.set_rex_vex_w_reverted();
 5045   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5046   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5047 }
 5048 
 5049 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 5050   assert(isByte(imm8), "invalid value");
 5051   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5052   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5053   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5054   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5055 }
 5056 
 5057 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5058   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5059   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5060   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5061 }
 5062 
 5063 void Assembler::psrldq(XMMRegister dst, int shift) {
 5064   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5065   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5066   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5067   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5068   emit_int24(0x73, (0xC0 | encode), shift);
 5069 }
 5070 
 5071 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5072   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5073          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5074          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5075   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5076   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5077   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5078 }
 5079 
 5080 void Assembler::pslldq(XMMRegister dst, int shift) {
 5081   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5082   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5083   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5084   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5085   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5086   emit_int24(0x73, (0xC0 | encode), shift);
 5087 }
 5088 
 5089 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5090   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5091          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5092          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5093   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5094   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5095   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5096 }
 5097 
 5098 void Assembler::ptest(XMMRegister dst, Address src) {
 5099   assert(VM_Version::supports_sse4_1(), "");
 5100   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5101   InstructionMark im(this);
 5102   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5103   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5104   emit_int8(0x17);
 5105   emit_operand(dst, src);
 5106 }
 5107 
 5108 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5109   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5110   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5111   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5112   emit_int8(0x17);
 5113   emit_int8((0xC0 | encode));
 5114 }
 5115 
 5116 void Assembler::vptest(XMMRegister dst, Address src) {
 5117   assert(VM_Version::supports_avx(), "");
 5118   InstructionMark im(this);
 5119   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5120   assert(dst != xnoreg, "sanity");
 5121   // swap src<->dst for encoding
 5122   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5123   emit_int8(0x17);
 5124   emit_operand(dst, src);
 5125 }
 5126 
 5127 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5128   assert(VM_Version::supports_avx(), "");
 5129   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5130   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5131   emit_int16(0x17, (0xC0 | encode));
 5132 }
 5133 
 5134 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5135   assert(VM_Version::supports_avx(), "");
 5136   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5137   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5138   emit_int16(0x17, (0xC0 | encode));
 5139 }
 5140 
 5141 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5142   assert(VM_Version::supports_avx512vlbw(), "");
 5143   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5144   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5145   attributes.set_is_evex_instruction();
 5146   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5147   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5148 }
 5149 
 5150 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5151   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5152   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5153   InstructionMark im(this);
 5154   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5155   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5156   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5157   emit_int8(0x60);
 5158   emit_operand(dst, src);
 5159 }
 5160 
 5161 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5162   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5163   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5164   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5165   emit_int16(0x60, (0xC0 | encode));
 5166 }
 5167 
 5168 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5169   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5170   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5171   InstructionMark im(this);
 5172   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5173   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5174   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5175   emit_int8(0x62);
 5176   emit_operand(dst, src);
 5177 }
 5178 
 5179 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5180   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5181   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5182   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5183   emit_int16(0x62, (0xC0 | encode));
 5184 }
 5185 
 5186 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5187   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5188   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5189   attributes.set_rex_vex_w_reverted();
 5190   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5191   emit_int16(0x6C, (0xC0 | encode));
 5192 }
 5193 
 5194 void Assembler::push(int32_t imm32) {
 5195   // in 64bits we push 64bits onto the stack but only
 5196   // take a 32bit immediate
 5197   emit_int8(0x68);
 5198   emit_int32(imm32);
 5199 }
 5200 
 5201 void Assembler::push(Register src) {
 5202   int encode = prefix_and_encode(src->encoding());
 5203   emit_int8(0x50 | encode);
 5204 }
 5205 
 5206 void Assembler::pushf() {
 5207   emit_int8((unsigned char)0x9C);
 5208 }
 5209 
 5210 #ifndef _LP64 // no 32bit push/pop on amd64
 5211 void Assembler::pushl(Address src) {
 5212   // Note this will push 64bit on 64bit
 5213   InstructionMark im(this);
 5214   prefix(src);
 5215   emit_int8((unsigned char)0xFF);
 5216   emit_operand(rsi, src);
 5217 }
 5218 #endif
 5219 
 5220 void Assembler::rcll(Register dst, int imm8) {
 5221   assert(isShiftCount(imm8), "illegal shift count");
 5222   int encode = prefix_and_encode(dst->encoding());
 5223   if (imm8 == 1) {
 5224     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5225   } else {
 5226     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5227   }
 5228 }
 5229 
 5230 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5231   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5232   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5233   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5234   emit_int16(0x53, (0xC0 | encode));
 5235 }
 5236 
 5237 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5238   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5239   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5240   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5241   emit_int16(0x53, (0xC0 | encode));
 5242 }
 5243 
 5244 void Assembler::rdtsc() {
 5245   emit_int16(0x0F, 0x31);
 5246 }
 5247 
 5248 // copies data from [esi] to [edi] using rcx pointer sized words
 5249 // generic
 5250 void Assembler::rep_mov() {
 5251   // REP
 5252   // MOVSQ
 5253   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5254   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5255 }
 5256 
 5257 // sets rcx bytes with rax, value at [edi]
 5258 void Assembler::rep_stosb() {
 5259   // REP
 5260   // STOSB
 5261   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5262   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5263 }
 5264 
 5265 // sets rcx pointer sized words with rax, value at [edi]
 5266 // generic
 5267 void Assembler::rep_stos() {
 5268   // REP
 5269   // LP64:STOSQ, LP32:STOSD
 5270   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5271   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5272 }
 5273 
 5274 // scans rcx pointer sized words at [edi] for occurance of rax,
 5275 // generic
 5276 void Assembler::repne_scan() { // repne_scan
 5277   // SCASQ
 5278   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5279   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5280 }
 5281 
 5282 #ifdef _LP64
 5283 // scans rcx 4 byte words at [edi] for occurance of rax,
 5284 // generic
 5285 void Assembler::repne_scanl() { // repne_scan
 5286   // SCASL
 5287   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5288 }
 5289 #endif
 5290 
 5291 void Assembler::ret(int imm16) {
 5292   if (imm16 == 0) {
 5293     emit_int8((unsigned char)0xC3);
 5294   } else {
 5295     emit_int8((unsigned char)0xC2);
 5296     emit_int16(imm16);
 5297   }
 5298 }
 5299 
 5300 void Assembler::roll(Register dst, int imm8) {
 5301   assert(isShiftCount(imm8), "illegal shift count");
 5302   int encode = prefix_and_encode(dst->encoding());
 5303   if (imm8 == 1) {
 5304     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5305   } else {
 5306     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5307   }
 5308 }
 5309 
 5310 void Assembler::roll(Register dst) {
 5311   int encode = prefix_and_encode(dst->encoding());
 5312   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5313 }
 5314 
 5315 void Assembler::rorl(Register dst, int imm8) {
 5316   assert(isShiftCount(imm8), "illegal shift count");
 5317   int encode = prefix_and_encode(dst->encoding());
 5318   if (imm8 == 1) {
 5319     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5320   } else {
 5321     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5322   }
 5323 }
 5324 
 5325 void Assembler::rorl(Register dst) {
 5326   int encode = prefix_and_encode(dst->encoding());
 5327   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5328 }
 5329 
 5330 #ifdef _LP64
 5331 void Assembler::rorq(Register dst) {
 5332   int encode = prefixq_and_encode(dst->encoding());
 5333   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5334 }
 5335 
 5336 void Assembler::rorq(Register dst, int imm8) {
 5337   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5338   int encode = prefixq_and_encode(dst->encoding());
 5339   if (imm8 == 1) {
 5340     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5341   } else {
 5342     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5343   }
 5344 }
 5345 
 5346 void Assembler::rolq(Register dst) {
 5347   int encode = prefixq_and_encode(dst->encoding());
 5348   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5349 }
 5350 
 5351 void Assembler::rolq(Register dst, int imm8) {
 5352   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5353   int encode = prefixq_and_encode(dst->encoding());
 5354   if (imm8 == 1) {
 5355     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5356   } else {
 5357     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5358   }
 5359 }
 5360 #endif
 5361 
 5362 void Assembler::sahf() {
 5363 #ifdef _LP64
 5364   // Not supported in 64bit mode
 5365   ShouldNotReachHere();
 5366 #endif
 5367   emit_int8((unsigned char)0x9E);
 5368 }
 5369 
 5370 void Assembler::sall(Address dst, int imm8) {
 5371   InstructionMark im(this);
 5372   assert(isShiftCount(imm8), "illegal shift count");
 5373   prefix(dst);
 5374   if (imm8 == 1) {
 5375     emit_int8((unsigned char)0xD1);
 5376     emit_operand(as_Register(4), dst);
 5377   }
 5378   else {
 5379     emit_int8((unsigned char)0xC1);
 5380     emit_operand(as_Register(4), dst);
 5381     emit_int8(imm8);
 5382   }
 5383 }
 5384 
 5385 void Assembler::sall(Address dst) {
 5386   InstructionMark im(this);
 5387   prefix(dst);
 5388   emit_int8((unsigned char)0xD3);
 5389   emit_operand(as_Register(4), dst);
 5390 }
 5391 
 5392 void Assembler::sall(Register dst, int imm8) {
 5393   assert(isShiftCount(imm8), "illegal shift count");
 5394   int encode = prefix_and_encode(dst->encoding());
 5395   if (imm8 == 1) {
 5396     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5397   } else {
 5398     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5399   }
 5400 }
 5401 
 5402 void Assembler::sall(Register dst) {
 5403   int encode = prefix_and_encode(dst->encoding());
 5404   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5405 }
 5406 
 5407 void Assembler::sarl(Address dst, int imm8) {
 5408   assert(isShiftCount(imm8), "illegal shift count");
 5409   InstructionMark im(this);
 5410   prefix(dst);
 5411   if (imm8 == 1) {
 5412     emit_int8((unsigned char)0xD1);
 5413     emit_operand(as_Register(7), dst);
 5414   }
 5415   else {
 5416     emit_int8((unsigned char)0xC1);
 5417     emit_operand(as_Register(7), dst);
 5418     emit_int8(imm8);
 5419   }
 5420 }
 5421 
 5422 void Assembler::sarl(Address dst) {
 5423   InstructionMark im(this);
 5424   prefix(dst);
 5425   emit_int8((unsigned char)0xD3);
 5426   emit_operand(as_Register(7), dst);
 5427 }
 5428 
 5429 void Assembler::sarl(Register dst, int imm8) {
 5430   int encode = prefix_and_encode(dst->encoding());
 5431   assert(isShiftCount(imm8), "illegal shift count");
 5432   if (imm8 == 1) {
 5433     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5434   } else {
 5435     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5436   }
 5437 }
 5438 
 5439 void Assembler::sarl(Register dst) {
 5440   int encode = prefix_and_encode(dst->encoding());
 5441   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5442 }
 5443 
 5444 void Assembler::sbbl(Address dst, int32_t imm32) {
 5445   InstructionMark im(this);
 5446   prefix(dst);
 5447   emit_arith_operand(0x81, rbx, dst, imm32);
 5448 }
 5449 
 5450 void Assembler::sbbl(Register dst, int32_t imm32) {
 5451   prefix(dst);
 5452   emit_arith(0x81, 0xD8, dst, imm32);
 5453 }
 5454 
 5455 
 5456 void Assembler::sbbl(Register dst, Address src) {
 5457   InstructionMark im(this);
 5458   prefix(src, dst);
 5459   emit_int8(0x1B);
 5460   emit_operand(dst, src);
 5461 }
 5462 
 5463 void Assembler::sbbl(Register dst, Register src) {
 5464   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5465   emit_arith(0x1B, 0xC0, dst, src);
 5466 }
 5467 
 5468 void Assembler::setb(Condition cc, Register dst) {
 5469   assert(0 <= cc && cc < 16, "illegal cc");
 5470   int encode = prefix_and_encode(dst->encoding(), true);
 5471   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5472 }
 5473 
 5474 void Assembler::sete(Register dst) {
 5475   int encode = prefix_and_encode(dst->encoding(), true);
 5476   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5477 }
 5478 
 5479 void Assembler::setl(Register dst) {
 5480   int encode = prefix_and_encode(dst->encoding(), true);
 5481   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5482 }
 5483 
 5484 void Assembler::setne(Register dst) {
 5485   int encode = prefix_and_encode(dst->encoding(), true);
 5486   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5487 }
 5488 
 5489 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5490   assert(VM_Version::supports_ssse3(), "");
 5491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5492   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5493   emit_int24(0x0F, (0xC0 | encode), imm8);
 5494 }
 5495 
 5496 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5497   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5498          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5499          0, "");
 5500   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5501   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5502   emit_int24(0x0F, (0xC0 | encode), imm8);
 5503 }
 5504 
 5505 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5506   assert(VM_Version::supports_evex(), "");
 5507   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5508   attributes.set_is_evex_instruction();
 5509   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5510   emit_int24(0x3, (0xC0 | encode), imm8);
 5511 }
 5512 
 5513 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5514   assert(VM_Version::supports_sse4_1(), "");
 5515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5517   emit_int24(0x0E, (0xC0 | encode), imm8);
 5518 }
 5519 
 5520 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5521   assert(VM_Version::supports_sha(), "");
 5522   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5523   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5524 }
 5525 
 5526 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5527   assert(VM_Version::supports_sha(), "");
 5528   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5529   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5530 }
 5531 
 5532 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5533   assert(VM_Version::supports_sha(), "");
 5534   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5535   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5536 }
 5537 
 5538 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5539   assert(VM_Version::supports_sha(), "");
 5540   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5541   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5542 }
 5543 
 5544 // xmm0 is implicit additional source to this instruction.
 5545 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5546   assert(VM_Version::supports_sha(), "");
 5547   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5548   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5549 }
 5550 
 5551 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5552   assert(VM_Version::supports_sha(), "");
 5553   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5554   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5555 }
 5556 
 5557 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5558   assert(VM_Version::supports_sha(), "");
 5559   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5560   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5561 }
 5562 
 5563 
 5564 void Assembler::shll(Register dst, int imm8) {
 5565   assert(isShiftCount(imm8), "illegal shift count");
 5566   int encode = prefix_and_encode(dst->encoding());
 5567   if (imm8 == 1 ) {
 5568     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5569   } else {
 5570     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5571   }
 5572 }
 5573 
 5574 void Assembler::shll(Register dst) {
 5575   int encode = prefix_and_encode(dst->encoding());
 5576   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5577 }
 5578 
 5579 void Assembler::shrl(Register dst, int imm8) {
 5580   assert(isShiftCount(imm8), "illegal shift count");
 5581   int encode = prefix_and_encode(dst->encoding());
 5582   if (imm8 == 1) {
 5583     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5584   }
 5585   else {
 5586     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5587   }
 5588 }
 5589 
 5590 void Assembler::shrl(Register dst) {
 5591   int encode = prefix_and_encode(dst->encoding());
 5592   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5593 }
 5594 
 5595 void Assembler::shrl(Address dst) {
 5596   InstructionMark im(this);
 5597   prefix(dst);
 5598   emit_int8((unsigned char)0xD3);
 5599   emit_operand(as_Register(5), dst);
 5600 }
 5601 
 5602 void Assembler::shrl(Address dst, int imm8) {
 5603   InstructionMark im(this);
 5604   assert(isShiftCount(imm8), "illegal shift count");
 5605   prefix(dst);
 5606   if (imm8 == 1) {
 5607     emit_int8((unsigned char)0xD1);
 5608     emit_operand(as_Register(5), dst);
 5609   }
 5610   else {
 5611     emit_int8((unsigned char)0xC1);
 5612     emit_operand(as_Register(5), dst);
 5613     emit_int8(imm8);
 5614   }
 5615 }
 5616 
 5617 
 5618 void Assembler::shldl(Register dst, Register src) {
 5619   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5620   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5621 }
 5622 
 5623 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5624   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5625   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5626 }
 5627 
 5628 void Assembler::shrdl(Register dst, Register src) {
 5629   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5630   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5631 }
 5632 
 5633 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5634   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5635   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5636 }
 5637 
 5638 // copies a single word from [esi] to [edi]
 5639 void Assembler::smovl() {
 5640   emit_int8((unsigned char)0xA5);
 5641 }
 5642 
 5643 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5644   assert(VM_Version::supports_sse4_1(), "");
 5645   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5646   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5647   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5648 }
 5649 
 5650 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5651   assert(VM_Version::supports_sse4_1(), "");
 5652   InstructionMark im(this);
 5653   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5654   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5655   emit_int8(0x0B);
 5656   emit_operand(dst, src);
 5657   emit_int8((unsigned char)rmode);
 5658 }
 5659 
 5660 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5662   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5663   attributes.set_rex_vex_w_reverted();
 5664   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5665   emit_int16(0x51, (0xC0 | encode));
 5666 }
 5667 
 5668 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5669   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5670   InstructionMark im(this);
 5671   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5672   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5673   attributes.set_rex_vex_w_reverted();
 5674   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5675   emit_int8(0x51);
 5676   emit_operand(dst, src);
 5677 }
 5678 
 5679 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5680   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5681   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5682   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5683   emit_int16(0x51, (0xC0 | encode));
 5684 }
 5685 
 5686 void Assembler::std() {
 5687   emit_int8((unsigned char)0xFD);
 5688 }
 5689 
 5690 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5691   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5692   InstructionMark im(this);
 5693   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5694   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5695   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5696   emit_int8(0x51);
 5697   emit_operand(dst, src);
 5698 }
 5699 
 5700 void Assembler::stmxcsr( Address dst) {
 5701   if (UseAVX > 0 ) {
 5702     assert(VM_Version::supports_avx(), "");
 5703     InstructionMark im(this);
 5704     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5705     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5706     emit_int8((unsigned char)0xAE);
 5707     emit_operand(as_Register(3), dst);
 5708   } else {
 5709     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5710     InstructionMark im(this);
 5711     prefix(dst);
 5712     emit_int16(0x0F, (unsigned char)0xAE);
 5713     emit_operand(as_Register(3), dst);
 5714   }
 5715 }
 5716 
 5717 void Assembler::subl(Address dst, int32_t imm32) {
 5718   InstructionMark im(this);
 5719   prefix(dst);
 5720   emit_arith_operand(0x81, rbp, dst, imm32);
 5721 }
 5722 
 5723 void Assembler::subl(Address dst, Register src) {
 5724   InstructionMark im(this);
 5725   prefix(dst, src);
 5726   emit_int8(0x29);
 5727   emit_operand(src, dst);
 5728 }
 5729 
 5730 void Assembler::subl(Register dst, int32_t imm32) {
 5731   prefix(dst);
 5732   emit_arith(0x81, 0xE8, dst, imm32);
 5733 }
 5734 
 5735 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5736 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5737   prefix(dst);
 5738   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5739 }
 5740 
 5741 void Assembler::subl(Register dst, Address src) {
 5742   InstructionMark im(this);
 5743   prefix(src, dst);
 5744   emit_int8(0x2B);
 5745   emit_operand(dst, src);
 5746 }
 5747 
 5748 void Assembler::subl(Register dst, Register src) {
 5749   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5750   emit_arith(0x2B, 0xC0, dst, src);
 5751 }
 5752 
 5753 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5754   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5755   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5756   attributes.set_rex_vex_w_reverted();
 5757   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5758   emit_int16(0x5C, (0xC0 | encode));
 5759 }
 5760 
 5761 void Assembler::subsd(XMMRegister dst, Address src) {
 5762   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5763   InstructionMark im(this);
 5764   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5765   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5766   attributes.set_rex_vex_w_reverted();
 5767   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5768   emit_int8(0x5C);
 5769   emit_operand(dst, src);
 5770 }
 5771 
 5772 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5773   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5774   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5775   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5776   emit_int16(0x5C, (0xC0 | encode));
 5777 }
 5778 
 5779 void Assembler::subss(XMMRegister dst, Address src) {
 5780   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5781   InstructionMark im(this);
 5782   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5783   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5784   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5785   emit_int8(0x5C);
 5786   emit_operand(dst, src);
 5787 }
 5788 
 5789 void Assembler::testb(Register dst, int imm8) {
 5790   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5791   (void) prefix_and_encode(dst->encoding(), true);
 5792   emit_arith_b(0xF6, 0xC0, dst, imm8);
 5793 }
 5794 
 5795 void Assembler::testb(Address dst, int imm8) {
 5796   InstructionMark im(this);
 5797   prefix(dst);
 5798   emit_int8((unsigned char)0xF6);
 5799   emit_operand(rax, dst, 1);
 5800   emit_int8(imm8);
 5801 }
 5802 
 5803 void Assembler::testl(Register dst, int32_t imm32) {
 5804   // not using emit_arith because test
 5805   // doesn't support sign-extension of
 5806   // 8bit operands
 5807   int encode = dst->encoding();
 5808   encode = prefix_and_encode(encode);
 5809   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5810   emit_int32(imm32);
 5811 }
 5812 
 5813 void Assembler::testl(Register dst, Register src) {
 5814   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5815   emit_arith(0x85, 0xC0, dst, src);
 5816 }
 5817 
 5818 void Assembler::testl(Register dst, Address src) {
 5819   InstructionMark im(this);
 5820   prefix(src, dst);
 5821   emit_int8((unsigned char)0x85);
 5822   emit_operand(dst, src);
 5823 }
 5824 
 5825 void Assembler::tzcntl(Register dst, Register src) {
 5826   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5827   emit_int8((unsigned char)0xF3);
 5828   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5829   emit_int24(0x0F,
 5830              (unsigned char)0xBC,
 5831              0xC0 | encode);
 5832 }
 5833 
 5834 void Assembler::tzcntq(Register dst, Register src) {
 5835   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5836   emit_int8((unsigned char)0xF3);
 5837   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5838   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5839 }
 5840 
 5841 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5842   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5843   InstructionMark im(this);
 5844   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5845   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5846   attributes.set_rex_vex_w_reverted();
 5847   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5848   emit_int8(0x2E);
 5849   emit_operand(dst, src);
 5850 }
 5851 
 5852 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 5853   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5854   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5855   attributes.set_rex_vex_w_reverted();
 5856   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5857   emit_int16(0x2E, (0xC0 | encode));
 5858 }
 5859 
 5860 void Assembler::ucomiss(XMMRegister dst, Address src) {
 5861   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5862   InstructionMark im(this);
 5863   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5864   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5865   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5866   emit_int8(0x2E);
 5867   emit_operand(dst, src);
 5868 }
 5869 
 5870 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 5871   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5873   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5874   emit_int16(0x2E, (0xC0 | encode));
 5875 }
 5876 
 5877 void Assembler::xabort(int8_t imm8) {
 5878   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 5879 }
 5880 
 5881 void Assembler::xaddb(Address dst, Register src) {
 5882   InstructionMark im(this);
 5883   prefix(dst, src, true);
 5884   emit_int16(0x0F, (unsigned char)0xC0);
 5885   emit_operand(src, dst);
 5886 }
 5887 
 5888 void Assembler::xaddw(Address dst, Register src) {
 5889   InstructionMark im(this);
 5890   emit_int8(0x66);
 5891   prefix(dst, src);
 5892   emit_int16(0x0F, (unsigned char)0xC1);
 5893   emit_operand(src, dst);
 5894 }
 5895 
 5896 void Assembler::xaddl(Address dst, Register src) {
 5897   InstructionMark im(this);
 5898   prefix(dst, src);
 5899   emit_int16(0x0F, (unsigned char)0xC1);
 5900   emit_operand(src, dst);
 5901 }
 5902 
 5903 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 5904   InstructionMark im(this);
 5905   relocate(rtype);
 5906   if (abort.is_bound()) {
 5907     address entry = target(abort);
 5908     assert(entry != NULL, "abort entry NULL");
 5909     intptr_t offset = entry - pc();
 5910     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5911     emit_int32(offset - 6); // 2 opcode + 4 address
 5912   } else {
 5913     abort.add_patch_at(code(), locator());
 5914     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5915     emit_int32(0);
 5916   }
 5917 }
 5918 
 5919 void Assembler::xchgb(Register dst, Address src) { // xchg
 5920   InstructionMark im(this);
 5921   prefix(src, dst, true);
 5922   emit_int8((unsigned char)0x86);
 5923   emit_operand(dst, src);
 5924 }
 5925 
 5926 void Assembler::xchgw(Register dst, Address src) { // xchg
 5927   InstructionMark im(this);
 5928   emit_int8(0x66);
 5929   prefix(src, dst);
 5930   emit_int8((unsigned char)0x87);
 5931   emit_operand(dst, src);
 5932 }
 5933 
 5934 void Assembler::xchgl(Register dst, Address src) { // xchg
 5935   InstructionMark im(this);
 5936   prefix(src, dst);
 5937   emit_int8((unsigned char)0x87);
 5938   emit_operand(dst, src);
 5939 }
 5940 
 5941 void Assembler::xchgl(Register dst, Register src) {
 5942   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5943   emit_int16((unsigned char)0x87, (0xC0 | encode));
 5944 }
 5945 
 5946 void Assembler::xend() {
 5947   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 5948 }
 5949 
 5950 void Assembler::xgetbv() {
 5951   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 5952 }
 5953 
 5954 void Assembler::xorl(Address dst, int32_t imm32) {
 5955   InstructionMark im(this);
 5956   prefix(dst);
 5957   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 5958 }
 5959 
 5960 void Assembler::xorl(Register dst, int32_t imm32) {
 5961   prefix(dst);
 5962   emit_arith(0x81, 0xF0, dst, imm32);
 5963 }
 5964 
 5965 void Assembler::xorl(Register dst, Address src) {
 5966   InstructionMark im(this);
 5967   prefix(src, dst);
 5968   emit_int8(0x33);
 5969   emit_operand(dst, src);
 5970 }
 5971 
 5972 void Assembler::xorl(Register dst, Register src) {
 5973   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5974   emit_arith(0x33, 0xC0, dst, src);
 5975 }
 5976 
 5977 void Assembler::xorl(Address dst, Register src) {
 5978   InstructionMark im(this);
 5979   prefix(dst, src);
 5980   emit_int8(0x31);
 5981   emit_operand(src, dst);
 5982 }
 5983 
 5984 void Assembler::xorb(Register dst, Address src) {
 5985   InstructionMark im(this);
 5986   prefix(src, dst);
 5987   emit_int8(0x32);
 5988   emit_operand(dst, src);
 5989 }
 5990 
 5991 void Assembler::xorb(Address dst, Register src) {
 5992   InstructionMark im(this);
 5993   prefix(dst, src, true);
 5994   emit_int8(0x30);
 5995   emit_operand(src, dst);
 5996 }
 5997 
 5998 void Assembler::xorw(Register dst, Register src) {
 5999   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6000   emit_arith(0x33, 0xC0, dst, src);
 6001 }
 6002 
 6003 // AVX 3-operands scalar float-point arithmetic instructions
 6004 
 6005 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6006   assert(VM_Version::supports_avx(), "");
 6007   InstructionMark im(this);
 6008   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6009   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6010   attributes.set_rex_vex_w_reverted();
 6011   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6012   emit_int8(0x58);
 6013   emit_operand(dst, src);
 6014 }
 6015 
 6016 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6017   assert(VM_Version::supports_avx(), "");
 6018   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6019   attributes.set_rex_vex_w_reverted();
 6020   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6021   emit_int16(0x58, (0xC0 | encode));
 6022 }
 6023 
 6024 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6025   assert(VM_Version::supports_avx(), "");
 6026   InstructionMark im(this);
 6027   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6028   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6029   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6030   emit_int8(0x58);
 6031   emit_operand(dst, src);
 6032 }
 6033 
 6034 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6035   assert(VM_Version::supports_avx(), "");
 6036   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6037   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6038   emit_int16(0x58, (0xC0 | encode));
 6039 }
 6040 
 6041 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6042   assert(VM_Version::supports_avx(), "");
 6043   InstructionMark im(this);
 6044   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6045   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6046   attributes.set_rex_vex_w_reverted();
 6047   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6048   emit_int8(0x5E);
 6049   emit_operand(dst, src);
 6050 }
 6051 
 6052 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6053   assert(VM_Version::supports_avx(), "");
 6054   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6055   attributes.set_rex_vex_w_reverted();
 6056   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6057   emit_int16(0x5E, (0xC0 | encode));
 6058 }
 6059 
 6060 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6061   assert(VM_Version::supports_avx(), "");
 6062   InstructionMark im(this);
 6063   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6064   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6065   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6066   emit_int8(0x5E);
 6067   emit_operand(dst, src);
 6068 }
 6069 
 6070 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6071   assert(VM_Version::supports_avx(), "");
 6072   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6073   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6074   emit_int16(0x5E, (0xC0 | encode));
 6075 }
 6076 
 6077 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6078   assert(VM_Version::supports_fma(), "");
 6079   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6080   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6081   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6082 }
 6083 
 6084 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6085   assert(VM_Version::supports_fma(), "");
 6086   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6087   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6088   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6089 }
 6090 
 6091 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6092   assert(VM_Version::supports_avx(), "");
 6093   InstructionMark im(this);
 6094   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6095   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6096   attributes.set_rex_vex_w_reverted();
 6097   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6098   emit_int8(0x59);
 6099   emit_operand(dst, src);
 6100 }
 6101 
 6102 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6103   assert(VM_Version::supports_avx(), "");
 6104   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6105   attributes.set_rex_vex_w_reverted();
 6106   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6107   emit_int16(0x59, (0xC0 | encode));
 6108 }
 6109 
 6110 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6111   assert(VM_Version::supports_avx(), "");
 6112   InstructionMark im(this);
 6113   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6114   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6115   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6116   emit_int8(0x59);
 6117   emit_operand(dst, src);
 6118 }
 6119 
 6120 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6121   assert(VM_Version::supports_avx(), "");
 6122   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6123   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6124   emit_int16(0x59, (0xC0 | encode));
 6125 }
 6126 
 6127 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6128   assert(VM_Version::supports_avx(), "");
 6129   InstructionMark im(this);
 6130   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6131   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6132   attributes.set_rex_vex_w_reverted();
 6133   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6134   emit_int8(0x5C);
 6135   emit_operand(dst, src);
 6136 }
 6137 
 6138 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6139   assert(VM_Version::supports_avx(), "");
 6140   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6141   attributes.set_rex_vex_w_reverted();
 6142   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6143   emit_int16(0x5C, (0xC0 | encode));
 6144 }
 6145 
 6146 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6147   assert(VM_Version::supports_avx(), "");
 6148   InstructionMark im(this);
 6149   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6150   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6151   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6152   emit_int8(0x5C);
 6153   emit_operand(dst, src);
 6154 }
 6155 
 6156 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6157   assert(VM_Version::supports_avx(), "");
 6158   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6160   emit_int16(0x5C, (0xC0 | encode));
 6161 }
 6162 
 6163 //====================VECTOR ARITHMETIC=====================================
 6164 
 6165 // Float-point vector arithmetic
 6166 
 6167 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6169   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6170   attributes.set_rex_vex_w_reverted();
 6171   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6172   emit_int16(0x58, (0xC0 | encode));
 6173 }
 6174 
 6175 void Assembler::addpd(XMMRegister dst, Address src) {
 6176   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6177   InstructionMark im(this);
 6178   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6179   attributes.set_rex_vex_w_reverted();
 6180   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6181   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6182   emit_int8(0x58);
 6183   emit_operand(dst, src);
 6184 }
 6185 
 6186 
 6187 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6188   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6189   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6190   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6191   emit_int16(0x58, (0xC0 | encode));
 6192 }
 6193 
 6194 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6195   assert(VM_Version::supports_avx(), "");
 6196   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6197   attributes.set_rex_vex_w_reverted();
 6198   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6199   emit_int16(0x58, (0xC0 | encode));
 6200 }
 6201 
 6202 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6203   assert(VM_Version::supports_avx(), "");
 6204   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6205   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6206   emit_int16(0x58, (0xC0 | encode));
 6207 }
 6208 
 6209 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6210   assert(VM_Version::supports_avx(), "");
 6211   InstructionMark im(this);
 6212   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6213   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6214   attributes.set_rex_vex_w_reverted();
 6215   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6216   emit_int8(0x58);
 6217   emit_operand(dst, src);
 6218 }
 6219 
 6220 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6221   assert(VM_Version::supports_avx(), "");
 6222   InstructionMark im(this);
 6223   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6224   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6225   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6226   emit_int8(0x58);
 6227   emit_operand(dst, src);
 6228 }
 6229 
 6230 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6231   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6232   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6233   attributes.set_rex_vex_w_reverted();
 6234   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6235   emit_int16(0x5C, (0xC0 | encode));
 6236 }
 6237 
 6238 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6239   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6240   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6241   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6242   emit_int16(0x5C, (0xC0 | encode));
 6243 }
 6244 
 6245 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6246   assert(VM_Version::supports_avx(), "");
 6247   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6248   attributes.set_rex_vex_w_reverted();
 6249   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6250   emit_int16(0x5C, (0xC0 | encode));
 6251 }
 6252 
 6253 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6254   assert(VM_Version::supports_avx(), "");
 6255   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6256   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6257   emit_int16(0x5C, (0xC0 | encode));
 6258 }
 6259 
 6260 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6261   assert(VM_Version::supports_avx(), "");
 6262   InstructionMark im(this);
 6263   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6264   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6265   attributes.set_rex_vex_w_reverted();
 6266   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6267   emit_int8(0x5C);
 6268   emit_operand(dst, src);
 6269 }
 6270 
 6271 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6272   assert(VM_Version::supports_avx(), "");
 6273   InstructionMark im(this);
 6274   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6275   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6276   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6277   emit_int8(0x5C);
 6278   emit_operand(dst, src);
 6279 }
 6280 
 6281 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6282   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6283   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6284   attributes.set_rex_vex_w_reverted();
 6285   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6286   emit_int16(0x59, (0xC0 | encode));
 6287 }
 6288 
 6289 void Assembler::mulpd(XMMRegister dst, Address src) {
 6290   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6291   InstructionMark im(this);
 6292   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6293   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6294   attributes.set_rex_vex_w_reverted();
 6295   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6296   emit_int8(0x59);
 6297   emit_operand(dst, src);
 6298 }
 6299 
 6300 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6301   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6303   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6304   emit_int16(0x59, (0xC0 | encode));
 6305 }
 6306 
 6307 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6308   assert(VM_Version::supports_avx(), "");
 6309   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6310   attributes.set_rex_vex_w_reverted();
 6311   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6312   emit_int16(0x59, (0xC0 | encode));
 6313 }
 6314 
 6315 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6316   assert(VM_Version::supports_avx(), "");
 6317   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6319   emit_int16(0x59, (0xC0 | encode));
 6320 }
 6321 
 6322 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6323   assert(VM_Version::supports_avx(), "");
 6324   InstructionMark im(this);
 6325   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6326   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6327   attributes.set_rex_vex_w_reverted();
 6328   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6329   emit_int8(0x59);
 6330   emit_operand(dst, src);
 6331 }
 6332 
 6333 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6334   assert(VM_Version::supports_avx(), "");
 6335   InstructionMark im(this);
 6336   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6337   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6338   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6339   emit_int8(0x59);
 6340   emit_operand(dst, src);
 6341 }
 6342 
 6343 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6344   assert(VM_Version::supports_fma(), "");
 6345   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6346   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6347   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6348 }
 6349 
 6350 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6351   assert(VM_Version::supports_fma(), "");
 6352   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6353   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6354   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6355 }
 6356 
 6357 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6358   assert(VM_Version::supports_fma(), "");
 6359   InstructionMark im(this);
 6360   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6361   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6362   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6363   emit_int8((unsigned char)0xB8);
 6364   emit_operand(dst, src2);
 6365 }
 6366 
 6367 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6368   assert(VM_Version::supports_fma(), "");
 6369   InstructionMark im(this);
 6370   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6371   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6372   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6373   emit_int8((unsigned char)0xB8);
 6374   emit_operand(dst, src2);
 6375 }
 6376 
 6377 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6379   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6380   attributes.set_rex_vex_w_reverted();
 6381   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6382   emit_int16(0x5E, (0xC0 | encode));
 6383 }
 6384 
 6385 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6386   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6387   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6388   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6389   emit_int16(0x5E, (0xC0 | encode));
 6390 }
 6391 
 6392 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6393   assert(VM_Version::supports_avx(), "");
 6394   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6395   attributes.set_rex_vex_w_reverted();
 6396   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6397   emit_int16(0x5E, (0xC0 | encode));
 6398 }
 6399 
 6400 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6401   assert(VM_Version::supports_avx(), "");
 6402   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6403   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6404   emit_int16(0x5E, (0xC0 | encode));
 6405 }
 6406 
 6407 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6408   assert(VM_Version::supports_avx(), "");
 6409   InstructionMark im(this);
 6410   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6411   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6412   attributes.set_rex_vex_w_reverted();
 6413   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6414   emit_int8(0x5E);
 6415   emit_operand(dst, src);
 6416 }
 6417 
 6418 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6419   assert(VM_Version::supports_avx(), "");
 6420   InstructionMark im(this);
 6421   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6422   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6423   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6424   emit_int8(0x5E);
 6425   emit_operand(dst, src);
 6426 }
 6427 
 6428 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6429   assert(VM_Version::supports_avx(), "");
 6430   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6431   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6432   emit_int24(0x09, (0xC0 | encode), (rmode));
 6433 }
 6434 
 6435 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6436   assert(VM_Version::supports_avx(), "");
 6437   InstructionMark im(this);
 6438   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6439   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6440   emit_int8(0x09);
 6441   emit_operand(dst, src);
 6442   emit_int8((rmode));
 6443 }
 6444 
 6445 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6446   assert(VM_Version::supports_evex(), "requires EVEX support");
 6447   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6448   attributes.set_is_evex_instruction();
 6449   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6450   emit_int24(0x09, (0xC0 | encode), (rmode));
 6451 }
 6452 
 6453 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6454   assert(VM_Version::supports_evex(), "requires EVEX support");
 6455   assert(dst != xnoreg, "sanity");
 6456   InstructionMark im(this);
 6457   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6458   attributes.set_is_evex_instruction();
 6459   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6460   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6461   emit_int8(0x09);
 6462   emit_operand(dst, src);
 6463   emit_int8((rmode));
 6464 }
 6465 
 6466 
 6467 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6468   assert(VM_Version::supports_avx(), "");
 6469   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6470   attributes.set_rex_vex_w_reverted();
 6471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6472   emit_int16(0x51, (0xC0 | encode));
 6473 }
 6474 
 6475 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6476   assert(VM_Version::supports_avx(), "");
 6477   InstructionMark im(this);
 6478   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6479   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6480   attributes.set_rex_vex_w_reverted();
 6481   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6482   emit_int8(0x51);
 6483   emit_operand(dst, src);
 6484 }
 6485 
 6486 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6487   assert(VM_Version::supports_avx(), "");
 6488   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6489   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6490   emit_int16(0x51, (0xC0 | encode));
 6491 }
 6492 
 6493 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6494   assert(VM_Version::supports_avx(), "");
 6495   InstructionMark im(this);
 6496   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6497   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6498   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6499   emit_int8(0x51);
 6500   emit_operand(dst, src);
 6501 }
 6502 
 6503 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6505   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6506   attributes.set_rex_vex_w_reverted();
 6507   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6508   emit_int16(0x54, (0xC0 | encode));
 6509 }
 6510 
 6511 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6512   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6513   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6514   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6515   emit_int16(0x54, (0xC0 | encode));
 6516 }
 6517 
 6518 void Assembler::andps(XMMRegister dst, Address src) {
 6519   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6520   InstructionMark im(this);
 6521   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6522   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6523   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6524   emit_int8(0x54);
 6525   emit_operand(dst, src);
 6526 }
 6527 
 6528 void Assembler::andpd(XMMRegister dst, Address src) {
 6529   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6530   InstructionMark im(this);
 6531   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6532   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6533   attributes.set_rex_vex_w_reverted();
 6534   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6535   emit_int8(0x54);
 6536   emit_operand(dst, src);
 6537 }
 6538 
 6539 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6540   assert(VM_Version::supports_avx(), "");
 6541   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6542   attributes.set_rex_vex_w_reverted();
 6543   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6544   emit_int16(0x54, (0xC0 | encode));
 6545 }
 6546 
 6547 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6548   assert(VM_Version::supports_avx(), "");
 6549   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6550   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6551   emit_int16(0x54, (0xC0 | encode));
 6552 }
 6553 
 6554 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6555   assert(VM_Version::supports_avx(), "");
 6556   InstructionMark im(this);
 6557   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6558   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6559   attributes.set_rex_vex_w_reverted();
 6560   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6561   emit_int8(0x54);
 6562   emit_operand(dst, src);
 6563 }
 6564 
 6565 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6566   assert(VM_Version::supports_avx(), "");
 6567   InstructionMark im(this);
 6568   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6569   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6570   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6571   emit_int8(0x54);
 6572   emit_operand(dst, src);
 6573 }
 6574 
 6575 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6576   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6577   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6578   attributes.set_rex_vex_w_reverted();
 6579   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6580   emit_int8(0x15);
 6581   emit_int8((0xC0 | encode));
 6582 }
 6583 
 6584 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6585   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6586   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6587   attributes.set_rex_vex_w_reverted();
 6588   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6589   emit_int16(0x14, (0xC0 | encode));
 6590 }
 6591 
 6592 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6593   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6594   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6595   attributes.set_rex_vex_w_reverted();
 6596   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6597   emit_int16(0x57, (0xC0 | encode));
 6598 }
 6599 
 6600 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6601   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6602   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6603   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6604   emit_int16(0x57, (0xC0 | encode));
 6605 }
 6606 
 6607 void Assembler::xorpd(XMMRegister dst, Address src) {
 6608   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6609   InstructionMark im(this);
 6610   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6611   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6612   attributes.set_rex_vex_w_reverted();
 6613   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6614   emit_int8(0x57);
 6615   emit_operand(dst, src);
 6616 }
 6617 
 6618 void Assembler::xorps(XMMRegister dst, Address src) {
 6619   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6620   InstructionMark im(this);
 6621   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6622   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6623   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6624   emit_int8(0x57);
 6625   emit_operand(dst, src);
 6626 }
 6627 
 6628 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6629   assert(VM_Version::supports_avx(), "");
 6630   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6631   attributes.set_rex_vex_w_reverted();
 6632   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6633   emit_int16(0x57, (0xC0 | encode));
 6634 }
 6635 
 6636 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6637   assert(VM_Version::supports_avx(), "");
 6638   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6639   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6640   emit_int16(0x57, (0xC0 | encode));
 6641 }
 6642 
 6643 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6644   assert(VM_Version::supports_avx(), "");
 6645   InstructionMark im(this);
 6646   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6647   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6648   attributes.set_rex_vex_w_reverted();
 6649   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6650   emit_int8(0x57);
 6651   emit_operand(dst, src);
 6652 }
 6653 
 6654 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6655   assert(VM_Version::supports_avx(), "");
 6656   InstructionMark im(this);
 6657   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6658   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6659   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6660   emit_int8(0x57);
 6661   emit_operand(dst, src);
 6662 }
 6663 
 6664 // Integer vector arithmetic
 6665 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6666   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6667          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6670   emit_int16(0x01, (0xC0 | encode));
 6671 }
 6672 
 6673 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6674   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6675          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6678   emit_int16(0x02, (0xC0 | encode));
 6679 }
 6680 
 6681 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6682   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6683   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6684   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6685   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6686 }
 6687 
 6688 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6689   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6691   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6692   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6693 }
 6694 
 6695 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6696   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6698   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6699   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6700 }
 6701 
 6702 void Assembler::paddd(XMMRegister dst, Address src) {
 6703   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6704   InstructionMark im(this);
 6705   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6706   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6707   emit_int8((unsigned char)0xFE);
 6708   emit_operand(dst, src);
 6709 }
 6710 
 6711 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6712   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6713   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6714   attributes.set_rex_vex_w_reverted();
 6715   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6716   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6717 }
 6718 
 6719 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6720   assert(VM_Version::supports_sse3(), "");
 6721   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6722   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6723   emit_int16(0x01, (0xC0 | encode));
 6724 }
 6725 
 6726 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6727   assert(VM_Version::supports_sse3(), "");
 6728   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6729   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6730   emit_int16(0x02, (0xC0 | encode));
 6731 }
 6732 
 6733 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6734   assert(UseAVX > 0, "requires some form of AVX");
 6735   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6736   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6737   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6738 }
 6739 
 6740 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6741   assert(UseAVX > 0, "requires some form of AVX");
 6742   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6743   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6744   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6745 }
 6746 
 6747 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6748   assert(UseAVX > 0, "requires some form of AVX");
 6749   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6750   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6751   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6752 }
 6753 
 6754 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6755   assert(UseAVX > 0, "requires some form of AVX");
 6756   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6757   attributes.set_rex_vex_w_reverted();
 6758   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6759   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6760 }
 6761 
 6762 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6763   assert(UseAVX > 0, "requires some form of AVX");
 6764   InstructionMark im(this);
 6765   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6766   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6767   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6768   emit_int8((unsigned char)0xFC);
 6769   emit_operand(dst, src);
 6770 }
 6771 
 6772 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6773   assert(UseAVX > 0, "requires some form of AVX");
 6774   InstructionMark im(this);
 6775   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6776   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6777   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6778   emit_int8((unsigned char)0xFD);
 6779   emit_operand(dst, src);
 6780 }
 6781 
 6782 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6783   assert(UseAVX > 0, "requires some form of AVX");
 6784   InstructionMark im(this);
 6785   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6786   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6787   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6788   emit_int8((unsigned char)0xFE);
 6789   emit_operand(dst, src);
 6790 }
 6791 
 6792 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6793   assert(UseAVX > 0, "requires some form of AVX");
 6794   InstructionMark im(this);
 6795   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6796   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6797   attributes.set_rex_vex_w_reverted();
 6798   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6799   emit_int8((unsigned char)0xD4);
 6800   emit_operand(dst, src);
 6801 }
 6802 
 6803 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6804   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6805   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6806   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6807   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6808 }
 6809 
 6810 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6811   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6813   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6814   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6815 }
 6816 
 6817 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6818   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6819   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6820   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6821 }
 6822 
 6823 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6824   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6825   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6826   attributes.set_rex_vex_w_reverted();
 6827   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6828   emit_int8((unsigned char)0xFB);
 6829   emit_int8((0xC0 | encode));
 6830 }
 6831 
 6832 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6833   assert(UseAVX > 0, "requires some form of AVX");
 6834   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6835   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6836   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6837 }
 6838 
 6839 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6840   assert(UseAVX > 0, "requires some form of AVX");
 6841   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6842   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6843   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6844 }
 6845 
 6846 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6847   assert(UseAVX > 0, "requires some form of AVX");
 6848   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6849   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6850   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6851 }
 6852 
 6853 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6854   assert(UseAVX > 0, "requires some form of AVX");
 6855   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6856   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6857   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6858 }
 6859 
 6860 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6861   assert(UseAVX > 0, "requires some form of AVX");
 6862   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6863   attributes.set_rex_vex_w_reverted();
 6864   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6865   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 6866 }
 6867 
 6868 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6869   assert(UseAVX > 0, "requires some form of AVX");
 6870   InstructionMark im(this);
 6871   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6872   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6873   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6874   emit_int8((unsigned char)0xF8);
 6875   emit_operand(dst, src);
 6876 }
 6877 
 6878 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6879   assert(UseAVX > 0, "requires some form of AVX");
 6880   InstructionMark im(this);
 6881   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6882   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6883   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6884   emit_int8((unsigned char)0xF9);
 6885   emit_operand(dst, src);
 6886 }
 6887 
 6888 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6889   assert(UseAVX > 0, "requires some form of AVX");
 6890   InstructionMark im(this);
 6891   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6892   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6893   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6894   emit_int8((unsigned char)0xFA);
 6895   emit_operand(dst, src);
 6896 }
 6897 
 6898 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6899   assert(UseAVX > 0, "requires some form of AVX");
 6900   InstructionMark im(this);
 6901   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6902   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6903   attributes.set_rex_vex_w_reverted();
 6904   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6905   emit_int8((unsigned char)0xFB);
 6906   emit_operand(dst, src);
 6907 }
 6908 
 6909 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 6910   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6911   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6912   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6913   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6914 }
 6915 
 6916 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 6917   assert(VM_Version::supports_sse4_1(), "");
 6918   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6919   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6920   emit_int16(0x40, (0xC0 | encode));
 6921 }
 6922 
 6923 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 6924   assert(VM_Version::supports_sse2(), "");
 6925   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6926   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6927   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6928 }
 6929 
 6930 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6931   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 6932          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 6933          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 6934   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6935   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6936   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 6937 }
 6938 
 6939 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6940   assert(UseAVX > 0, "requires some form of AVX");
 6941   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6942   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6943   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6944 }
 6945 
 6946 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6947   assert(UseAVX > 0, "requires some form of AVX");
 6948   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6949   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6950   emit_int16(0x40, (0xC0 | encode));
 6951 }
 6952 
 6953 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6954   assert(UseAVX > 2, "requires some form of EVEX");
 6955   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6956   attributes.set_is_evex_instruction();
 6957   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6958   emit_int16(0x40, (0xC0 | encode));
 6959 }
 6960 
 6961 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6962   assert(UseAVX > 0, "requires some form of AVX");
 6963   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6964   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6965   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6966 }
 6967 
 6968 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6969   assert(UseAVX > 0, "requires some form of AVX");
 6970   InstructionMark im(this);
 6971   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6972   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6973   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6974   emit_int8((unsigned char)0xD5);
 6975   emit_operand(dst, src);
 6976 }
 6977 
 6978 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6979   assert(UseAVX > 0, "requires some form of AVX");
 6980   InstructionMark im(this);
 6981   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6982   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6983   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6984   emit_int8(0x40);
 6985   emit_operand(dst, src);
 6986 }
 6987 
 6988 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6989   assert(UseAVX > 2, "requires some form of EVEX");
 6990   InstructionMark im(this);
 6991   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6992   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6993   attributes.set_is_evex_instruction();
 6994   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6995   emit_int8(0x40);
 6996   emit_operand(dst, src);
 6997 }
 6998 
 6999 // Min, max
 7000 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7001   assert(VM_Version::supports_sse4_1(), "");
 7002   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7003   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7004   emit_int16(0x38, (0xC0 | encode));
 7005 }
 7006 
 7007 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7008   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7009         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7010   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7011   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7012   emit_int16(0x38, (0xC0 | encode));
 7013 }
 7014 
 7015 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7016   assert(VM_Version::supports_sse2(), "");
 7017   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7018   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7019   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7020 }
 7021 
 7022 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7023   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7024         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7026   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7027   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7028 }
 7029 
 7030 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7031   assert(VM_Version::supports_sse4_1(), "");
 7032   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7033   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7034   emit_int16(0x39, (0xC0 | encode));
 7035 }
 7036 
 7037 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7038   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7039         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7040   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7041   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7042   emit_int16(0x39, (0xC0 | encode));
 7043 }
 7044 
 7045 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7046   assert(UseAVX > 2, "requires AVX512F");
 7047   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7048   attributes.set_is_evex_instruction();
 7049   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7050   emit_int16(0x39, (0xC0 | encode));
 7051 }
 7052 
 7053 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7054   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7055   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7056   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7057   emit_int16(0x5D, (0xC0 | encode));
 7058 }
 7059 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7060   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7061   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7062   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7063   emit_int16(0x5D, (0xC0 | encode));
 7064 }
 7065 
 7066 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7067   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7068   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7069   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7070   emit_int16(0x5D, (0xC0 | encode));
 7071 }
 7072 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7073   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7074   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7075   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7076   emit_int16(0x5D, (0xC0 | encode));
 7077 }
 7078 
 7079 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7080   assert(VM_Version::supports_sse4_1(), "");
 7081   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7082   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7083   emit_int16(0x3C, (0xC0 | encode));
 7084 }
 7085 
 7086 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7087   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7088         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7089   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7090   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7091   emit_int16(0x3C, (0xC0 | encode));
 7092 }
 7093 
 7094 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7095   assert(VM_Version::supports_sse2(), "");
 7096   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7097   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7098   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7099 }
 7100 
 7101 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7102   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7103         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7106   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7107 }
 7108 
 7109 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7110   assert(VM_Version::supports_sse4_1(), "");
 7111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7112   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7113   emit_int16(0x3D, (0xC0 | encode));
 7114 }
 7115 
 7116 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7117   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7118         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7120   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7121   emit_int16(0x3D, (0xC0 | encode));
 7122 }
 7123 
 7124 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7125   assert(UseAVX > 2, "requires AVX512F");
 7126   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7127   attributes.set_is_evex_instruction();
 7128   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7129   emit_int16(0x3D, (0xC0 | encode));
 7130 }
 7131 
 7132 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7133   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7134   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7135   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7136   emit_int16(0x5F, (0xC0 | encode));
 7137 }
 7138 
 7139 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7140   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7142   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7143   emit_int16(0x5F, (0xC0 | encode));
 7144 }
 7145 
 7146 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7147   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7148   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7149   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7150   emit_int16(0x5F, (0xC0 | encode));
 7151 }
 7152 
 7153 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7154   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7155   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7156   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7157   emit_int16(0x5F, (0xC0 | encode));
 7158 }
 7159 
 7160 // Shift packed integers left by specified number of bits.
 7161 void Assembler::psllw(XMMRegister dst, int shift) {
 7162   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7163   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7164   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7165   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7166   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7167 }
 7168 
 7169 void Assembler::pslld(XMMRegister dst, int shift) {
 7170   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7171   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7172   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7173   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7174   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7175 }
 7176 
 7177 void Assembler::psllq(XMMRegister dst, int shift) {
 7178   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7179   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7180   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7181   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7182   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7183 }
 7184 
 7185 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7186   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7187   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7188   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7189   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7190 }
 7191 
 7192 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7193   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7194   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7195   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7196   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7197 }
 7198 
 7199 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7201   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7202   attributes.set_rex_vex_w_reverted();
 7203   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7204   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7205 }
 7206 
 7207 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7208   assert(UseAVX > 0, "requires some form of AVX");
 7209   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7210   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7211   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7212   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7213 }
 7214 
 7215 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7216   assert(UseAVX > 0, "requires some form of AVX");
 7217   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7218   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7219   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7220   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7221   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7222 }
 7223 
 7224 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7225   assert(UseAVX > 0, "requires some form of AVX");
 7226   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7227   attributes.set_rex_vex_w_reverted();
 7228   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7229   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7230   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7231 }
 7232 
 7233 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7234   assert(UseAVX > 0, "requires some form of AVX");
 7235   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7236   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7237   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7238 }
 7239 
 7240 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7241   assert(UseAVX > 0, "requires some form of AVX");
 7242   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7243   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7244   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7245 }
 7246 
 7247 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7248   assert(UseAVX > 0, "requires some form of AVX");
 7249   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7250   attributes.set_rex_vex_w_reverted();
 7251   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7252   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7253 }
 7254 
 7255 // Shift packed integers logically right by specified number of bits.
 7256 void Assembler::psrlw(XMMRegister dst, int shift) {
 7257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7258   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7259   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7260   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7261   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7262 }
 7263 
 7264 void Assembler::psrld(XMMRegister dst, int shift) {
 7265   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7267   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7268   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7269   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7270 }
 7271 
 7272 void Assembler::psrlq(XMMRegister dst, int shift) {
 7273   // Do not confuse it with psrldq SSE2 instruction which
 7274   // shifts 128 bit value in xmm register by number of bytes.
 7275   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7276   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7277   attributes.set_rex_vex_w_reverted();
 7278   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7279   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7280   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7281 }
 7282 
 7283 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7285   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7286   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7287   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7288 }
 7289 
 7290 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7291   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7292   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7293   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7294   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7295 }
 7296 
 7297 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7298   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7299   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7300   attributes.set_rex_vex_w_reverted();
 7301   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7302   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7303 }
 7304 
 7305 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7306   assert(UseAVX > 0, "requires some form of AVX");
 7307   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7308   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7309   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7310   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7311 }
 7312 
 7313 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7314   assert(UseAVX > 0, "requires some form of AVX");
 7315   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7316   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7317   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7318   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7319 }
 7320 
 7321 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7322   assert(UseAVX > 0, "requires some form of AVX");
 7323   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7324   attributes.set_rex_vex_w_reverted();
 7325   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7326   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7327   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7328 }
 7329 
 7330 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7331   assert(UseAVX > 0, "requires some form of AVX");
 7332   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7333   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7334   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7335 }
 7336 
 7337 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7338   assert(UseAVX > 0, "requires some form of AVX");
 7339   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7340   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7341   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7342 }
 7343 
 7344 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7345   assert(UseAVX > 0, "requires some form of AVX");
 7346   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7347   attributes.set_rex_vex_w_reverted();
 7348   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7349   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7350 }
 7351 
 7352 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7353   assert(VM_Version::supports_avx512bw(), "");
 7354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7355   attributes.set_is_evex_instruction();
 7356   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7357   emit_int16(0x10, (0xC0 | encode));
 7358 }
 7359 
 7360 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7361   assert(VM_Version::supports_avx512bw(), "");
 7362   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7363   attributes.set_is_evex_instruction();
 7364   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7365   emit_int16(0x12, (0xC0 | encode));
 7366 }
 7367 
 7368 // Shift packed integers arithmetically right by specified number of bits.
 7369 void Assembler::psraw(XMMRegister dst, int shift) {
 7370   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7372   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7373   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7374   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7375 }
 7376 
 7377 void Assembler::psrad(XMMRegister dst, int shift) {
 7378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7379   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7380   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7381   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7382   emit_int8(0x72);
 7383   emit_int8((0xC0 | encode));
 7384   emit_int8(shift & 0xFF);
 7385 }
 7386 
 7387 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7388   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7389   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7390   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7391   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7392 }
 7393 
 7394 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7395   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7397   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7398   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7399 }
 7400 
 7401 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7402   assert(UseAVX > 0, "requires some form of AVX");
 7403   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7404   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7405   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7406   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7407 }
 7408 
 7409 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7410   assert(UseAVX > 0, "requires some form of AVX");
 7411   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7412   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7413   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7414   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7415 }
 7416 
 7417 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7418   assert(UseAVX > 0, "requires some form of AVX");
 7419   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7420   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7421   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7422 }
 7423 
 7424 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7425   assert(UseAVX > 0, "requires some form of AVX");
 7426   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7427   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7428   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7429 }
 7430 
 7431 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7432   assert(UseAVX > 2, "requires AVX512");
 7433   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7434   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7435   attributes.set_is_evex_instruction();
 7436   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7437   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7438 }
 7439 
 7440 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7441   assert(UseAVX > 2, "requires AVX512");
 7442   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7443   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7444   attributes.set_is_evex_instruction();
 7445   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7446   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7447 }
 7448 
 7449 // logical operations packed integers
 7450 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7451   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7453   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7454   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7455 }
 7456 
 7457 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7458   assert(UseAVX > 0, "requires some form of AVX");
 7459   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7460   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7461   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7462 }
 7463 
 7464 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7465   assert(UseAVX > 0, "requires some form of AVX");
 7466   InstructionMark im(this);
 7467   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7468   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7469   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7470   emit_int8((unsigned char)0xDB);
 7471   emit_operand(dst, src);
 7472 }
 7473 
 7474 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7475   assert(VM_Version::supports_evex(), "");
 7476   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7477   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7478   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7479 }
 7480 
 7481 //Variable Shift packed integers logically left.
 7482 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7483   assert(UseAVX > 1, "requires AVX2");
 7484   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7485   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7486   emit_int16(0x47, (0xC0 | encode));
 7487 }
 7488 
 7489 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7490   assert(UseAVX > 1, "requires AVX2");
 7491   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7492   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7493   emit_int16(0x47, (0xC0 | encode));
 7494 }
 7495 
 7496 //Variable Shift packed integers logically right.
 7497 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7498   assert(UseAVX > 1, "requires AVX2");
 7499   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7500   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7501   emit_int16(0x45, (0xC0 | encode));
 7502 }
 7503 
 7504 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7505   assert(UseAVX > 1, "requires AVX2");
 7506   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7507   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7508   emit_int16(0x45, (0xC0 | encode));
 7509 }
 7510 
 7511 //Variable right Shift arithmetic packed integers .
 7512 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7513   assert(UseAVX > 1, "requires AVX2");
 7514   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7515   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7516   emit_int16(0x46, (0xC0 | encode));
 7517 }
 7518 
 7519 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7520   assert(VM_Version::supports_avx512bw(), "");
 7521   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7522   attributes.set_is_evex_instruction();
 7523   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7524   emit_int16(0x11, (0xC0 | encode));
 7525 }
 7526 
 7527 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7528   assert(UseAVX > 2, "requires AVX512");
 7529   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7530   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7531   attributes.set_is_evex_instruction();
 7532   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7533   emit_int16(0x46, (0xC0 | encode));
 7534 }
 7535 
 7536 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7537   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7538   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7539   attributes.set_is_evex_instruction();
 7540   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7541   emit_int16(0x71, (0xC0 | encode));
 7542 }
 7543 
 7544 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7545   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7546   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7547   attributes.set_is_evex_instruction();
 7548   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7549   emit_int16(0x73, (0xC0 | encode));
 7550 }
 7551 
 7552 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7553   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7554   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7555   attributes.set_rex_vex_w_reverted();
 7556   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7557   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7558 }
 7559 
 7560 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7561   assert(UseAVX > 0, "requires some form of AVX");
 7562   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7564   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7565 }
 7566 
 7567 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7568   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7569   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7570   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7571   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7572 }
 7573 
 7574 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7575   assert(UseAVX > 0, "requires some form of AVX");
 7576   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7578   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7579 }
 7580 
 7581 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7582   assert(UseAVX > 0, "requires some form of AVX");
 7583   InstructionMark im(this);
 7584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7585   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7586   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7587   emit_int8((unsigned char)0xEB);
 7588   emit_operand(dst, src);
 7589 }
 7590 
 7591 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7592   assert(VM_Version::supports_evex(), "");
 7593   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7595   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7596 }
 7597 
 7598 
 7599 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7600   assert(VM_Version::supports_evex(), "");
 7601   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7603   attributes.set_is_evex_instruction();
 7604   attributes.set_embedded_opmask_register_specifier(mask);
 7605   if (merge) {
 7606     attributes.reset_is_clear_context();
 7607   }
 7608   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7609   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7610 }
 7611 
 7612 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7613   assert(VM_Version::supports_evex(), "");
 7614   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7615   InstructionMark im(this);
 7616   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7617   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7618   attributes.set_is_evex_instruction();
 7619   attributes.set_embedded_opmask_register_specifier(mask);
 7620   if (merge) {
 7621     attributes.reset_is_clear_context();
 7622   }
 7623   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7624   emit_int8((unsigned char)0xEB);
 7625   emit_operand(dst, src);
 7626 }
 7627 
 7628 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7629   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7630   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7631   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7632   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7633 }
 7634 
 7635 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7636   assert(UseAVX > 0, "requires some form of AVX");
 7637   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7638   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7639   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7640 }
 7641 
 7642 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7643   assert(UseAVX > 0, "requires some form of AVX");
 7644   InstructionMark im(this);
 7645   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7646   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7647   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7648   emit_int8((unsigned char)0xEF);
 7649   emit_operand(dst, src);
 7650 }
 7651 
 7652 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7653   assert(UseAVX > 2, "requires some form of EVEX");
 7654   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7655   attributes.set_rex_vex_w_reverted();
 7656   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7657   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7658 }
 7659 
 7660 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7661   assert(VM_Version::supports_evex(), "");
 7662   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7663   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7664   attributes.set_is_evex_instruction();
 7665   attributes.set_embedded_opmask_register_specifier(mask);
 7666   if (merge) {
 7667     attributes.reset_is_clear_context();
 7668   }
 7669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7670   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7671 }
 7672 
 7673 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7674   assert(VM_Version::supports_evex(), "requires EVEX support");
 7675   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7676   attributes.set_is_evex_instruction();
 7677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7678   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7679 }
 7680 
 7681 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7682   assert(VM_Version::supports_evex(), "requires EVEX support");
 7683   assert(dst != xnoreg, "sanity");
 7684   InstructionMark im(this);
 7685   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7686   attributes.set_is_evex_instruction();
 7687   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7688   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7689   emit_int8((unsigned char)0xEF);
 7690   emit_operand(dst, src);
 7691 }
 7692 
 7693 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7694   assert(VM_Version::supports_evex(), "requires EVEX support");
 7695   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7696   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7697   attributes.set_is_evex_instruction();
 7698   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7699   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7700 }
 7701 
 7702 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7703   assert(VM_Version::supports_evex(), "requires EVEX support");
 7704   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7705   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7706   attributes.set_is_evex_instruction();
 7707   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7708   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7709 }
 7710 
 7711 // Register is a class, but it would be assigned numerical value.
 7712 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7713 PRAGMA_DIAG_PUSH
 7714 PRAGMA_NONNULL_IGNORED
 7715 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7716   assert(VM_Version::supports_evex(), "requires EVEX support");
 7717   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7718   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7719   attributes.set_is_evex_instruction();
 7720   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7721   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7722 }
 7723 
 7724 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7725   assert(VM_Version::supports_evex(), "requires EVEX support");
 7726   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7727   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7728   attributes.set_is_evex_instruction();
 7729   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7730   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7731 }
 7732 PRAGMA_DIAG_POP
 7733 
 7734 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7735   assert(VM_Version::supports_evex(), "requires EVEX support");
 7736   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7737   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7738   attributes.set_is_evex_instruction();
 7739   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7740   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7741 }
 7742 
 7743 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7744   assert(VM_Version::supports_evex(), "requires EVEX support");
 7745   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7746   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7747   attributes.set_is_evex_instruction();
 7748   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7749   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7750 }
 7751 
 7752 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7753   assert(VM_Version::supports_evex(), "requires EVEX support");
 7754   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7755   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7756   attributes.set_is_evex_instruction();
 7757   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7758   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7759 }
 7760 
 7761 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7762   assert(VM_Version::supports_evex(), "requires EVEX support");
 7763   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7764   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7765   attributes.set_is_evex_instruction();
 7766   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7767   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7768 }
 7769 
 7770 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7771   assert(VM_Version::supports_evex(), "requires EVEX support");
 7772   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7773   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7774   attributes.set_is_evex_instruction();
 7775   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7776   emit_int8(0x25);
 7777   emit_int8((unsigned char)(0xC0 | encode));
 7778   emit_int8(imm8);
 7779 }
 7780 
 7781 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 7782   assert(VM_Version::supports_evex(), "requires EVEX support");
 7783   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7784   assert(dst != xnoreg, "sanity");
 7785   InstructionMark im(this);
 7786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7787   attributes.set_is_evex_instruction();
 7788   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7789   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7790   emit_int8(0x25);
 7791   emit_operand(dst, src3);
 7792   emit_int8(imm8);
 7793 }
 7794 
 7795 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7796   assert(VM_Version::supports_evex(), "requires EVEX support");
 7797   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7798   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7799   attributes.set_is_evex_instruction();
 7800   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7801   emit_int8(0x25);
 7802   emit_int8((unsigned char)(0xC0 | encode));
 7803   emit_int8(imm8);
 7804 }
 7805 
 7806 // vinserti forms
 7807 
 7808 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7809   assert(VM_Version::supports_avx2(), "");
 7810   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7811   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7812   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7813   // last byte:
 7814   // 0x00 - insert into lower 128 bits
 7815   // 0x01 - insert into upper 128 bits
 7816   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 7817 }
 7818 
 7819 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7820   assert(VM_Version::supports_avx2(), "");
 7821   assert(dst != xnoreg, "sanity");
 7822   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7823   InstructionMark im(this);
 7824   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7825   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7826   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7827   emit_int8(0x38);
 7828   emit_operand(dst, src);
 7829   // 0x00 - insert into lower 128 bits
 7830   // 0x01 - insert into upper 128 bits
 7831   emit_int8(imm8 & 0x01);
 7832 }
 7833 
 7834 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7835   assert(VM_Version::supports_evex(), "");
 7836   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7837   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7838   attributes.set_is_evex_instruction();
 7839   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7840   // imm8:
 7841   // 0x00 - insert into q0 128 bits (0..127)
 7842   // 0x01 - insert into q1 128 bits (128..255)
 7843   // 0x02 - insert into q2 128 bits (256..383)
 7844   // 0x03 - insert into q3 128 bits (384..511)
 7845   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 7846 }
 7847 
 7848 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7849   assert(VM_Version::supports_avx(), "");
 7850   assert(dst != xnoreg, "sanity");
 7851   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7852   InstructionMark im(this);
 7853   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7854   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7855   attributes.set_is_evex_instruction();
 7856   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7857   emit_int8(0x18);
 7858   emit_operand(dst, src);
 7859   // 0x00 - insert into q0 128 bits (0..127)
 7860   // 0x01 - insert into q1 128 bits (128..255)
 7861   // 0x02 - insert into q2 128 bits (256..383)
 7862   // 0x03 - insert into q3 128 bits (384..511)
 7863   emit_int8(imm8 & 0x03);
 7864 }
 7865 
 7866 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7867   assert(VM_Version::supports_evex(), "");
 7868   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7869   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7870   attributes.set_is_evex_instruction();
 7871   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7872   //imm8:
 7873   // 0x00 - insert into lower 256 bits
 7874   // 0x01 - insert into upper 256 bits
 7875   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 7876 }
 7877 
 7878 
 7879 // vinsertf forms
 7880 
 7881 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7882   assert(VM_Version::supports_avx(), "");
 7883   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7884   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7885   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7886   // imm8:
 7887   // 0x00 - insert into lower 128 bits
 7888   // 0x01 - insert into upper 128 bits
 7889   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 7890 }
 7891 
 7892 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7893   assert(VM_Version::supports_avx(), "");
 7894   assert(dst != xnoreg, "sanity");
 7895   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7896   InstructionMark im(this);
 7897   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7898   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7899   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7900   emit_int8(0x18);
 7901   emit_operand(dst, src);
 7902   // 0x00 - insert into lower 128 bits
 7903   // 0x01 - insert into upper 128 bits
 7904   emit_int8(imm8 & 0x01);
 7905 }
 7906 
 7907 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7908   assert(VM_Version::supports_avx2(), "");
 7909   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7910   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7911   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7912   // imm8:
 7913   // 0x00 - insert into q0 128 bits (0..127)
 7914   // 0x01 - insert into q1 128 bits (128..255)
 7915   // 0x02 - insert into q0 128 bits (256..383)
 7916   // 0x03 - insert into q1 128 bits (384..512)
 7917   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 7918 }
 7919 
 7920 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7921   assert(VM_Version::supports_avx(), "");
 7922   assert(dst != xnoreg, "sanity");
 7923   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7924   InstructionMark im(this);
 7925   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7926   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7927   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7928   emit_int8(0x18);
 7929   emit_operand(dst, src);
 7930   // 0x00 - insert into q0 128 bits (0..127)
 7931   // 0x01 - insert into q1 128 bits (128..255)
 7932   // 0x02 - insert into q0 128 bits (256..383)
 7933   // 0x03 - insert into q1 128 bits (384..512)
 7934   emit_int8(imm8 & 0x03);
 7935 }
 7936 
 7937 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7938   assert(VM_Version::supports_evex(), "");
 7939   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7940   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7941   attributes.set_is_evex_instruction();
 7942   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7943   // imm8:
 7944   // 0x00 - insert into lower 256 bits
 7945   // 0x01 - insert into upper 256 bits
 7946   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 7947 }
 7948 
 7949 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7950   assert(VM_Version::supports_evex(), "");
 7951   assert(dst != xnoreg, "sanity");
 7952   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7953   InstructionMark im(this);
 7954   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7955   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 7956   attributes.set_is_evex_instruction();
 7957   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7958   emit_int8(0x1A);
 7959   emit_operand(dst, src);
 7960   // 0x00 - insert into lower 256 bits
 7961   // 0x01 - insert into upper 256 bits
 7962   emit_int8(imm8 & 0x01);
 7963 }
 7964 
 7965 
 7966 // vextracti forms
 7967 
 7968 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7969   assert(VM_Version::supports_avx2(), "");
 7970   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7971   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7972   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7973   // imm8:
 7974   // 0x00 - extract from lower 128 bits
 7975   // 0x01 - extract from upper 128 bits
 7976   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 7977 }
 7978 
 7979 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 7980   assert(VM_Version::supports_avx2(), "");
 7981   assert(src != xnoreg, "sanity");
 7982   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7983   InstructionMark im(this);
 7984   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7985   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7986   attributes.reset_is_clear_context();
 7987   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7988   emit_int8(0x39);
 7989   emit_operand(src, dst);
 7990   // 0x00 - extract from lower 128 bits
 7991   // 0x01 - extract from upper 128 bits
 7992   emit_int8(imm8 & 0x01);
 7993 }
 7994 
 7995 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7996   assert(VM_Version::supports_evex(), "");
 7997   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7998   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7999   attributes.set_is_evex_instruction();
 8000   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8001   // imm8:
 8002   // 0x00 - extract from bits 127:0
 8003   // 0x01 - extract from bits 255:128
 8004   // 0x02 - extract from bits 383:256
 8005   // 0x03 - extract from bits 511:384
 8006   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8007 }
 8008 
 8009 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8010   assert(VM_Version::supports_evex(), "");
 8011   assert(src != xnoreg, "sanity");
 8012   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8013   InstructionMark im(this);
 8014   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8015   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8016   attributes.reset_is_clear_context();
 8017   attributes.set_is_evex_instruction();
 8018   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8019   emit_int8(0x39);
 8020   emit_operand(src, dst);
 8021   // 0x00 - extract from bits 127:0
 8022   // 0x01 - extract from bits 255:128
 8023   // 0x02 - extract from bits 383:256
 8024   // 0x03 - extract from bits 511:384
 8025   emit_int8(imm8 & 0x03);
 8026 }
 8027 
 8028 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8029   assert(VM_Version::supports_avx512dq(), "");
 8030   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8031   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8032   attributes.set_is_evex_instruction();
 8033   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8034   // imm8:
 8035   // 0x00 - extract from bits 127:0
 8036   // 0x01 - extract from bits 255:128
 8037   // 0x02 - extract from bits 383:256
 8038   // 0x03 - extract from bits 511:384
 8039   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8040 }
 8041 
 8042 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8043   assert(VM_Version::supports_evex(), "");
 8044   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8045   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8046   attributes.set_is_evex_instruction();
 8047   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8048   // imm8:
 8049   // 0x00 - extract from lower 256 bits
 8050   // 0x01 - extract from upper 256 bits
 8051   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8052 }
 8053 
 8054 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8055   assert(VM_Version::supports_evex(), "");
 8056   assert(src != xnoreg, "sanity");
 8057   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8058   InstructionMark im(this);
 8059   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8060   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8061   attributes.reset_is_clear_context();
 8062   attributes.set_is_evex_instruction();
 8063   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8064   emit_int8(0x38);
 8065   emit_operand(src, dst);
 8066   // 0x00 - extract from lower 256 bits
 8067   // 0x01 - extract from upper 256 bits
 8068   emit_int8(imm8 & 0x01);
 8069 }
 8070 // vextractf forms
 8071 
 8072 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8073   assert(VM_Version::supports_avx(), "");
 8074   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8075   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8076   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8077   // imm8:
 8078   // 0x00 - extract from lower 128 bits
 8079   // 0x01 - extract from upper 128 bits
 8080   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8081 }
 8082 
 8083 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8084   assert(VM_Version::supports_avx(), "");
 8085   assert(src != xnoreg, "sanity");
 8086   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8087   InstructionMark im(this);
 8088   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8089   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8090   attributes.reset_is_clear_context();
 8091   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8092   emit_int8(0x19);
 8093   emit_operand(src, dst);
 8094   // 0x00 - extract from lower 128 bits
 8095   // 0x01 - extract from upper 128 bits
 8096   emit_int8(imm8 & 0x01);
 8097 }
 8098 
 8099 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8100   assert(VM_Version::supports_evex(), "");
 8101   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8102   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8103   attributes.set_is_evex_instruction();
 8104   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8105   // imm8:
 8106   // 0x00 - extract from bits 127:0
 8107   // 0x01 - extract from bits 255:128
 8108   // 0x02 - extract from bits 383:256
 8109   // 0x03 - extract from bits 511:384
 8110   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8111 }
 8112 
 8113 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8114   assert(VM_Version::supports_evex(), "");
 8115   assert(src != xnoreg, "sanity");
 8116   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8117   InstructionMark im(this);
 8118   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8119   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8120   attributes.reset_is_clear_context();
 8121   attributes.set_is_evex_instruction();
 8122   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8123   emit_int8(0x19);
 8124   emit_operand(src, dst);
 8125   // 0x00 - extract from bits 127:0
 8126   // 0x01 - extract from bits 255:128
 8127   // 0x02 - extract from bits 383:256
 8128   // 0x03 - extract from bits 511:384
 8129   emit_int8(imm8 & 0x03);
 8130 }
 8131 
 8132 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8133   assert(VM_Version::supports_avx512dq(), "");
 8134   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8135   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8136   attributes.set_is_evex_instruction();
 8137   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8138   // imm8:
 8139   // 0x00 - extract from bits 127:0
 8140   // 0x01 - extract from bits 255:128
 8141   // 0x02 - extract from bits 383:256
 8142   // 0x03 - extract from bits 511:384
 8143   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8144 }
 8145 
 8146 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8147   assert(VM_Version::supports_evex(), "");
 8148   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8149   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8150   attributes.set_is_evex_instruction();
 8151   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8152   // imm8:
 8153   // 0x00 - extract from lower 256 bits
 8154   // 0x01 - extract from upper 256 bits
 8155   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8156 }
 8157 
 8158 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8159   assert(VM_Version::supports_evex(), "");
 8160   assert(src != xnoreg, "sanity");
 8161   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8162   InstructionMark im(this);
 8163   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8164   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8165   attributes.reset_is_clear_context();
 8166   attributes.set_is_evex_instruction();
 8167   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8168   emit_int8(0x1B);
 8169   emit_operand(src, dst);
 8170   // 0x00 - extract from lower 256 bits
 8171   // 0x01 - extract from upper 256 bits
 8172   emit_int8(imm8 & 0x01);
 8173 }
 8174 
 8175 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8176 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8177   assert(VM_Version::supports_avx2(), "");
 8178   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8179   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8180   emit_int16(0x78, (0xC0 | encode));
 8181 }
 8182 
 8183 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8184   assert(VM_Version::supports_avx2(), "");
 8185   assert(dst != xnoreg, "sanity");
 8186   InstructionMark im(this);
 8187   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8188   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8189   // swap src<->dst for encoding
 8190   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8191   emit_int8(0x78);
 8192   emit_operand(dst, src);
 8193 }
 8194 
 8195 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8196 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8197   assert(VM_Version::supports_avx2(), "");
 8198   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8199   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8200   emit_int16(0x79, (0xC0 | encode));
 8201 }
 8202 
 8203 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8204   assert(VM_Version::supports_avx2(), "");
 8205   assert(dst != xnoreg, "sanity");
 8206   InstructionMark im(this);
 8207   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8208   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8209   // swap src<->dst for encoding
 8210   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8211   emit_int8(0x79);
 8212   emit_operand(dst, src);
 8213 }
 8214 
 8215 // xmm/mem sourced byte/word/dword/qword replicate
 8216 
 8217 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 8218 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
 8219   assert(UseAVX >= 2, "");
 8220   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8221   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8222   emit_int16(0x58, (0xC0 | encode));
 8223 }
 8224 
 8225 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
 8226   assert(VM_Version::supports_avx2(), "");
 8227   assert(dst != xnoreg, "sanity");
 8228   InstructionMark im(this);
 8229   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8230   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8231   // swap src<->dst for encoding
 8232   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8233   emit_int8(0x58);
 8234   emit_operand(dst, src);
 8235 }
 8236 
 8237 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 8238 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
 8239   assert(VM_Version::supports_avx2(), "");
 8240   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8241   attributes.set_rex_vex_w_reverted();
 8242   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8243   emit_int16(0x59, (0xC0 | encode));
 8244 }
 8245 
 8246 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
 8247   assert(VM_Version::supports_avx2(), "");
 8248   assert(dst != xnoreg, "sanity");
 8249   InstructionMark im(this);
 8250   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8251   attributes.set_rex_vex_w_reverted();
 8252   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 8253   // swap src<->dst for encoding
 8254   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8255   emit_int8(0x59);
 8256   emit_operand(dst, src);
 8257 }
 8258 
 8259 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
 8260   assert(vector_len != Assembler::AVX_128bit, "");
 8261   assert(VM_Version::supports_avx512dq(), "");
 8262   assert(dst != xnoreg, "sanity");
 8263   InstructionMark im(this);
 8264   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8265   attributes.set_rex_vex_w_reverted();
 8266   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 8267   // swap src<->dst for encoding
 8268   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8269   emit_int8(0x5A);
 8270   emit_operand(dst, src);
 8271 }
 8272 
 8273 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
 8274   assert(vector_len != Assembler::AVX_128bit, "");
 8275   assert(VM_Version::supports_avx512dq(), "");
 8276   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8277   attributes.set_rex_vex_w_reverted();
 8278   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8279   emit_int16(0x5A, (0xC0 | encode));
 8280 }
 8281 
 8282 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
 8283   assert(vector_len != Assembler::AVX_128bit, "");
 8284   assert(VM_Version::supports_avx512dq(), "");
 8285   assert(dst != xnoreg, "sanity");
 8286   InstructionMark im(this);
 8287   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8288   attributes.set_rex_vex_w_reverted();
 8289   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 8290   // swap src<->dst for encoding
 8291   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8292   emit_int8(0x5A);
 8293   emit_operand(dst, src);
 8294 }
 8295 
 8296 // scalar single/double precision replicate
 8297 
 8298 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
 8299 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
 8300   assert(VM_Version::supports_avx2(), "");
 8301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8302   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8303   emit_int16(0x18, (0xC0 | encode));
 8304 }
 8305 
 8306 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
 8307   assert(VM_Version::supports_avx(), "");
 8308   assert(dst != xnoreg, "sanity");
 8309   InstructionMark im(this);
 8310   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8311   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8312   // swap src<->dst for encoding
 8313   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8314   emit_int8(0x18);
 8315   emit_operand(dst, src);
 8316 }
 8317 
 8318 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
 8319 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
 8320   assert(VM_Version::supports_avx2(), "");
 8321   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 8322   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8323   attributes.set_rex_vex_w_reverted();
 8324   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8325   emit_int16(0x19, (0xC0 | encode));
 8326 }
 8327 
 8328 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
 8329   assert(VM_Version::supports_avx(), "");
 8330   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 8331   assert(dst != xnoreg, "sanity");
 8332   InstructionMark im(this);
 8333   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8334   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 8335   attributes.set_rex_vex_w_reverted();
 8336   // swap src<->dst for encoding
 8337   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8338   emit_int8(0x19);
 8339   emit_operand(dst, src);
 8340 }
 8341 
 8342 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
 8343   assert(VM_Version::supports_avx(), "");
 8344   assert(vector_len == AVX_256bit, "");
 8345   assert(dst != xnoreg, "sanity");
 8346   InstructionMark im(this);
 8347   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8348   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8349   // swap src<->dst for encoding
 8350   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8351   emit_int8(0x1A);
 8352   emit_operand(dst, src);
 8353 }
 8354 
 8355 // gpr source broadcast forms
 8356 
 8357 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8358 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
 8359   assert(VM_Version::supports_avx512bw(), "");
 8360   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8361   attributes.set_is_evex_instruction();
 8362   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8363   emit_int16(0x7A, (0xC0 | encode));
 8364 }
 8365 
 8366 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8367 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
 8368   assert(VM_Version::supports_avx512bw(), "");
 8369   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8370   attributes.set_is_evex_instruction();
 8371   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8372   emit_int16(0x7B, (0xC0 | encode));
 8373 }
 8374 
 8375 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 8376 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
 8377   assert(VM_Version::supports_evex(), "");
 8378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8379   attributes.set_is_evex_instruction();
 8380   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8381   emit_int16(0x7C, (0xC0 | encode));
 8382 }
 8383 
 8384 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 8385 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
 8386   assert(VM_Version::supports_evex(), "");
 8387   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8388   attributes.set_is_evex_instruction();
 8389   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8390   emit_int16(0x7C, (0xC0 | encode));
 8391 }
 8392 
 8393 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8394   assert(VM_Version::supports_avx2(), "");
 8395   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8396   assert(dst != xnoreg, "sanity");
 8397   assert(src.isxmmindex(),"expected to be xmm index");
 8398   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8399   InstructionMark im(this);
 8400   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8401   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8402   emit_int8((unsigned char)0x90);
 8403   emit_operand(dst, src);
 8404 }
 8405 
 8406 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8407   assert(VM_Version::supports_avx2(), "");
 8408   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8409   assert(dst != xnoreg, "sanity");
 8410   assert(src.isxmmindex(),"expected to be xmm index");
 8411   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8412   InstructionMark im(this);
 8413   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8414   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8415   emit_int8((unsigned char)0x90);
 8416   emit_operand(dst, src);
 8417 }
 8418 
 8419 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8420   assert(VM_Version::supports_avx2(), "");
 8421   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8422   assert(dst != xnoreg, "sanity");
 8423   assert(src.isxmmindex(),"expected to be xmm index");
 8424   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8425   InstructionMark im(this);
 8426   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8427   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8428   emit_int8((unsigned char)0x92);
 8429   emit_operand(dst, src);
 8430 }
 8431 
 8432 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8433   assert(VM_Version::supports_avx2(), "");
 8434   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8435   assert(dst != xnoreg, "sanity");
 8436   assert(src.isxmmindex(),"expected to be xmm index");
 8437   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8438   InstructionMark im(this);
 8439   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 8440   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8441   emit_int8((unsigned char)0x92);
 8442   emit_operand(dst, src);
 8443 }
 8444 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8445   assert(VM_Version::supports_evex(), "");
 8446   assert(dst != xnoreg, "sanity");
 8447   assert(src.isxmmindex(),"expected to be xmm index");
 8448   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8449   assert(mask != k0, "instruction will #UD if mask is in k0");
 8450   InstructionMark im(this);
 8451   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8452   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8453   attributes.reset_is_clear_context();
 8454   attributes.set_embedded_opmask_register_specifier(mask);
 8455   attributes.set_is_evex_instruction();
 8456   // swap src<->dst for encoding
 8457   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8458   emit_int8((unsigned char)0x90);
 8459   emit_operand(dst, src);
 8460 }
 8461 
 8462 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8463   assert(VM_Version::supports_evex(), "");
 8464   assert(dst != xnoreg, "sanity");
 8465   assert(src.isxmmindex(),"expected to be xmm index");
 8466   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8467   assert(mask != k0, "instruction will #UD if mask is in k0");
 8468   InstructionMark im(this);
 8469   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8470   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8471   attributes.reset_is_clear_context();
 8472   attributes.set_embedded_opmask_register_specifier(mask);
 8473   attributes.set_is_evex_instruction();
 8474   // swap src<->dst for encoding
 8475   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8476   emit_int8((unsigned char)0x90);
 8477   emit_operand(dst, src);
 8478 }
 8479 
 8480 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8481   assert(VM_Version::supports_evex(), "");
 8482   assert(dst != xnoreg, "sanity");
 8483   assert(src.isxmmindex(),"expected to be xmm index");
 8484   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8485   assert(mask != k0, "instruction will #UD if mask is in k0");
 8486   InstructionMark im(this);
 8487   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8488   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8489   attributes.reset_is_clear_context();
 8490   attributes.set_embedded_opmask_register_specifier(mask);
 8491   attributes.set_is_evex_instruction();
 8492   // swap src<->dst for encoding
 8493   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8494   emit_int8((unsigned char)0x92);
 8495   emit_operand(dst, src);
 8496 }
 8497 
 8498 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8499   assert(VM_Version::supports_evex(), "");
 8500   assert(dst != xnoreg, "sanity");
 8501   assert(src.isxmmindex(),"expected to be xmm index");
 8502   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8503   assert(mask != k0, "instruction will #UD if mask is in k0");
 8504   InstructionMark im(this);
 8505   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8506   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8507   attributes.reset_is_clear_context();
 8508   attributes.set_embedded_opmask_register_specifier(mask);
 8509   attributes.set_is_evex_instruction();
 8510   // swap src<->dst for encoding
 8511   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8512   emit_int8((unsigned char)0x92);
 8513   emit_operand(dst, src);
 8514 }
 8515 
 8516 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8517   assert(VM_Version::supports_evex(), "");
 8518   assert(mask != k0, "instruction will #UD if mask is in k0");
 8519   InstructionMark im(this);
 8520   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8521   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8522   attributes.reset_is_clear_context();
 8523   attributes.set_embedded_opmask_register_specifier(mask);
 8524   attributes.set_is_evex_instruction();
 8525   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8526   emit_int8((unsigned char)0xA0);
 8527   emit_operand(src, dst);
 8528 }
 8529 
 8530 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8531   assert(VM_Version::supports_evex(), "");
 8532   assert(mask != k0, "instruction will #UD if mask is in k0");
 8533   InstructionMark im(this);
 8534   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8535   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8536   attributes.reset_is_clear_context();
 8537   attributes.set_embedded_opmask_register_specifier(mask);
 8538   attributes.set_is_evex_instruction();
 8539   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8540   emit_int8((unsigned char)0xA0);
 8541   emit_operand(src, dst);
 8542 }
 8543 
 8544 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8545   assert(VM_Version::supports_evex(), "");
 8546   assert(mask != k0, "instruction will #UD if mask is in k0");
 8547   InstructionMark im(this);
 8548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8549   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8550   attributes.reset_is_clear_context();
 8551   attributes.set_embedded_opmask_register_specifier(mask);
 8552   attributes.set_is_evex_instruction();
 8553   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8554   emit_int8((unsigned char)0xA2);
 8555   emit_operand(src, dst);
 8556 }
 8557 
 8558 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8559   assert(VM_Version::supports_evex(), "");
 8560   assert(mask != k0, "instruction will #UD if mask is in k0");
 8561   InstructionMark im(this);
 8562   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8563   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8564   attributes.reset_is_clear_context();
 8565   attributes.set_embedded_opmask_register_specifier(mask);
 8566   attributes.set_is_evex_instruction();
 8567   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8568   emit_int8((unsigned char)0xA2);
 8569   emit_operand(src, dst);
 8570 }
 8571 // Carry-Less Multiplication Quadword
 8572 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
 8573   assert(VM_Version::supports_clmul(), "");
 8574   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8575   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8576   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8577 }
 8578 
 8579 // Carry-Less Multiplication Quadword
 8580 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
 8581   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
 8582   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8583   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8584   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8585 }
 8586 
 8587 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
 8588   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
 8589   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8590   attributes.set_is_evex_instruction();
 8591   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8592   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8593 }
 8594 
 8595 void Assembler::vzeroupper_uncached() {
 8596   if (VM_Version::supports_vzeroupper()) {
 8597     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 8598     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8599     emit_int8(0x77);
 8600   }
 8601 }
 8602 
 8603 void Assembler::fld_x(Address adr) {
 8604   InstructionMark im(this);
 8605   emit_int8((unsigned char)0xDB);
 8606   emit_operand32(rbp, adr);
 8607 }
 8608 
 8609 void Assembler::fstp_x(Address adr) {
 8610   InstructionMark im(this);
 8611   emit_int8((unsigned char)0xDB);
 8612   emit_operand32(rdi, adr);
 8613 }
 8614 
 8615 void Assembler::emit_operand32(Register reg, Address adr) {
 8616   assert(reg->encoding() < 8, "no extended registers");
 8617   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 8618   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 8619                adr._rspec);
 8620 }
 8621 
 8622 #ifndef _LP64
 8623 // 32bit only pieces of the assembler
 8624 
 8625 void Assembler::emms() {
 8626   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
 8627   emit_int16(0x0F, 0x77);
 8628 }
 8629 
 8630 void Assembler::vzeroupper() {
 8631   vzeroupper_uncached();
 8632 }
 8633 
 8634 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
 8635   // NO PREFIX AS NEVER 64BIT
 8636   InstructionMark im(this);
 8637   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
 8638   emit_data(imm32, rspec, 0);
 8639 }
 8640 
 8641 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
 8642   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
 8643   InstructionMark im(this);
 8644   emit_int8((unsigned char)0x81);
 8645   emit_operand(rdi, src1);
 8646   emit_data(imm32, rspec, 0);
 8647 }
 8648 
 8649 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
 8650 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
 8651 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
 8652 void Assembler::cmpxchg8(Address adr) {
 8653   InstructionMark im(this);
 8654   emit_int16(0x0F, (unsigned char)0xC7);
 8655   emit_operand(rcx, adr);
 8656 }
 8657 
 8658 void Assembler::decl(Register dst) {
 8659   // Don't use it directly. Use MacroAssembler::decrementl() instead.
 8660  emit_int8(0x48 | dst->encoding());
 8661 }
 8662 
 8663 // 64bit doesn't use the x87
 8664 
 8665 void Assembler::emit_farith(int b1, int b2, int i) {
 8666   assert(isByte(b1) && isByte(b2), "wrong opcode");
 8667   assert(0 <= i &&  i < 8, "illegal stack offset");
 8668   emit_int16(b1, b2 + i);
 8669 }
 8670 
 8671 void Assembler::fabs() {
 8672   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
 8673 }
 8674 
 8675 void Assembler::fadd(int i) {
 8676   emit_farith(0xD8, 0xC0, i);
 8677 }
 8678 
 8679 void Assembler::fadd_d(Address src) {
 8680   InstructionMark im(this);
 8681   emit_int8((unsigned char)0xDC);
 8682   emit_operand32(rax, src);
 8683 }
 8684 
 8685 void Assembler::fadd_s(Address src) {
 8686   InstructionMark im(this);
 8687   emit_int8((unsigned char)0xD8);
 8688   emit_operand32(rax, src);
 8689 }
 8690 
 8691 void Assembler::fadda(int i) {
 8692   emit_farith(0xDC, 0xC0, i);
 8693 }
 8694 
 8695 void Assembler::faddp(int i) {
 8696   emit_farith(0xDE, 0xC0, i);
 8697 }
 8698 
 8699 void Assembler::fchs() {
 8700   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
 8701 }
 8702 
 8703 void Assembler::fcom(int i) {
 8704   emit_farith(0xD8, 0xD0, i);
 8705 }
 8706 
 8707 void Assembler::fcomp(int i) {
 8708   emit_farith(0xD8, 0xD8, i);
 8709 }
 8710 
 8711 void Assembler::fcomp_d(Address src) {
 8712   InstructionMark im(this);
 8713   emit_int8((unsigned char)0xDC);
 8714   emit_operand32(rbx, src);
 8715 }
 8716 
 8717 void Assembler::fcomp_s(Address src) {
 8718   InstructionMark im(this);
 8719   emit_int8((unsigned char)0xD8);
 8720   emit_operand32(rbx, src);
 8721 }
 8722 
 8723 void Assembler::fcompp() {
 8724   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
 8725 }
 8726 
 8727 void Assembler::fcos() {
 8728   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
 8729 }
 8730 
 8731 void Assembler::fdecstp() {
 8732   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
 8733 }
 8734 
 8735 void Assembler::fdiv(int i) {
 8736   emit_farith(0xD8, 0xF0, i);
 8737 }
 8738 
 8739 void Assembler::fdiv_d(Address src) {
 8740   InstructionMark im(this);
 8741   emit_int8((unsigned char)0xDC);
 8742   emit_operand32(rsi, src);
 8743 }
 8744 
 8745 void Assembler::fdiv_s(Address src) {
 8746   InstructionMark im(this);
 8747   emit_int8((unsigned char)0xD8);
 8748   emit_operand32(rsi, src);
 8749 }
 8750 
 8751 void Assembler::fdiva(int i) {
 8752   emit_farith(0xDC, 0xF8, i);
 8753 }
 8754 
 8755 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
 8756 //       is erroneous for some of the floating-point instructions below.
 8757 
 8758 void Assembler::fdivp(int i) {
 8759   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
 8760 }
 8761 
 8762 void Assembler::fdivr(int i) {
 8763   emit_farith(0xD8, 0xF8, i);
 8764 }
 8765 
 8766 void Assembler::fdivr_d(Address src) {
 8767   InstructionMark im(this);
 8768   emit_int8((unsigned char)0xDC);
 8769   emit_operand32(rdi, src);
 8770 }
 8771 
 8772 void Assembler::fdivr_s(Address src) {
 8773   InstructionMark im(this);
 8774   emit_int8((unsigned char)0xD8);
 8775   emit_operand32(rdi, src);
 8776 }
 8777 
 8778 void Assembler::fdivra(int i) {
 8779   emit_farith(0xDC, 0xF0, i);
 8780 }
 8781 
 8782 void Assembler::fdivrp(int i) {
 8783   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
 8784 }
 8785 
 8786 void Assembler::ffree(int i) {
 8787   emit_farith(0xDD, 0xC0, i);
 8788 }
 8789 
 8790 void Assembler::fild_d(Address adr) {
 8791   InstructionMark im(this);
 8792   emit_int8((unsigned char)0xDF);
 8793   emit_operand32(rbp, adr);
 8794 }
 8795 
 8796 void Assembler::fild_s(Address adr) {
 8797   InstructionMark im(this);
 8798   emit_int8((unsigned char)0xDB);
 8799   emit_operand32(rax, adr);
 8800 }
 8801 
 8802 void Assembler::fincstp() {
 8803   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
 8804 }
 8805 
 8806 void Assembler::finit() {
 8807   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
 8808 }
 8809 
 8810 void Assembler::fist_s(Address adr) {
 8811   InstructionMark im(this);
 8812   emit_int8((unsigned char)0xDB);
 8813   emit_operand32(rdx, adr);
 8814 }
 8815 
 8816 void Assembler::fistp_d(Address adr) {
 8817   InstructionMark im(this);
 8818   emit_int8((unsigned char)0xDF);
 8819   emit_operand32(rdi, adr);
 8820 }
 8821 
 8822 void Assembler::fistp_s(Address adr) {
 8823   InstructionMark im(this);
 8824   emit_int8((unsigned char)0xDB);
 8825   emit_operand32(rbx, adr);
 8826 }
 8827 
 8828 void Assembler::fld1() {
 8829   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
 8830 }
 8831 
 8832 void Assembler::fld_d(Address adr) {
 8833   InstructionMark im(this);
 8834   emit_int8((unsigned char)0xDD);
 8835   emit_operand32(rax, adr);
 8836 }
 8837 
 8838 void Assembler::fld_s(Address adr) {
 8839   InstructionMark im(this);
 8840   emit_int8((unsigned char)0xD9);
 8841   emit_operand32(rax, adr);
 8842 }
 8843 
 8844 
 8845 void Assembler::fld_s(int index) {
 8846   emit_farith(0xD9, 0xC0, index);
 8847 }
 8848 
 8849 void Assembler::fldcw(Address src) {
 8850   InstructionMark im(this);
 8851   emit_int8((unsigned char)0xD9);
 8852   emit_operand32(rbp, src);
 8853 }
 8854 
 8855 void Assembler::fldenv(Address src) {
 8856   InstructionMark im(this);
 8857   emit_int8((unsigned char)0xD9);
 8858   emit_operand32(rsp, src);
 8859 }
 8860 
 8861 void Assembler::fldlg2() {
 8862   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
 8863 }
 8864 
 8865 void Assembler::fldln2() {
 8866   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
 8867 }
 8868 
 8869 void Assembler::fldz() {
 8870   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
 8871 }
 8872 
 8873 void Assembler::flog() {
 8874   fldln2();
 8875   fxch();
 8876   fyl2x();
 8877 }
 8878 
 8879 void Assembler::flog10() {
 8880   fldlg2();
 8881   fxch();
 8882   fyl2x();
 8883 }
 8884 
 8885 void Assembler::fmul(int i) {
 8886   emit_farith(0xD8, 0xC8, i);
 8887 }
 8888 
 8889 void Assembler::fmul_d(Address src) {
 8890   InstructionMark im(this);
 8891   emit_int8((unsigned char)0xDC);
 8892   emit_operand32(rcx, src);
 8893 }
 8894 
 8895 void Assembler::fmul_s(Address src) {
 8896   InstructionMark im(this);
 8897   emit_int8((unsigned char)0xD8);
 8898   emit_operand32(rcx, src);
 8899 }
 8900 
 8901 void Assembler::fmula(int i) {
 8902   emit_farith(0xDC, 0xC8, i);
 8903 }
 8904 
 8905 void Assembler::fmulp(int i) {
 8906   emit_farith(0xDE, 0xC8, i);
 8907 }
 8908 
 8909 void Assembler::fnsave(Address dst) {
 8910   InstructionMark im(this);
 8911   emit_int8((unsigned char)0xDD);
 8912   emit_operand32(rsi, dst);
 8913 }
 8914 
 8915 void Assembler::fnstcw(Address src) {
 8916   InstructionMark im(this);
 8917   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
 8918   emit_operand32(rdi, src);
 8919 }
 8920 
 8921 void Assembler::fnstsw_ax() {
 8922   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
 8923 }
 8924 
 8925 void Assembler::fprem() {
 8926   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
 8927 }
 8928 
 8929 void Assembler::fprem1() {
 8930   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
 8931 }
 8932 
 8933 void Assembler::frstor(Address src) {
 8934   InstructionMark im(this);
 8935   emit_int8((unsigned char)0xDD);
 8936   emit_operand32(rsp, src);
 8937 }
 8938 
 8939 void Assembler::fsin() {
 8940   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
 8941 }
 8942 
 8943 void Assembler::fsqrt() {
 8944   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
 8945 }
 8946 
 8947 void Assembler::fst_d(Address adr) {
 8948   InstructionMark im(this);
 8949   emit_int8((unsigned char)0xDD);
 8950   emit_operand32(rdx, adr);
 8951 }
 8952 
 8953 void Assembler::fst_s(Address adr) {
 8954   InstructionMark im(this);
 8955   emit_int8((unsigned char)0xD9);
 8956   emit_operand32(rdx, adr);
 8957 }
 8958 
 8959 void Assembler::fstp_d(Address adr) {
 8960   InstructionMark im(this);
 8961   emit_int8((unsigned char)0xDD);
 8962   emit_operand32(rbx, adr);
 8963 }
 8964 
 8965 void Assembler::fstp_d(int index) {
 8966   emit_farith(0xDD, 0xD8, index);
 8967 }
 8968 
 8969 void Assembler::fstp_s(Address adr) {
 8970   InstructionMark im(this);
 8971   emit_int8((unsigned char)0xD9);
 8972   emit_operand32(rbx, adr);
 8973 }
 8974 
 8975 void Assembler::fsub(int i) {
 8976   emit_farith(0xD8, 0xE0, i);
 8977 }
 8978 
 8979 void Assembler::fsub_d(Address src) {
 8980   InstructionMark im(this);
 8981   emit_int8((unsigned char)0xDC);
 8982   emit_operand32(rsp, src);
 8983 }
 8984 
 8985 void Assembler::fsub_s(Address src) {
 8986   InstructionMark im(this);
 8987   emit_int8((unsigned char)0xD8);
 8988   emit_operand32(rsp, src);
 8989 }
 8990 
 8991 void Assembler::fsuba(int i) {
 8992   emit_farith(0xDC, 0xE8, i);
 8993 }
 8994 
 8995 void Assembler::fsubp(int i) {
 8996   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
 8997 }
 8998 
 8999 void Assembler::fsubr(int i) {
 9000   emit_farith(0xD8, 0xE8, i);
 9001 }
 9002 
 9003 void Assembler::fsubr_d(Address src) {
 9004   InstructionMark im(this);
 9005   emit_int8((unsigned char)0xDC);
 9006   emit_operand32(rbp, src);
 9007 }
 9008 
 9009 void Assembler::fsubr_s(Address src) {
 9010   InstructionMark im(this);
 9011   emit_int8((unsigned char)0xD8);
 9012   emit_operand32(rbp, src);
 9013 }
 9014 
 9015 void Assembler::fsubra(int i) {
 9016   emit_farith(0xDC, 0xE0, i);
 9017 }
 9018 
 9019 void Assembler::fsubrp(int i) {
 9020   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
 9021 }
 9022 
 9023 void Assembler::ftan() {
 9024   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
 9025 }
 9026 
 9027 void Assembler::ftst() {
 9028   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
 9029 }
 9030 
 9031 void Assembler::fucomi(int i) {
 9032   // make sure the instruction is supported (introduced for P6, together with cmov)
 9033   guarantee(VM_Version::supports_cmov(), "illegal instruction");
 9034   emit_farith(0xDB, 0xE8, i);
 9035 }
 9036 
 9037 void Assembler::fucomip(int i) {
 9038   // make sure the instruction is supported (introduced for P6, together with cmov)
 9039   guarantee(VM_Version::supports_cmov(), "illegal instruction");
 9040   emit_farith(0xDF, 0xE8, i);
 9041 }
 9042 
 9043 void Assembler::fwait() {
 9044   emit_int8((unsigned char)0x9B);
 9045 }
 9046 
 9047 void Assembler::fxch(int i) {
 9048   emit_farith(0xD9, 0xC8, i);
 9049 }
 9050 
 9051 void Assembler::fyl2x() {
 9052   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
 9053 }
 9054 
 9055 void Assembler::frndint() {
 9056   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
 9057 }
 9058 
 9059 void Assembler::f2xm1() {
 9060   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
 9061 }
 9062 
 9063 void Assembler::fldl2e() {
 9064   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
 9065 }
 9066 #endif // !_LP64
 9067 
 9068 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
 9069 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
 9070 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
 9071 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
 9072 
 9073 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
 9074 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
 9075   if (pre > 0) {
 9076     emit_int8(simd_pre[pre]);
 9077   }
 9078   if (rex_w) {
 9079     prefixq(adr, xreg);
 9080   } else {
 9081     prefix(adr, xreg);
 9082   }
 9083   if (opc > 0) {
 9084     emit_int8(0x0F);
 9085     int opc2 = simd_opc[opc];
 9086     if (opc2 > 0) {
 9087       emit_int8(opc2);
 9088     }
 9089   }
 9090 }
 9091 
 9092 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
 9093   if (pre > 0) {
 9094     emit_int8(simd_pre[pre]);
 9095   }
 9096   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
 9097   if (opc > 0) {
 9098     emit_int8(0x0F);
 9099     int opc2 = simd_opc[opc];
 9100     if (opc2 > 0) {
 9101       emit_int8(opc2);
 9102     }
 9103   }
 9104   return encode;
 9105 }
 9106 
 9107 
 9108 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
 9109   int vector_len = _attributes->get_vector_len();
 9110   bool vex_w = _attributes->is_rex_vex_w();
 9111   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
 9112     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
 9113     byte1 = (~byte1) & 0xE0;
 9114     byte1 |= opc;
 9115 
 9116     int byte2 = ((~nds_enc) & 0xf) << 3;
 9117     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
 9118 
 9119     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
 9120   } else {
 9121     int byte1 = vex_r ? VEX_R : 0;
 9122     byte1 = (~byte1) & 0x80;
 9123     byte1 |= ((~nds_enc) & 0xf) << 3;
 9124     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
 9125     emit_int16((unsigned char)VEX_2bytes, byte1);
 9126   }
 9127 }
 9128 
 9129 // This is a 4 byte encoding
 9130 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
 9131   // EVEX 0x62 prefix
 9132   // byte1 = EVEX_4bytes;
 9133 
 9134   bool vex_w = _attributes->is_rex_vex_w();
 9135   int evex_encoding = (vex_w ? VEX_W : 0);
 9136   // EVEX.b is not currently used for broadcast of single element or data rounding modes
 9137   _attributes->set_evex_encoding(evex_encoding);
 9138 
 9139   // P0: byte 2, initialized to RXBR`00mm
 9140   // instead of not'd
 9141   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
 9142   byte2 = (~byte2) & 0xF0;
 9143   // confine opc opcode extensions in mm bits to lower two bits
 9144   // of form {0F, 0F_38, 0F_3A}
 9145   byte2 |= opc;
 9146 
 9147   // P1: byte 3 as Wvvvv1pp
 9148   int byte3 = ((~nds_enc) & 0xf) << 3;
 9149   // p[10] is always 1
 9150   byte3 |= EVEX_F;
 9151   byte3 |= (vex_w & 1) << 7;
 9152   // confine pre opcode extensions in pp bits to lower two bits
 9153   // of form {66, F3, F2}
 9154   byte3 |= pre;
 9155 
 9156   // P2: byte 4 as zL'Lbv'aaa
 9157   // kregs are implemented in the low 3 bits as aaa
 9158   int byte4 = (_attributes->is_no_reg_mask()) ?
 9159               0 :
 9160               _attributes->get_embedded_opmask_register_specifier();
 9161   // EVEX.v` for extending EVEX.vvvv or VIDX
 9162   byte4 |= (evex_v ? 0: EVEX_V);
 9163   // third EXEC.b for broadcast actions
 9164   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
 9165   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
 9166   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
 9167   // last is EVEX.z for zero/merge actions
 9168   if (_attributes->is_no_reg_mask() == false &&
 9169       _attributes->get_embedded_opmask_register_specifier() != 0) {
 9170     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
 9171   }
 9172 
 9173   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
 9174 }
 9175 
 9176 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
 9177   bool vex_r = (xreg_enc & 8) == 8;
 9178   bool vex_b = adr.base_needs_rex();
 9179   bool vex_x;
 9180   if (adr.isxmmindex()) {
 9181     vex_x = adr.xmmindex_needs_rex();
 9182   } else {
 9183     vex_x = adr.index_needs_rex();
 9184   }
 9185   set_attributes(attributes);
 9186   attributes->set_current_assembler(this);
 9187 
 9188   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
 9189   // is allowed in legacy mode and has resources which will fit in it.
 9190   // Pure EVEX instructions will have is_evex_instruction set in their definition.
 9191   if (!attributes->is_legacy_mode()) {
 9192     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
 9193       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
 9194           attributes->set_is_legacy_mode();
 9195       }
 9196     }
 9197   }
 9198 
 9199   if (UseAVX > 2) {
 9200     assert(((!attributes->uses_vl()) ||
 9201             (attributes->get_vector_len() == AVX_512bit) ||
 9202             (!_legacy_mode_vl) ||
 9203             (attributes->is_legacy_mode())),"XMM register should be 0-15");
 9204     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
 9205   }
 9206 
 9207   clear_managed();
 9208   if (UseAVX > 2 && !attributes->is_legacy_mode())
 9209   {
 9210     bool evex_r = (xreg_enc >= 16);
 9211     bool evex_v;
 9212     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
 9213     if (adr.isxmmindex())  {
 9214       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
 9215     } else {
 9216       evex_v = (nds_enc >= 16);
 9217     }
 9218     attributes->set_is_evex_instruction();
 9219     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
 9220   } else {
 9221     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
 9222       attributes->set_rex_vex_w(false);
 9223     }
 9224     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
 9225   }
 9226 }
 9227 
 9228 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
 9229   bool vex_r = (dst_enc & 8) == 8;
 9230   bool vex_b = (src_enc & 8) == 8;
 9231   bool vex_x = false;
 9232   set_attributes(attributes);
 9233   attributes->set_current_assembler(this);
 9234 
 9235   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
 9236   // is allowed in legacy mode and has resources which will fit in it.
 9237   // Pure EVEX instructions will have is_evex_instruction set in their definition.
 9238   if (!attributes->is_legacy_mode()) {
 9239     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
 9240       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
 9241           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
 9242           attributes->set_is_legacy_mode();
 9243       }
 9244     }
 9245   }
 9246 
 9247   if (UseAVX > 2) {
 9248     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
 9249     // Instruction with uses_vl true are vector instructions
 9250     // All the vector instructions with AVX_512bit length can have legacy_mode as false
 9251     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
 9252     // Rest all should have legacy_mode set as true
 9253     assert(((!attributes->uses_vl()) ||
 9254             (attributes->get_vector_len() == AVX_512bit) ||
 9255             (!_legacy_mode_vl) ||
 9256             (attributes->is_legacy_mode())),"XMM register should be 0-15");
 9257     // Instruction with legacy_mode true should have dst, nds and src < 15
 9258     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
 9259   }
 9260 
 9261   clear_managed();
 9262   if (UseAVX > 2 && !attributes->is_legacy_mode())
 9263   {
 9264     bool evex_r = (dst_enc >= 16);
 9265     bool evex_v = (nds_enc >= 16);
 9266     // can use vex_x as bank extender on rm encoding
 9267     vex_x = (src_enc >= 16);
 9268     attributes->set_is_evex_instruction();
 9269     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
 9270   } else {
 9271     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
 9272       attributes->set_rex_vex_w(false);
 9273     }
 9274     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
 9275   }
 9276 
 9277   // return modrm byte components for operands
 9278   return (((dst_enc & 7) << 3) | (src_enc & 7));
 9279 }
 9280 
 9281 
 9282 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
 9283                             VexOpcode opc, InstructionAttr *attributes) {
 9284   if (UseAVX > 0) {
 9285     int xreg_enc = xreg->encoding();
 9286     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
 9287     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
 9288   } else {
 9289     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
 9290     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
 9291   }
 9292 }
 9293 
 9294 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
 9295                                       VexOpcode opc, InstructionAttr *attributes) {
 9296   int dst_enc = dst->encoding();
 9297   int src_enc = src->encoding();
 9298   if (UseAVX > 0) {
 9299     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
 9300     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
 9301   } else {
 9302     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
 9303     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
 9304   }
 9305 }
 9306 
 9307 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9308   assert(VM_Version::supports_avx(), "");
 9309   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 9311   emit_int16(0x5F, (0xC0 | encode));
 9312 }
 9313 
 9314 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9315   assert(VM_Version::supports_avx(), "");
 9316   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9317   attributes.set_rex_vex_w_reverted();
 9318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 9319   emit_int16(0x5F, (0xC0 | encode));
 9320 }
 9321 
 9322 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9323   assert(VM_Version::supports_avx(), "");
 9324   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9325   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 9326   emit_int16(0x5D, (0xC0 | encode));
 9327 }
 9328 
 9329 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9330   assert(VM_Version::supports_avx(), "");
 9331   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9332   attributes.set_rex_vex_w_reverted();
 9333   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 9334   emit_int16(0x5D, (0xC0 | encode));
 9335 }
 9336 
 9337 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
 9338   assert(VM_Version::supports_avx(), "");
 9339   assert(vector_len <= AVX_256bit, "");
 9340   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9341   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9342   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
 9343 }
 9344 
 9345 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9346   assert(VM_Version::supports_avx(), "");
 9347   assert(vector_len <= AVX_256bit, "");
 9348   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9349   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9350   int src2_enc = src2->encoding();
 9351   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9352 }
 9353 
 9354 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9355   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
 9356   assert(vector_len <= AVX_256bit, "");
 9357   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9358   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9359   int src2_enc = src2->encoding();
 9360   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9361 }
 9362 
 9363 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 9364   assert(VM_Version::supports_avx2(), "");
 9365   assert(vector_len <= AVX_256bit, "");
 9366   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9367   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9368   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
 9369 }
 9370 
 9371 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
 9372   assert(VM_Version::supports_avx(), "");
 9373   assert(vector_len <= AVX_256bit, "");
 9374   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9375   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9376   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
 9377 }
 9378 
 9379 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9380                         ComparisonPredicateFP comparison, int vector_len) {
 9381   assert(VM_Version::supports_evex(), "");
 9382   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
 9383   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9384   attributes.set_is_evex_instruction();
 9385   attributes.set_embedded_opmask_register_specifier(mask);
 9386   attributes.reset_is_clear_context();
 9387   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9388   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
 9389 }
 9390 
 9391 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9392                         ComparisonPredicateFP comparison, int vector_len) {
 9393   assert(VM_Version::supports_evex(), "");
 9394   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
 9395   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9396   attributes.set_is_evex_instruction();
 9397   attributes.set_embedded_opmask_register_specifier(mask);
 9398   attributes.reset_is_clear_context();
 9399   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9400   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
 9401 }
 9402 
 9403 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
 9404   assert(VM_Version::supports_sse4_1(), "");
 9405   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9407   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9408   emit_int16(0x14, (0xC0 | encode));
 9409 }
 9410 
 9411 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
 9412   assert(VM_Version::supports_sse4_1(), "");
 9413   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9415   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9416   emit_int16(0x15, (0xC0 | encode));
 9417 }
 9418 
 9419 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
 9420   assert(VM_Version::supports_sse4_1(), "");
 9421   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9422   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9423   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9424   emit_int16(0x10, (0xC0 | encode));
 9425 }
 9426 
 9427 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9428   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
 9429   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9430   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9431   int src2_enc = src2->encoding();
 9432   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9433 }
 9434 
 9435 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 9436   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9437   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9438   emit_int24(0x0C, (0xC0 | encode), imm8);
 9439 }
 9440 
 9441 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9442   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9443   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9445   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9446   emit_int16(0x64, (0xC0 | encode));
 9447 }
 9448 
 9449 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9450   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9451   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9452   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9453   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9454   emit_int16(0x65, (0xC0 | encode));
 9455 }
 9456 
 9457 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9458   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9459   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9460   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9461   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9462   emit_int16(0x66, (0xC0 | encode));
 9463 }
 9464 
 9465 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9466   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9467   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9469   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9470   emit_int16(0x37, (0xC0 | encode));
 9471 }
 9472 
 9473 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9474                         int comparison, bool is_signed, int vector_len) {
 9475   assert(VM_Version::supports_evex(), "");
 9476   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9477   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
 9478   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9479   attributes.set_is_evex_instruction();
 9480   attributes.set_embedded_opmask_register_specifier(mask);
 9481   attributes.reset_is_clear_context();
 9482   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9483   int opcode = is_signed ? 0x1F : 0x1E;
 9484   emit_int24(opcode, (0xC0 | encode), comparison);
 9485 }
 9486 
 9487 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9488                         int comparison, bool is_signed, int vector_len) {
 9489   assert(VM_Version::supports_evex(), "");
 9490   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9491   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
 9492   InstructionMark im(this);
 9493   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9494   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 9495   attributes.set_is_evex_instruction();
 9496   attributes.set_embedded_opmask_register_specifier(mask);
 9497   attributes.reset_is_clear_context();
 9498   int dst_enc = kdst->encoding();
 9499   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9500   int opcode = is_signed ? 0x1F : 0x1E;
 9501   emit_int8((unsigned char)opcode);
 9502   emit_operand(as_Register(dst_enc), src);
 9503   emit_int8((unsigned char)comparison);
 9504 }
 9505 
 9506 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9507                         int comparison, bool is_signed, int vector_len) {
 9508   assert(VM_Version::supports_evex(), "");
 9509   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9510   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
 9511   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9512   attributes.set_is_evex_instruction();
 9513   attributes.set_embedded_opmask_register_specifier(mask);
 9514   attributes.reset_is_clear_context();
 9515   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9516   int opcode = is_signed ? 0x1F : 0x1E;
 9517   emit_int24(opcode, (0xC0 | encode), comparison);
 9518 }
 9519 
 9520 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9521                         int comparison, bool is_signed, int vector_len) {
 9522   assert(VM_Version::supports_evex(), "");
 9523   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9524   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
 9525   InstructionMark im(this);
 9526   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9527   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 9528   attributes.set_is_evex_instruction();
 9529   attributes.set_embedded_opmask_register_specifier(mask);
 9530   attributes.reset_is_clear_context();
 9531   int dst_enc = kdst->encoding();
 9532   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9533   int opcode = is_signed ? 0x1F : 0x1E;
 9534   emit_int8((unsigned char)opcode);
 9535   emit_operand(as_Register(dst_enc), src);
 9536   emit_int8((unsigned char)comparison);
 9537 }
 9538 
 9539 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9540                         int comparison, bool is_signed, int vector_len) {
 9541   assert(VM_Version::supports_evex(), "");
 9542   assert(VM_Version::supports_avx512bw(), "");
 9543   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9544   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
 9545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9546   attributes.set_is_evex_instruction();
 9547   attributes.set_embedded_opmask_register_specifier(mask);
 9548   attributes.reset_is_clear_context();
 9549   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9550   int opcode = is_signed ? 0x3F : 0x3E;
 9551   emit_int24(opcode, (0xC0 | encode), comparison);
 9552 }
 9553 
 9554 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9555                         int comparison, bool is_signed, int vector_len) {
 9556   assert(VM_Version::supports_evex(), "");
 9557   assert(VM_Version::supports_avx512bw(), "");
 9558   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9559   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
 9560   InstructionMark im(this);
 9561   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9562   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 9563   attributes.set_is_evex_instruction();
 9564   attributes.set_embedded_opmask_register_specifier(mask);
 9565   attributes.reset_is_clear_context();
 9566   int dst_enc = kdst->encoding();
 9567   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9568   int opcode = is_signed ? 0x3F : 0x3E;
 9569   emit_int8((unsigned char)opcode);
 9570   emit_operand(as_Register(dst_enc), src);
 9571   emit_int8((unsigned char)comparison);
 9572 }
 9573 
 9574 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9575                         int comparison, bool is_signed, int vector_len) {
 9576   assert(VM_Version::supports_evex(), "");
 9577   assert(VM_Version::supports_avx512bw(), "");
 9578   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9579   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
 9580   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9581   attributes.set_is_evex_instruction();
 9582   attributes.set_embedded_opmask_register_specifier(mask);
 9583   attributes.reset_is_clear_context();
 9584   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9585   int opcode = is_signed ? 0x3F : 0x3E;
 9586   emit_int24(opcode, (0xC0 | encode), comparison);
 9587 }
 9588 
 9589 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9590                         int comparison, bool is_signed, int vector_len) {
 9591   assert(VM_Version::supports_evex(), "");
 9592   assert(VM_Version::supports_avx512bw(), "");
 9593   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9594   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
 9595   InstructionMark im(this);
 9596   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9597   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 9598   attributes.set_is_evex_instruction();
 9599   attributes.set_embedded_opmask_register_specifier(mask);
 9600   attributes.reset_is_clear_context();
 9601   int dst_enc = kdst->encoding();
 9602   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9603   int opcode = is_signed ? 0x3F : 0x3E;
 9604   emit_int8((unsigned char)opcode);
 9605   emit_operand(as_Register(dst_enc), src);
 9606   emit_int8((unsigned char)comparison);
 9607 }
 9608 
 9609 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
 9610   assert(VM_Version::supports_avx(), "");
 9611   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9613   int mask_enc = mask->encoding();
 9614   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
 9615 }
 9616 
 9617 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9618   assert(VM_Version::supports_evex(), "");
 9619   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
 9620   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9621   attributes.set_is_evex_instruction();
 9622   attributes.set_embedded_opmask_register_specifier(mask);
 9623   if (merge) {
 9624     attributes.reset_is_clear_context();
 9625   }
 9626   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9627   emit_int16(0x65, (0xC0 | encode));
 9628 }
 9629 
 9630 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9631   assert(VM_Version::supports_evex(), "");
 9632   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
 9633   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9634   attributes.set_is_evex_instruction();
 9635   attributes.set_embedded_opmask_register_specifier(mask);
 9636   if (merge) {
 9637     attributes.reset_is_clear_context();
 9638   }
 9639   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9640   emit_int16(0x65, (0xC0 | encode));
 9641 }
 9642 
 9643 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9644   assert(VM_Version::supports_evex(), "");
 9645   assert(VM_Version::supports_avx512bw(), "");
 9646   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
 9647   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9648   attributes.set_is_evex_instruction();
 9649   attributes.set_embedded_opmask_register_specifier(mask);
 9650   if (merge) {
 9651     attributes.reset_is_clear_context();
 9652   }
 9653   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9654   emit_int16(0x66, (0xC0 | encode));
 9655 }
 9656 
 9657 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9658   assert(VM_Version::supports_evex(), "");
 9659   assert(VM_Version::supports_avx512bw(), "");
 9660   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
 9661   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9662   attributes.set_is_evex_instruction();
 9663   attributes.set_embedded_opmask_register_specifier(mask);
 9664   if (merge) {
 9665     attributes.reset_is_clear_context();
 9666   }
 9667   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9668   emit_int16(0x66, (0xC0 | encode));
 9669 }
 9670 
 9671 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9672   assert(VM_Version::supports_evex(), "");
 9673   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
 9674   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9675   attributes.set_is_evex_instruction();
 9676   attributes.set_embedded_opmask_register_specifier(mask);
 9677   if (merge) {
 9678     attributes.reset_is_clear_context();
 9679   }
 9680   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9681   emit_int16(0x64, (0xC0 | encode));
 9682 }
 9683 
 9684 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9685   assert(VM_Version::supports_evex(), "");
 9686   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
 9687   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9688   attributes.set_is_evex_instruction();
 9689   attributes.set_embedded_opmask_register_specifier(mask);
 9690   if (merge) {
 9691     attributes.reset_is_clear_context();
 9692   }
 9693   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9694   emit_int16(0x64, (0xC0 | encode));
 9695 }
 9696 
 9697 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
 9698   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
 9699   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9700   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 9701   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 9702 }
 9703 
 9704 void Assembler::shlxl(Register dst, Register src1, Register src2) {
 9705   assert(VM_Version::supports_bmi2(), "");
 9706   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9707   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9708   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9709 }
 9710 
 9711 void Assembler::shlxq(Register dst, Register src1, Register src2) {
 9712   assert(VM_Version::supports_bmi2(), "");
 9713   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9714   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9715   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9716 }
 9717 
 9718 void Assembler::shrxl(Register dst, Register src1, Register src2) {
 9719   assert(VM_Version::supports_bmi2(), "");
 9720   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9721   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
 9722   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9723 }
 9724 
 9725 void Assembler::shrxq(Register dst, Register src1, Register src2) {
 9726   assert(VM_Version::supports_bmi2(), "");
 9727   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9728   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
 9729   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9730 }
 9731 
 9732 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
 9733   assert(VM_Version::supports_avx512vlbw(), "");
 9734   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9735   attributes.set_is_evex_instruction();
 9736   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 9737   emit_int16(0x29, (0xC0 | encode));
 9738 }
 9739 
 9740 #ifndef _LP64
 9741 
 9742 void Assembler::incl(Register dst) {
 9743   // Don't use it directly. Use MacroAssembler::incrementl() instead.
 9744   emit_int8(0x40 | dst->encoding());
 9745 }
 9746 
 9747 void Assembler::lea(Register dst, Address src) {
 9748   leal(dst, src);
 9749 }
 9750 
 9751 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
 9752   InstructionMark im(this);
 9753   emit_int8((unsigned char)0xC7);
 9754   emit_operand(rax, dst);
 9755   emit_data((int)imm32, rspec, 0);
 9756 }
 9757 
 9758 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
 9759   InstructionMark im(this);
 9760   int encode = prefix_and_encode(dst->encoding());
 9761   emit_int8((0xB8 | encode));
 9762   emit_data((int)imm32, rspec, 0);
 9763 }
 9764 
 9765 void Assembler::popa() { // 32bit
 9766   emit_int8(0x61);
 9767 }
 9768 
 9769 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
 9770   InstructionMark im(this);
 9771   emit_int8(0x68);
 9772   emit_data(imm32, rspec, 0);
 9773 }
 9774 
 9775 void Assembler::pusha() { // 32bit
 9776   emit_int8(0x60);
 9777 }
 9778 
 9779 void Assembler::set_byte_if_not_zero(Register dst) {
 9780   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
 9781 }
 9782 
 9783 #else // LP64
 9784 
 9785 void Assembler::set_byte_if_not_zero(Register dst) {
 9786   int enc = prefix_and_encode(dst->encoding(), true);
 9787   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
 9788 }
 9789 
 9790 // 64bit only pieces of the assembler
 9791 // This should only be used by 64bit instructions that can use rip-relative
 9792 // it cannot be used by instructions that want an immediate value.
 9793 
 9794 bool Assembler::reachable(AddressLiteral adr) {
 9795   int64_t disp;
 9796   relocInfo::relocType relocType = adr.reloc();
 9797 
 9798   // None will force a 64bit literal to the code stream. Likely a placeholder
 9799   // for something that will be patched later and we need to certain it will
 9800   // always be reachable.
 9801   if (relocType == relocInfo::none) {
 9802     return false;
 9803   }
 9804   if (relocType == relocInfo::internal_word_type) {
 9805     // This should be rip relative and easily reachable.
 9806     return true;
 9807   }
 9808   if (relocType == relocInfo::virtual_call_type ||
 9809       relocType == relocInfo::opt_virtual_call_type ||
 9810       relocType == relocInfo::static_call_type ||
 9811       relocType == relocInfo::static_stub_type ) {
 9812     // This should be rip relative within the code cache and easily
 9813     // reachable until we get huge code caches. (At which point
 9814     // ic code is going to have issues).
 9815     return true;
 9816   }
 9817   if (relocType != relocInfo::external_word_type &&
 9818       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
 9819       relocType != relocInfo::poll_type &&         // relocs to identify them
 9820       relocType != relocInfo::runtime_call_type ) {
 9821     return false;
 9822   }
 9823 
 9824   // Stress the correction code
 9825   if (ForceUnreachable) {
 9826     // Must be runtimecall reloc, see if it is in the codecache
 9827     // Flipping stuff in the codecache to be unreachable causes issues
 9828     // with things like inline caches where the additional instructions
 9829     // are not handled.
 9830     if (CodeCache::find_blob(adr._target) == NULL) {
 9831       return false;
 9832     }
 9833   }
 9834   // For external_word_type/runtime_call_type if it is reachable from where we
 9835   // are now (possibly a temp buffer) and where we might end up
 9836   // anywhere in the codeCache then we are always reachable.
 9837   // This would have to change if we ever save/restore shared code
 9838   // to be more pessimistic.
 9839   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
 9840   if (!is_simm32(disp)) return false;
 9841   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
 9842   if (!is_simm32(disp)) return false;
 9843 
 9844   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
 9845 
 9846   // Because rip relative is a disp + address_of_next_instruction and we
 9847   // don't know the value of address_of_next_instruction we apply a fudge factor
 9848   // to make sure we will be ok no matter the size of the instruction we get placed into.
 9849   // We don't have to fudge the checks above here because they are already worst case.
 9850 
 9851   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
 9852   // + 4 because better safe than sorry.
 9853   const int fudge = 12 + 4;
 9854   if (disp < 0) {
 9855     disp -= fudge;
 9856   } else {
 9857     disp += fudge;
 9858   }
 9859   return is_simm32(disp);
 9860 }
 9861 
 9862 void Assembler::emit_data64(jlong data,
 9863                             relocInfo::relocType rtype,
 9864                             int format) {
 9865   if (rtype == relocInfo::none) {
 9866     emit_int64(data);
 9867   } else {
 9868     emit_data64(data, Relocation::spec_simple(rtype), format);
 9869   }
 9870 }
 9871 
 9872 void Assembler::emit_data64(jlong data,
 9873                             RelocationHolder const& rspec,
 9874                             int format) {
 9875   assert(imm_operand == 0, "default format must be immediate in this file");
 9876   assert(imm_operand == format, "must be immediate");
 9877   assert(inst_mark() != NULL, "must be inside InstructionMark");
 9878   // Do not use AbstractAssembler::relocate, which is not intended for
 9879   // embedded words.  Instead, relocate to the enclosing instruction.
 9880   code_section()->relocate(inst_mark(), rspec, format);
 9881 #ifdef ASSERT
 9882   check_relocation(rspec, format);
 9883 #endif
 9884   emit_int64(data);
 9885 }
 9886 
 9887 void Assembler::prefix(Register reg) {
 9888   if (reg->encoding() >= 8) {
 9889     prefix(REX_B);
 9890   }
 9891 }
 9892 
 9893 void Assembler::prefix(Register dst, Register src, Prefix p) {
 9894   if (src->encoding() >= 8) {
 9895     p = (Prefix)(p | REX_B);
 9896   }
 9897   if (dst->encoding() >= 8) {
 9898     p = (Prefix)(p | REX_R);
 9899   }
 9900   if (p != Prefix_EMPTY) {
 9901     // do not generate an empty prefix
 9902     prefix(p);
 9903   }
 9904 }
 9905 
 9906 void Assembler::prefix(Register dst, Address adr, Prefix p) {
 9907   if (adr.base_needs_rex()) {
 9908     if (adr.index_needs_rex()) {
 9909       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
 9910     } else {
 9911       prefix(REX_B);
 9912     }
 9913   } else {
 9914     if (adr.index_needs_rex()) {
 9915       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
 9916     }
 9917   }
 9918   if (dst->encoding() >= 8) {
 9919     p = (Prefix)(p | REX_R);
 9920   }
 9921   if (p != Prefix_EMPTY) {
 9922     // do not generate an empty prefix
 9923     prefix(p);
 9924   }
 9925 }
 9926 
 9927 void Assembler::prefix(Address adr) {
 9928   if (adr.base_needs_rex()) {
 9929     if (adr.index_needs_rex()) {
 9930       prefix(REX_XB);
 9931     } else {
 9932       prefix(REX_B);
 9933     }
 9934   } else {
 9935     if (adr.index_needs_rex()) {
 9936       prefix(REX_X);
 9937     }
 9938   }
 9939 }
 9940 
 9941 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
 9942   if (reg->encoding() < 8) {
 9943     if (adr.base_needs_rex()) {
 9944       if (adr.index_needs_rex()) {
 9945         prefix(REX_XB);
 9946       } else {
 9947         prefix(REX_B);
 9948       }
 9949     } else {
 9950       if (adr.index_needs_rex()) {
 9951         prefix(REX_X);
 9952       } else if (byteinst && reg->encoding() >= 4) {
 9953         prefix(REX);
 9954       }
 9955     }
 9956   } else {
 9957     if (adr.base_needs_rex()) {
 9958       if (adr.index_needs_rex()) {
 9959         prefix(REX_RXB);
 9960       } else {
 9961         prefix(REX_RB);
 9962       }
 9963     } else {
 9964       if (adr.index_needs_rex()) {
 9965         prefix(REX_RX);
 9966       } else {
 9967         prefix(REX_R);
 9968       }
 9969     }
 9970   }
 9971 }
 9972 
 9973 void Assembler::prefix(Address adr, XMMRegister reg) {
 9974   if (reg->encoding() < 8) {
 9975     if (adr.base_needs_rex()) {
 9976       if (adr.index_needs_rex()) {
 9977         prefix(REX_XB);
 9978       } else {
 9979         prefix(REX_B);
 9980       }
 9981     } else {
 9982       if (adr.index_needs_rex()) {
 9983         prefix(REX_X);
 9984       }
 9985     }
 9986   } else {
 9987     if (adr.base_needs_rex()) {
 9988       if (adr.index_needs_rex()) {
 9989         prefix(REX_RXB);
 9990       } else {
 9991         prefix(REX_RB);
 9992       }
 9993     } else {
 9994       if (adr.index_needs_rex()) {
 9995         prefix(REX_RX);
 9996       } else {
 9997         prefix(REX_R);
 9998       }
 9999     }
10000   }
10001 }
10002 
10003 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
10004   if (reg_enc >= 8) {
10005     prefix(REX_B);
10006     reg_enc -= 8;
10007   } else if (byteinst && reg_enc >= 4) {
10008     prefix(REX);
10009   }
10010   return reg_enc;
10011 }
10012 
10013 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
10014   if (dst_enc < 8) {
10015     if (src_enc >= 8) {
10016       prefix(REX_B);
10017       src_enc -= 8;
10018     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
10019       prefix(REX);
10020     }
10021   } else {
10022     if (src_enc < 8) {
10023       prefix(REX_R);
10024     } else {
10025       prefix(REX_RB);
10026       src_enc -= 8;
10027     }
10028     dst_enc -= 8;
10029   }
10030   return dst_enc << 3 | src_enc;
10031 }
10032 
10033 int8_t Assembler::get_prefixq(Address adr) {
10034   int8_t prfx = get_prefixq(adr, rax);
10035   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
10036   return prfx;
10037 }
10038 
10039 int8_t Assembler::get_prefixq(Address adr, Register src) {
10040   int8_t prfx = (int8_t)(REX_W +
10041                          ((int)adr.base_needs_rex()) +
10042                          ((int)adr.index_needs_rex() << 1) +
10043                          ((int)(src->encoding() >= 8) << 2));
10044 #ifdef ASSERT
10045   if (src->encoding() < 8) {
10046     if (adr.base_needs_rex()) {
10047       if (adr.index_needs_rex()) {
10048         assert(prfx == REX_WXB, "must be");
10049       } else {
10050         assert(prfx == REX_WB, "must be");
10051       }
10052     } else {
10053       if (adr.index_needs_rex()) {
10054         assert(prfx == REX_WX, "must be");
10055       } else {
10056         assert(prfx == REX_W, "must be");
10057       }
10058     }
10059   } else {
10060     if (adr.base_needs_rex()) {
10061       if (adr.index_needs_rex()) {
10062         assert(prfx == REX_WRXB, "must be");
10063       } else {
10064         assert(prfx == REX_WRB, "must be");
10065       }
10066     } else {
10067       if (adr.index_needs_rex()) {
10068         assert(prfx == REX_WRX, "must be");
10069       } else {
10070         assert(prfx == REX_WR, "must be");
10071       }
10072     }
10073   }
10074 #endif
10075   return prfx;
10076 }
10077 
10078 void Assembler::prefixq(Address adr) {
10079   emit_int8(get_prefixq(adr));
10080 }
10081 
10082 void Assembler::prefixq(Address adr, Register src) {
10083   emit_int8(get_prefixq(adr, src));
10084 }
10085 
10086 void Assembler::prefixq(Address adr, XMMRegister src) {
10087   if (src->encoding() < 8) {
10088     if (adr.base_needs_rex()) {
10089       if (adr.index_needs_rex()) {
10090         prefix(REX_WXB);
10091       } else {
10092         prefix(REX_WB);
10093       }
10094     } else {
10095       if (adr.index_needs_rex()) {
10096         prefix(REX_WX);
10097       } else {
10098         prefix(REX_W);
10099       }
10100     }
10101   } else {
10102     if (adr.base_needs_rex()) {
10103       if (adr.index_needs_rex()) {
10104         prefix(REX_WRXB);
10105       } else {
10106         prefix(REX_WRB);
10107       }
10108     } else {
10109       if (adr.index_needs_rex()) {
10110         prefix(REX_WRX);
10111       } else {
10112         prefix(REX_WR);
10113       }
10114     }
10115   }
10116 }
10117 
10118 int Assembler::prefixq_and_encode(int reg_enc) {
10119   if (reg_enc < 8) {
10120     prefix(REX_W);
10121   } else {
10122     prefix(REX_WB);
10123     reg_enc -= 8;
10124   }
10125   return reg_enc;
10126 }
10127 
10128 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
10129   if (dst_enc < 8) {
10130     if (src_enc < 8) {
10131       prefix(REX_W);
10132     } else {
10133       prefix(REX_WB);
10134       src_enc -= 8;
10135     }
10136   } else {
10137     if (src_enc < 8) {
10138       prefix(REX_WR);
10139     } else {
10140       prefix(REX_WRB);
10141       src_enc -= 8;
10142     }
10143     dst_enc -= 8;
10144   }
10145   return dst_enc << 3 | src_enc;
10146 }
10147 
10148 void Assembler::adcq(Register dst, int32_t imm32) {
10149   (void) prefixq_and_encode(dst->encoding());
10150   emit_arith(0x81, 0xD0, dst, imm32);
10151 }
10152 
10153 void Assembler::adcq(Register dst, Address src) {
10154   InstructionMark im(this);
10155   emit_int16(get_prefixq(src, dst), 0x13);
10156   emit_operand(dst, src);
10157 }
10158 
10159 void Assembler::adcq(Register dst, Register src) {
10160   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10161   emit_arith(0x13, 0xC0, dst, src);
10162 }
10163 
10164 void Assembler::addq(Address dst, int32_t imm32) {
10165   InstructionMark im(this);
10166   prefixq(dst);
10167   emit_arith_operand(0x81, rax, dst, imm32);
10168 }
10169 
10170 void Assembler::addq(Address dst, Register src) {
10171   InstructionMark im(this);
10172   emit_int16(get_prefixq(dst, src), 0x01);
10173   emit_operand(src, dst);
10174 }
10175 
10176 void Assembler::addq(Register dst, int32_t imm32) {
10177   (void) prefixq_and_encode(dst->encoding());
10178   emit_arith(0x81, 0xC0, dst, imm32);
10179 }
10180 
10181 void Assembler::addq(Register dst, Address src) {
10182   InstructionMark im(this);
10183   emit_int16(get_prefixq(src, dst), 0x03);
10184   emit_operand(dst, src);
10185 }
10186 
10187 void Assembler::addq(Register dst, Register src) {
10188   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10189   emit_arith(0x03, 0xC0, dst, src);
10190 }
10191 
10192 void Assembler::adcxq(Register dst, Register src) {
10193   //assert(VM_Version::supports_adx(), "adx instructions not supported");
10194   emit_int8(0x66);
10195   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10196   emit_int32(0x0F,
10197              0x38,
10198              (unsigned char)0xF6,
10199              (0xC0 | encode));
10200 }
10201 
10202 void Assembler::adoxq(Register dst, Register src) {
10203   //assert(VM_Version::supports_adx(), "adx instructions not supported");
10204   emit_int8((unsigned char)0xF3);
10205   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10206   emit_int32(0x0F,
10207              0x38,
10208              (unsigned char)0xF6,
10209              (0xC0 | encode));
10210 }
10211 
10212 void Assembler::andq(Address dst, int32_t imm32) {
10213   InstructionMark im(this);
10214   prefixq(dst);
10215   emit_arith_operand(0x81, as_Register(4), dst, imm32);
10216 }
10217 
10218 void Assembler::andq(Register dst, int32_t imm32) {
10219   (void) prefixq_and_encode(dst->encoding());
10220   emit_arith(0x81, 0xE0, dst, imm32);
10221 }
10222 
10223 void Assembler::andq(Register dst, Address src) {
10224   InstructionMark im(this);
10225   emit_int16(get_prefixq(src, dst), 0x23);
10226   emit_operand(dst, src);
10227 }
10228 
10229 void Assembler::andq(Register dst, Register src) {
10230   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10231   emit_arith(0x23, 0xC0, dst, src);
10232 }
10233 
10234 void Assembler::andq(Address dst, Register src) {
10235   InstructionMark im(this);
10236   emit_int16(get_prefixq(dst, src), 0x21);
10237   emit_operand(src, dst);
10238 }
10239 
10240 void Assembler::andnq(Register dst, Register src1, Register src2) {
10241   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10242   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10243   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10244   emit_int16((unsigned char)0xF2, (0xC0 | encode));
10245 }
10246 
10247 void Assembler::andnq(Register dst, Register src1, Address src2) {
10248   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10249   InstructionMark im(this);
10250   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10251   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10252   emit_int8((unsigned char)0xF2);
10253   emit_operand(dst, src2);
10254 }
10255 
10256 void Assembler::bsfq(Register dst, Register src) {
10257   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10258   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
10259 }
10260 
10261 void Assembler::bsrq(Register dst, Register src) {
10262   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10263   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
10264 }
10265 
10266 void Assembler::bswapq(Register reg) {
10267   int encode = prefixq_and_encode(reg->encoding());
10268   emit_int16(0x0F, (0xC8 | encode));
10269 }
10270 
10271 void Assembler::blsiq(Register dst, Register src) {
10272   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10273   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10274   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10275   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10276 }
10277 
10278 void Assembler::blsiq(Register dst, Address src) {
10279   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10280   InstructionMark im(this);
10281   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10282   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10283   emit_int8((unsigned char)0xF3);
10284   emit_operand(rbx, src);
10285 }
10286 
10287 void Assembler::blsmskq(Register dst, Register src) {
10288   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10289   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10290   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10291   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10292 }
10293 
10294 void Assembler::blsmskq(Register dst, Address src) {
10295   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10296   InstructionMark im(this);
10297   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10298   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10299   emit_int8((unsigned char)0xF3);
10300   emit_operand(rdx, src);
10301 }
10302 
10303 void Assembler::blsrq(Register dst, Register src) {
10304   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10305   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10306   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10307   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10308 }
10309 
10310 void Assembler::blsrq(Register dst, Address src) {
10311   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10312   InstructionMark im(this);
10313   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10314   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10315   emit_int8((unsigned char)0xF3);
10316   emit_operand(rcx, src);
10317 }
10318 
10319 void Assembler::cdqq() {
10320   emit_int16(REX_W, (unsigned char)0x99);
10321 }
10322 
10323 void Assembler::clflush(Address adr) {
10324   assert(VM_Version::supports_clflush(), "should do");
10325   prefix(adr);
10326   emit_int16(0x0F, (unsigned char)0xAE);
10327   emit_operand(rdi, adr);
10328 }
10329 
10330 void Assembler::clflushopt(Address adr) {
10331   assert(VM_Version::supports_clflushopt(), "should do!");
10332   // adr should be base reg only with no index or offset
10333   assert(adr.index() == noreg, "index should be noreg");
10334   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
10335   assert(adr.disp() == 0, "displacement should be 0");
10336   // instruction prefix is 0x66
10337   emit_int8(0x66);
10338   prefix(adr);
10339   // opcode family is 0x0F 0xAE
10340   emit_int16(0x0F, (unsigned char)0xAE);
10341   // extended opcode byte is 7 == rdi
10342   emit_operand(rdi, adr);
10343 }
10344 
10345 void Assembler::clwb(Address adr) {
10346   assert(VM_Version::supports_clwb(), "should do!");
10347   // adr should be base reg only with no index or offset
10348   assert(adr.index() == noreg, "index should be noreg");
10349   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
10350   assert(adr.disp() == 0, "displacement should be 0");
10351   // instruction prefix is 0x66
10352   emit_int8(0x66);
10353   prefix(adr);
10354   // opcode family is 0x0f 0xAE
10355   emit_int16(0x0F, (unsigned char)0xAE);
10356   // extended opcode byte is 6 == rsi
10357   emit_operand(rsi, adr);
10358 }
10359 
10360 void Assembler::cmovq(Condition cc, Register dst, Register src) {
10361   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10362   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
10363 }
10364 
10365 void Assembler::cmovq(Condition cc, Register dst, Address src) {
10366   InstructionMark im(this);
10367   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
10368   emit_operand(dst, src);
10369 }
10370 
10371 void Assembler::cmpq(Address dst, int32_t imm32) {
10372   InstructionMark im(this);
10373   emit_int16(get_prefixq(dst), (unsigned char)0x81);
10374   emit_operand(rdi, dst, 4);
10375   emit_int32(imm32);
10376 }
10377 
10378 void Assembler::cmpq(Register dst, int32_t imm32) {
10379   (void) prefixq_and_encode(dst->encoding());
10380   emit_arith(0x81, 0xF8, dst, imm32);
10381 }
10382 
10383 void Assembler::cmpq(Address dst, Register src) {
10384   InstructionMark im(this);
10385   emit_int16(get_prefixq(dst, src), 0x39);
10386   emit_operand(src, dst);
10387 }
10388 
10389 void Assembler::cmpq(Register dst, Register src) {
10390   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10391   emit_arith(0x3B, 0xC0, dst, src);
10392 }
10393 
10394 void Assembler::cmpq(Register dst, Address src) {
10395   InstructionMark im(this);
10396   emit_int16(get_prefixq(src, dst), 0x3B);
10397   emit_operand(dst, src);
10398 }
10399 
10400 void Assembler::cmpxchgq(Register reg, Address adr) {
10401   InstructionMark im(this);
10402   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
10403   emit_operand(reg, adr);
10404 }
10405 
10406 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
10407   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10408   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10409   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10410   emit_int16(0x2A, (0xC0 | encode));
10411 }
10412 
10413 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
10414   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10415   InstructionMark im(this);
10416   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10417   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10418   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10419   emit_int8(0x2A);
10420   emit_operand(dst, src);
10421 }
10422 
10423 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
10424   NOT_LP64(assert(VM_Version::supports_sse(), ""));
10425   InstructionMark im(this);
10426   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10427   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10428   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10429   emit_int8(0x2A);
10430   emit_operand(dst, src);
10431 }
10432 
10433 void Assembler::cvttsd2siq(Register dst, Address src) {
10434   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10435   // F2 REX.W 0F 2C /r
10436   // CVTTSD2SI r64, xmm1/m64
10437   InstructionMark im(this);
10438   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
10439   emit_operand(dst, src);
10440 }
10441 
10442 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
10443   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10444   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10445   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10446   emit_int16(0x2C, (0xC0 | encode));
10447 }
10448 
10449 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
10450   NOT_LP64(assert(VM_Version::supports_sse(), ""));
10451   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10452   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10453   emit_int16(0x2C, (0xC0 | encode));
10454 }
10455 
10456 void Assembler::decl(Register dst) {
10457   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10458   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
10459   int encode = prefix_and_encode(dst->encoding());
10460   emit_int16((unsigned char)0xFF, (0xC8 | encode));
10461 }
10462 
10463 void Assembler::decq(Register dst) {
10464   // Don't use it directly. Use MacroAssembler::decrementq() instead.
10465   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10466   int encode = prefixq_and_encode(dst->encoding());
10467   emit_int16((unsigned char)0xFF, 0xC8 | encode);
10468 }
10469 
10470 void Assembler::decq(Address dst) {
10471   // Don't use it directly. Use MacroAssembler::decrementq() instead.
10472   InstructionMark im(this);
10473   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
10474   emit_operand(rcx, dst);
10475 }
10476 
10477 void Assembler::fxrstor(Address src) {
10478   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
10479   emit_operand(as_Register(1), src);
10480 }
10481 
10482 void Assembler::xrstor(Address src) {
10483   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
10484   emit_operand(as_Register(5), src);
10485 }
10486 
10487 void Assembler::fxsave(Address dst) {
10488   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
10489   emit_operand(as_Register(0), dst);
10490 }
10491 
10492 void Assembler::xsave(Address dst) {
10493   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
10494   emit_operand(as_Register(4), dst);
10495 }
10496 
10497 void Assembler::idivq(Register src) {
10498   int encode = prefixq_and_encode(src->encoding());
10499   emit_int16((unsigned char)0xF7, (0xF8 | encode));
10500 }
10501 
10502 void Assembler::imulq(Register dst, Register src) {
10503   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10504   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
10505 }
10506 
10507 void Assembler::imulq(Register src) {
10508   int encode = prefixq_and_encode(src->encoding());
10509   emit_int16((unsigned char)0xF7, (0xE8 | encode));
10510 }
10511 
10512 void Assembler::imulq(Register dst, Address src, int32_t value) {
10513   InstructionMark im(this);
10514   prefixq(src, dst);
10515   if (is8bit(value)) {
10516     emit_int8((unsigned char)0x6B);
10517     emit_operand(dst, src);
10518     emit_int8(value);
10519   } else {
10520     emit_int8((unsigned char)0x69);
10521     emit_operand(dst, src);
10522     emit_int32(value);
10523   }
10524 }
10525 
10526 void Assembler::imulq(Register dst, Register src, int value) {
10527   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10528   if (is8bit(value)) {
10529     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
10530   } else {
10531     emit_int16(0x69, (0xC0 | encode));
10532     emit_int32(value);
10533   }
10534 }
10535 
10536 void Assembler::imulq(Register dst, Address src) {
10537   InstructionMark im(this);
10538   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
10539   emit_operand(dst, src);
10540 }
10541 
10542 void Assembler::incl(Register dst) {
10543   // Don't use it directly. Use MacroAssembler::incrementl() instead.
10544   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10545   int encode = prefix_and_encode(dst->encoding());
10546   emit_int16((unsigned char)0xFF, (0xC0 | encode));
10547 }
10548 
10549 void Assembler::incq(Register dst) {
10550   // Don't use it directly. Use MacroAssembler::incrementq() instead.
10551   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10552   int encode = prefixq_and_encode(dst->encoding());
10553   emit_int16((unsigned char)0xFF, (0xC0 | encode));
10554 }
10555 
10556 void Assembler::incq(Address dst) {
10557   // Don't use it directly. Use MacroAssembler::incrementq() instead.
10558   InstructionMark im(this);
10559   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
10560   emit_operand(rax, dst);
10561 }
10562 
10563 void Assembler::lea(Register dst, Address src) {
10564   leaq(dst, src);
10565 }
10566 
10567 void Assembler::leaq(Register dst, Address src) {
10568   InstructionMark im(this);
10569   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
10570   emit_operand(dst, src);
10571 }
10572 
10573 void Assembler::mov64(Register dst, int64_t imm64) {
10574   InstructionMark im(this);
10575   int encode = prefixq_and_encode(dst->encoding());
10576   emit_int8(0xB8 | encode);
10577   emit_int64(imm64);
10578 }
10579 
10580 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
10581   InstructionMark im(this);
10582   int encode = prefixq_and_encode(dst->encoding());
10583   emit_int8(0xB8 | encode);
10584   emit_data64(imm64, rtype, format);
10585 }
10586 
10587 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
10588   InstructionMark im(this);
10589   int encode = prefixq_and_encode(dst->encoding());
10590   emit_int8(0xB8 | encode);
10591   emit_data64(imm64, rspec);
10592 }
10593 
10594 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
10595   InstructionMark im(this);
10596   int encode = prefix_and_encode(dst->encoding());
10597   emit_int8(0xB8 | encode);
10598   emit_data((int)imm32, rspec, narrow_oop_operand);
10599 }
10600 
10601 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
10602   InstructionMark im(this);
10603   prefix(dst);
10604   emit_int8((unsigned char)0xC7);
10605   emit_operand(rax, dst, 4);
10606   emit_data((int)imm32, rspec, narrow_oop_operand);
10607 }
10608 
10609 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10610   InstructionMark im(this);
10611   int encode = prefix_and_encode(src1->encoding());
10612   emit_int16((unsigned char)0x81, (0xF8 | encode));
10613   emit_data((int)imm32, rspec, narrow_oop_operand);
10614 }
10615 
10616 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10617   InstructionMark im(this);
10618   prefix(src1);
10619   emit_int8((unsigned char)0x81);
10620   emit_operand(rax, src1, 4);
10621   emit_data((int)imm32, rspec, narrow_oop_operand);
10622 }
10623 
10624 void Assembler::lzcntq(Register dst, Register src) {
10625   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
10626   emit_int8((unsigned char)0xF3);
10627   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10628   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
10629 }
10630 
10631 void Assembler::movdq(XMMRegister dst, Register src) {
10632   // table D-1 says MMX/SSE2
10633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10634   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10635   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10636   emit_int16(0x6E, (0xC0 | encode));
10637 }
10638 
10639 void Assembler::movdq(Register dst, XMMRegister src) {
10640   // table D-1 says MMX/SSE2
10641   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10642   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10643   // swap src/dst to get correct prefix
10644   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10645   emit_int16(0x7E,
10646              (0xC0 | encode));
10647 }
10648 
10649 void Assembler::movq(Register dst, Register src) {
10650   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10651   emit_int16((unsigned char)0x8B,
10652              (0xC0 | encode));
10653 }
10654 
10655 void Assembler::movq(Register dst, Address src) {
10656   InstructionMark im(this);
10657   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
10658   emit_operand(dst, src);
10659 }
10660 
10661 void Assembler::movq(Address dst, Register src) {
10662   InstructionMark im(this);
10663   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
10664   emit_operand(src, dst);
10665 }
10666 
10667 void Assembler::movq(Address dst, int32_t imm32) {
10668   InstructionMark im(this);
10669   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
10670   emit_operand(as_Register(0), dst);
10671   emit_int32(imm32);
10672 }
10673 
10674 void Assembler::movq(Register dst, int32_t imm32) {
10675   int encode = prefixq_and_encode(dst->encoding());
10676   emit_int16((unsigned char)0xC7, (0xC0 | encode));
10677   emit_int32(imm32);
10678 }
10679 
10680 void Assembler::movsbq(Register dst, Address src) {
10681   InstructionMark im(this);
10682   emit_int24(get_prefixq(src, dst),
10683              0x0F,
10684              (unsigned char)0xBE);
10685   emit_operand(dst, src);
10686 }
10687 
10688 void Assembler::movsbq(Register dst, Register src) {
10689   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10690   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
10691 }
10692 
10693 void Assembler::movslq(Register dst, int32_t imm32) {
10694   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
10695   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
10696   // as a result we shouldn't use until tested at runtime...
10697   ShouldNotReachHere();
10698   InstructionMark im(this);
10699   int encode = prefixq_and_encode(dst->encoding());
10700   emit_int8(0xC7 | encode);
10701   emit_int32(imm32);
10702 }
10703 
10704 void Assembler::movslq(Address dst, int32_t imm32) {
10705   assert(is_simm32(imm32), "lost bits");
10706   InstructionMark im(this);
10707   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
10708   emit_operand(rax, dst, 4);
10709   emit_int32(imm32);
10710 }
10711 
10712 void Assembler::movslq(Register dst, Address src) {
10713   InstructionMark im(this);
10714   emit_int16(get_prefixq(src, dst), 0x63);
10715   emit_operand(dst, src);
10716 }
10717 
10718 void Assembler::movslq(Register dst, Register src) {
10719   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10720   emit_int16(0x63, (0xC0 | encode));
10721 }
10722 
10723 void Assembler::movswq(Register dst, Address src) {
10724   InstructionMark im(this);
10725   emit_int24(get_prefixq(src, dst),
10726              0x0F,
10727              (unsigned char)0xBF);
10728   emit_operand(dst, src);
10729 }
10730 
10731 void Assembler::movswq(Register dst, Register src) {
10732   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10733   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
10734 }
10735 
10736 void Assembler::movzbq(Register dst, Address src) {
10737   InstructionMark im(this);
10738   emit_int24(get_prefixq(src, dst),
10739              0x0F,
10740              (unsigned char)0xB6);
10741   emit_operand(dst, src);
10742 }
10743 
10744 void Assembler::movzbq(Register dst, Register src) {
10745   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10746   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
10747 }
10748 
10749 void Assembler::movzwq(Register dst, Address src) {
10750   InstructionMark im(this);
10751   emit_int24(get_prefixq(src, dst),
10752              0x0F,
10753              (unsigned char)0xB7);
10754   emit_operand(dst, src);
10755 }
10756 
10757 void Assembler::movzwq(Register dst, Register src) {
10758   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10759   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
10760 }
10761 
10762 void Assembler::mulq(Address src) {
10763   InstructionMark im(this);
10764   emit_int16(get_prefixq(src), (unsigned char)0xF7);
10765   emit_operand(rsp, src);
10766 }
10767 
10768 void Assembler::mulq(Register src) {
10769   int encode = prefixq_and_encode(src->encoding());
10770   emit_int16((unsigned char)0xF7, (0xE0 | encode));
10771 }
10772 
10773 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
10774   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
10775   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10776   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
10777   emit_int16((unsigned char)0xF6, (0xC0 | encode));
10778 }
10779 
10780 void Assembler::negq(Register dst) {
10781   int encode = prefixq_and_encode(dst->encoding());
10782   emit_int16((unsigned char)0xF7, (0xD8 | encode));
10783 }
10784 
10785 void Assembler::negq(Address dst) {
10786   InstructionMark im(this);
10787   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
10788   emit_operand(as_Register(3), dst);
10789 }
10790 
10791 void Assembler::notq(Register dst) {
10792   int encode = prefixq_and_encode(dst->encoding());
10793   emit_int16((unsigned char)0xF7, (0xD0 | encode));
10794 }
10795 
10796 void Assembler::btsq(Address dst, int imm8) {
10797   assert(isByte(imm8), "not a byte");
10798   InstructionMark im(this);
10799   emit_int24(get_prefixq(dst),
10800              0x0F,
10801              (unsigned char)0xBA);
10802   emit_operand(rbp /* 5 */, dst, 1);
10803   emit_int8(imm8);
10804 }
10805 
10806 void Assembler::btrq(Address dst, int imm8) {
10807   assert(isByte(imm8), "not a byte");
10808   InstructionMark im(this);
10809   emit_int24(get_prefixq(dst),
10810              0x0F,
10811              (unsigned char)0xBA);
10812   emit_operand(rsi /* 6 */, dst, 1);
10813   emit_int8(imm8);
10814 }
10815 
10816 void Assembler::orq(Address dst, int32_t imm32) {
10817   InstructionMark im(this);
10818   prefixq(dst);
10819   emit_arith_operand(0x81, as_Register(1), dst, imm32);
10820 }
10821 
10822 void Assembler::orq(Address dst, Register src) {
10823   InstructionMark im(this);
10824   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
10825   emit_operand(src, dst);
10826 }
10827 
10828 void Assembler::orq(Register dst, int32_t imm32) {
10829   (void) prefixq_and_encode(dst->encoding());
10830   emit_arith(0x81, 0xC8, dst, imm32);
10831 }
10832 
10833 void Assembler::orq(Register dst, Address src) {
10834   InstructionMark im(this);
10835   emit_int16(get_prefixq(src, dst), 0x0B);
10836   emit_operand(dst, src);
10837 }
10838 
10839 void Assembler::orq(Register dst, Register src) {
10840   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10841   emit_arith(0x0B, 0xC0, dst, src);
10842 }
10843 
10844 void Assembler::popcntq(Register dst, Address src) {
10845   assert(VM_Version::supports_popcnt(), "must support");
10846   InstructionMark im(this);
10847   emit_int32((unsigned char)0xF3,
10848              get_prefixq(src, dst),
10849              0x0F,
10850              (unsigned char)0xB8);
10851   emit_operand(dst, src);
10852 }
10853 
10854 void Assembler::popcntq(Register dst, Register src) {
10855   assert(VM_Version::supports_popcnt(), "must support");
10856   emit_int8((unsigned char)0xF3);
10857   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10858   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
10859 }
10860 
10861 void Assembler::popq(Address dst) {
10862   InstructionMark im(this);
10863   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
10864   emit_operand(rax, dst);
10865 }
10866 
10867 void Assembler::popq(Register dst) {
10868   emit_int8((unsigned char)0x58 | dst->encoding());
10869 }
10870 
10871 // Precomputable: popa, pusha, vzeroupper
10872 
10873 // The result of these routines are invariant from one invocation to another
10874 // invocation for the duration of a run. Caching the result on bootstrap
10875 // and copying it out on subsequent invocations can thus be beneficial
10876 static bool     precomputed = false;
10877 
10878 static u_char* popa_code  = NULL;
10879 static int     popa_len   = 0;
10880 
10881 static u_char* pusha_code = NULL;
10882 static int     pusha_len  = 0;
10883 
10884 static u_char* vzup_code  = NULL;
10885 static int     vzup_len   = 0;
10886 
10887 void Assembler::precompute_instructions() {
10888   assert(!Universe::is_fully_initialized(), "must still be single threaded");
10889   guarantee(!precomputed, "only once");
10890   precomputed = true;
10891   ResourceMark rm;
10892 
10893   // Make a temporary buffer big enough for the routines we're capturing
10894   int size = 256;
10895   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
10896   CodeBuffer buffer((address)tmp_code, size);
10897   MacroAssembler masm(&buffer);
10898 
10899   address begin_popa  = masm.code_section()->end();
10900   masm.popa_uncached();
10901   address end_popa    = masm.code_section()->end();
10902   masm.pusha_uncached();
10903   address end_pusha   = masm.code_section()->end();
10904   masm.vzeroupper_uncached();
10905   address end_vzup    = masm.code_section()->end();
10906 
10907   // Save the instructions to permanent buffers.
10908   popa_len = (int)(end_popa - begin_popa);
10909   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
10910   memcpy(popa_code, begin_popa, popa_len);
10911 
10912   pusha_len = (int)(end_pusha - end_popa);
10913   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
10914   memcpy(pusha_code, end_popa, pusha_len);
10915 
10916   vzup_len = (int)(end_vzup - end_pusha);
10917   if (vzup_len > 0) {
10918     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
10919     memcpy(vzup_code, end_pusha, vzup_len);
10920   } else {
10921     vzup_code = pusha_code; // dummy
10922   }
10923 
10924   assert(masm.code()->total_oop_size() == 0 &&
10925          masm.code()->total_metadata_size() == 0 &&
10926          masm.code()->total_relocation_size() == 0,
10927          "pre-computed code can't reference oops, metadata or contain relocations");
10928 }
10929 
10930 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
10931   assert(src != NULL, "code to copy must have been pre-computed");
10932   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
10933   address end = code_section->end();
10934   memcpy(end, src, src_len);
10935   code_section->set_end(end + src_len);
10936 }
10937 
10938 void Assembler::popa() { // 64bit
10939   emit_copy(code_section(), popa_code, popa_len);
10940 }
10941 
10942 void Assembler::popa_uncached() { // 64bit
10943   movq(r15, Address(rsp, 0));
10944   movq(r14, Address(rsp, wordSize));
10945   movq(r13, Address(rsp, 2 * wordSize));
10946   movq(r12, Address(rsp, 3 * wordSize));
10947   movq(r11, Address(rsp, 4 * wordSize));
10948   movq(r10, Address(rsp, 5 * wordSize));
10949   movq(r9,  Address(rsp, 6 * wordSize));
10950   movq(r8,  Address(rsp, 7 * wordSize));
10951   movq(rdi, Address(rsp, 8 * wordSize));
10952   movq(rsi, Address(rsp, 9 * wordSize));
10953   movq(rbp, Address(rsp, 10 * wordSize));
10954   // Skip rsp as it is restored automatically to the value
10955   // before the corresponding pusha when popa is done.
10956   movq(rbx, Address(rsp, 12 * wordSize));
10957   movq(rdx, Address(rsp, 13 * wordSize));
10958   movq(rcx, Address(rsp, 14 * wordSize));
10959   movq(rax, Address(rsp, 15 * wordSize));
10960 
10961   addq(rsp, 16 * wordSize);
10962 }
10963 
10964 // Does not actually store the value of rsp on the stack.
10965 // The slot for rsp just contains an arbitrary value.
10966 void Assembler::pusha() { // 64bit
10967   emit_copy(code_section(), pusha_code, pusha_len);
10968 }
10969 
10970 // Does not actually store the value of rsp on the stack.
10971 // The slot for rsp just contains an arbitrary value.
10972 void Assembler::pusha_uncached() { // 64bit
10973   subq(rsp, 16 * wordSize);
10974 
10975   movq(Address(rsp, 15 * wordSize), rax);
10976   movq(Address(rsp, 14 * wordSize), rcx);
10977   movq(Address(rsp, 13 * wordSize), rdx);
10978   movq(Address(rsp, 12 * wordSize), rbx);
10979   // Skip rsp as the value is normally not used. There are a few places where
10980   // the original value of rsp needs to be known but that can be computed
10981   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
10982   movq(Address(rsp, 10 * wordSize), rbp);
10983   movq(Address(rsp, 9 * wordSize), rsi);
10984   movq(Address(rsp, 8 * wordSize), rdi);
10985   movq(Address(rsp, 7 * wordSize), r8);
10986   movq(Address(rsp, 6 * wordSize), r9);
10987   movq(Address(rsp, 5 * wordSize), r10);
10988   movq(Address(rsp, 4 * wordSize), r11);
10989   movq(Address(rsp, 3 * wordSize), r12);
10990   movq(Address(rsp, 2 * wordSize), r13);
10991   movq(Address(rsp, wordSize), r14);
10992   movq(Address(rsp, 0), r15);
10993 }
10994 
10995 void Assembler::vzeroupper() {
10996   emit_copy(code_section(), vzup_code, vzup_len);
10997 }
10998 
10999 void Assembler::pushq(Address src) {
11000   InstructionMark im(this);
11001   emit_int16(get_prefixq(src), (unsigned char)0xFF);
11002   emit_operand(rsi, src);
11003 }
11004 
11005 void Assembler::rclq(Register dst, int imm8) {
11006   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11007   int encode = prefixq_and_encode(dst->encoding());
11008   if (imm8 == 1) {
11009     emit_int16((unsigned char)0xD1, (0xD0 | encode));
11010   } else {
11011     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
11012   }
11013 }
11014 
11015 void Assembler::rcrq(Register dst, int imm8) {
11016   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11017   int encode = prefixq_and_encode(dst->encoding());
11018   if (imm8 == 1) {
11019     emit_int16((unsigned char)0xD1, (0xD8 | encode));
11020   } else {
11021     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
11022   }
11023 }
11024 
11025 
11026 void Assembler::rorxq(Register dst, Register src, int imm8) {
11027   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11028   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11029   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
11030   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
11031 }
11032 
11033 void Assembler::rorxd(Register dst, Register src, int imm8) {
11034   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11035   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
11037   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
11038 }
11039 
11040 #ifdef _LP64
11041 void Assembler::salq(Address dst, int imm8) {
11042   InstructionMark im(this);
11043   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11044   if (imm8 == 1) {
11045     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
11046     emit_operand(as_Register(4), dst);
11047   }
11048   else {
11049     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
11050     emit_operand(as_Register(4), dst);
11051     emit_int8(imm8);
11052   }
11053 }
11054 
11055 void Assembler::salq(Address dst) {
11056   InstructionMark im(this);
11057   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
11058   emit_operand(as_Register(4), dst);
11059 }
11060 
11061 void Assembler::salq(Register dst, int imm8) {
11062   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11063   int encode = prefixq_and_encode(dst->encoding());
11064   if (imm8 == 1) {
11065     emit_int16((unsigned char)0xD1, (0xE0 | encode));
11066   } else {
11067     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
11068   }
11069 }
11070 
11071 void Assembler::salq(Register dst) {
11072   int encode = prefixq_and_encode(dst->encoding());
11073   emit_int16((unsigned char)0xD3, (0xE0 | encode));
11074 }
11075 
11076 void Assembler::sarq(Address dst, int imm8) {
11077   InstructionMark im(this);
11078   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11079   if (imm8 == 1) {
11080     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
11081     emit_operand(as_Register(7), dst);
11082   }
11083   else {
11084     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
11085     emit_operand(as_Register(7), dst);
11086     emit_int8(imm8);
11087   }
11088 }
11089 
11090 void Assembler::sarq(Address dst) {
11091   InstructionMark im(this);
11092   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
11093   emit_operand(as_Register(7), dst);
11094 }
11095 
11096 void Assembler::sarq(Register dst, int imm8) {
11097   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11098   int encode = prefixq_and_encode(dst->encoding());
11099   if (imm8 == 1) {
11100     emit_int16((unsigned char)0xD1, (0xF8 | encode));
11101   } else {
11102     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
11103   }
11104 }
11105 
11106 void Assembler::sarq(Register dst) {
11107   int encode = prefixq_and_encode(dst->encoding());
11108   emit_int16((unsigned char)0xD3, (0xF8 | encode));
11109 }
11110 #endif
11111 
11112 void Assembler::sbbq(Address dst, int32_t imm32) {
11113   InstructionMark im(this);
11114   prefixq(dst);
11115   emit_arith_operand(0x81, rbx, dst, imm32);
11116 }
11117 
11118 void Assembler::sbbq(Register dst, int32_t imm32) {
11119   (void) prefixq_and_encode(dst->encoding());
11120   emit_arith(0x81, 0xD8, dst, imm32);
11121 }
11122 
11123 void Assembler::sbbq(Register dst, Address src) {
11124   InstructionMark im(this);
11125   emit_int16(get_prefixq(src, dst), 0x1B);
11126   emit_operand(dst, src);
11127 }
11128 
11129 void Assembler::sbbq(Register dst, Register src) {
11130   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11131   emit_arith(0x1B, 0xC0, dst, src);
11132 }
11133 
11134 void Assembler::shlq(Register dst, int imm8) {
11135   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11136   int encode = prefixq_and_encode(dst->encoding());
11137   if (imm8 == 1) {
11138     emit_int16((unsigned char)0xD1, (0xE0 | encode));
11139   } else {
11140     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
11141   }
11142 }
11143 
11144 void Assembler::shlq(Register dst) {
11145   int encode = prefixq_and_encode(dst->encoding());
11146   emit_int16((unsigned char)0xD3, (0xE0 | encode));
11147 }
11148 
11149 void Assembler::shrq(Register dst, int imm8) {
11150   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11151   int encode = prefixq_and_encode(dst->encoding());
11152   if (imm8 == 1) {
11153     emit_int16((unsigned char)0xD1, (0xE8 | encode));
11154   }
11155   else {
11156     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
11157   }
11158 }
11159 
11160 void Assembler::shrq(Register dst) {
11161   int encode = prefixq_and_encode(dst->encoding());
11162   emit_int16((unsigned char)0xD3, 0xE8 | encode);
11163 }
11164 
11165 void Assembler::shrq(Address dst) {
11166   InstructionMark im(this);
11167   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
11168   emit_operand(as_Register(5), dst);
11169 }
11170 
11171 void Assembler::shrq(Address dst, int imm8) {
11172   InstructionMark im(this);
11173   assert(isShiftCount(imm8 >> 1), "illegal shift count");
11174   if (imm8 == 1) {
11175     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
11176     emit_operand(as_Register(5), dst);
11177   }
11178   else {
11179     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
11180     emit_operand(as_Register(5), dst);
11181     emit_int8(imm8);
11182   }
11183 }
11184 
11185 void Assembler::subq(Address dst, int32_t imm32) {
11186   InstructionMark im(this);
11187   prefixq(dst);
11188   emit_arith_operand(0x81, rbp, dst, imm32);
11189 }
11190 
11191 void Assembler::subq(Address dst, Register src) {
11192   InstructionMark im(this);
11193   emit_int16(get_prefixq(dst, src), 0x29);
11194   emit_operand(src, dst);
11195 }
11196 
11197 void Assembler::subq(Register dst, int32_t imm32) {
11198   (void) prefixq_and_encode(dst->encoding());
11199   emit_arith(0x81, 0xE8, dst, imm32);
11200 }
11201 
11202 // Force generation of a 4 byte immediate value even if it fits into 8bit
11203 void Assembler::subq_imm32(Register dst, int32_t imm32) {
11204   (void) prefixq_and_encode(dst->encoding());
11205   emit_arith_imm32(0x81, 0xE8, dst, imm32);
11206 }
11207 
11208 void Assembler::subq(Register dst, Address src) {
11209   InstructionMark im(this);
11210   emit_int16(get_prefixq(src, dst), 0x2B);
11211   emit_operand(dst, src);
11212 }
11213 
11214 void Assembler::subq(Register dst, Register src) {
11215   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11216   emit_arith(0x2B, 0xC0, dst, src);
11217 }
11218 
11219 void Assembler::testq(Address dst, int32_t imm32) {
11220   InstructionMark im(this);
11221   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
11222   emit_operand(as_Register(0), dst);
11223   emit_int32(imm32);
11224 }
11225 
11226 void Assembler::testq(Register dst, int32_t imm32) {
11227   // not using emit_arith because test
11228   // doesn't support sign-extension of
11229   // 8bit operands
11230   int encode = dst->encoding();
11231   encode = prefixq_and_encode(encode);
11232   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11233   emit_int32(imm32);
11234 }
11235 
11236 void Assembler::testq(Register dst, Register src) {
11237   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11238   emit_arith(0x85, 0xC0, dst, src);
11239 }
11240 
11241 void Assembler::testq(Register dst, Address src) {
11242   InstructionMark im(this);
11243   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
11244   emit_operand(dst, src);
11245 }
11246 
11247 void Assembler::xaddq(Address dst, Register src) {
11248   InstructionMark im(this);
11249   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
11250   emit_operand(src, dst);
11251 }
11252 
11253 void Assembler::xchgq(Register dst, Address src) {
11254   InstructionMark im(this);
11255   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
11256   emit_operand(dst, src);
11257 }
11258 
11259 void Assembler::xchgq(Register dst, Register src) {
11260   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11261   emit_int16((unsigned char)0x87, (0xc0 | encode));
11262 }
11263 
11264 void Assembler::xorq(Register dst, Register src) {
11265   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11266   emit_arith(0x33, 0xC0, dst, src);
11267 }
11268 
11269 void Assembler::xorq(Register dst, Address src) {
11270   InstructionMark im(this);
11271   emit_int16(get_prefixq(src, dst), 0x33);
11272   emit_operand(dst, src);
11273 }
11274 
11275 void Assembler::xorq(Register dst, int32_t imm32) {
11276   (void) prefixq_and_encode(dst->encoding());
11277   emit_arith(0x81, 0xF0, dst, imm32);
11278 }
11279 
11280 void Assembler::xorq(Address dst, int32_t imm32) {
11281   InstructionMark im(this);
11282   prefixq(dst);
11283   emit_arith_operand(0x81, as_Register(6), dst, imm32);
11284 }
11285 
11286 void Assembler::xorq(Address dst, Register src) {
11287   InstructionMark im(this);
11288   emit_int16(get_prefixq(dst, src), 0x31);
11289   emit_operand(src, dst);
11290 }
11291 
11292 #endif // !LP64
11293 
11294 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
11295   if (VM_Version::supports_evex()) {
11296     _tuple_type = tuple_type;
11297     _input_size_in_bits = input_size_in_bits;
11298   }
11299 }