1 /*
    2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert((op1 & 0x01) == 1, "should be 32bit operation");
  304   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  305   if (is8bit(imm32)) {
  306     emit_int24(op1 | 0x02,        // set sign bit
  307                op2 | encode(dst),
  308                imm32 & 0xFF);
  309   } else {
  310     emit_int16(op1, (op2 | encode(dst)));
  311     emit_int32(imm32);
  312   }
  313 }
  314 
  315 // Force generation of a 4 byte immediate value even if it fits into 8bit
  316 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  317   assert(isByte(op1) && isByte(op2), "wrong opcode");
  318   assert((op1 & 0x01) == 1, "should be 32bit operation");
  319   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  320   emit_int16(op1, (op2 | encode(dst)));
  321   emit_int32(imm32);
  322 }
  323 
  324 // immediate-to-memory forms
  325 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  326   assert((op1 & 0x01) == 1, "should be 32bit operation");
  327   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  328   if (is8bit(imm32)) {
  329     emit_int8(op1 | 0x02); // set sign bit
  330     emit_operand(rm, adr, 1);
  331     emit_int8(imm32 & 0xFF);
  332   } else {
  333     emit_int8(op1);
  334     emit_operand(rm, adr, 4);
  335     emit_int32(imm32);
  336   }
  337 }
  338 
  339 
  340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  341   assert(isByte(op1) && isByte(op2), "wrong opcode");
  342   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  343 }
  344 
  345 
  346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  347                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  348   int mod_idx = 0;
  349   // We will test if the displacement fits the compressed format and if so
  350   // apply the compression to the displacment iff the result is8bit.
  351   if (VM_Version::supports_evex() && is_evex_inst) {
  352     switch (cur_tuple_type) {
  353     case EVEX_FV:
  354       if ((cur_encoding & VEX_W) == VEX_W) {
  355         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  356       } else {
  357         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  358       }
  359       break;
  360 
  361     case EVEX_HV:
  362       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  363       break;
  364 
  365     case EVEX_FVM:
  366       break;
  367 
  368     case EVEX_T1S:
  369       switch (in_size_in_bits) {
  370       case EVEX_8bit:
  371         break;
  372 
  373       case EVEX_16bit:
  374         mod_idx = 1;
  375         break;
  376 
  377       case EVEX_32bit:
  378         mod_idx = 2;
  379         break;
  380 
  381       case EVEX_64bit:
  382         mod_idx = 3;
  383         break;
  384       }
  385       break;
  386 
  387     case EVEX_T1F:
  388     case EVEX_T2:
  389     case EVEX_T4:
  390       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  391       break;
  392 
  393     case EVEX_T8:
  394       break;
  395 
  396     case EVEX_HVM:
  397       break;
  398 
  399     case EVEX_QVM:
  400       break;
  401 
  402     case EVEX_OVM:
  403       break;
  404 
  405     case EVEX_M128:
  406       break;
  407 
  408     case EVEX_DUP:
  409       break;
  410 
  411     default:
  412       assert(0, "no valid evex tuple_table entry");
  413       break;
  414     }
  415 
  416     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  417       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  418       if ((disp % disp_factor) == 0) {
  419         int new_disp = disp / disp_factor;
  420         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  421           disp = new_disp;
  422         }
  423       } else {
  424         return false;
  425       }
  426     }
  427   }
  428   return (-0x80 <= disp && disp < 0x80);
  429 }
  430 
  431 
  432 bool Assembler::emit_compressed_disp_byte(int &disp) {
  433   int mod_idx = 0;
  434   // We will test if the displacement fits the compressed format and if so
  435   // apply the compression to the displacment iff the result is8bit.
  436   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  437     int evex_encoding = _attributes->get_evex_encoding();
  438     int tuple_type = _attributes->get_tuple_type();
  439     switch (tuple_type) {
  440     case EVEX_FV:
  441       if ((evex_encoding & VEX_W) == VEX_W) {
  442         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  443       } else {
  444         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  445       }
  446       break;
  447 
  448     case EVEX_HV:
  449       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  450       break;
  451 
  452     case EVEX_FVM:
  453       break;
  454 
  455     case EVEX_T1S:
  456       switch (_attributes->get_input_size()) {
  457       case EVEX_8bit:
  458         break;
  459 
  460       case EVEX_16bit:
  461         mod_idx = 1;
  462         break;
  463 
  464       case EVEX_32bit:
  465         mod_idx = 2;
  466         break;
  467 
  468       case EVEX_64bit:
  469         mod_idx = 3;
  470         break;
  471       }
  472       break;
  473 
  474     case EVEX_T1F:
  475     case EVEX_T2:
  476     case EVEX_T4:
  477       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  478       break;
  479 
  480     case EVEX_T8:
  481       break;
  482 
  483     case EVEX_HVM:
  484       break;
  485 
  486     case EVEX_QVM:
  487       break;
  488 
  489     case EVEX_OVM:
  490       break;
  491 
  492     case EVEX_M128:
  493       break;
  494 
  495     case EVEX_DUP:
  496       break;
  497 
  498     default:
  499       assert(0, "no valid evex tuple_table entry");
  500       break;
  501     }
  502 
  503     int vector_len = _attributes->get_vector_len();
  504     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  505       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  506       if ((disp % disp_factor) == 0) {
  507         int new_disp = disp / disp_factor;
  508         if (is8bit(new_disp)) {
  509           disp = new_disp;
  510         }
  511       } else {
  512         return false;
  513       }
  514     }
  515   }
  516   return is8bit(disp);
  517 }
  518 
  519 static bool is_valid_encoding(int reg_enc) {
  520   return reg_enc >= 0;
  521 }
  522 
  523 static int raw_encode(Register reg) {
  524   assert(reg == noreg || reg->is_valid(), "sanity");
  525   int reg_enc = (intptr_t)reg;
  526   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  527   return reg_enc;
  528 }
  529 
  530 static int raw_encode(XMMRegister xmmreg) {
  531   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  532   int xmmreg_enc = (intptr_t)xmmreg;
  533   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  534   return xmmreg_enc;
  535 }
  536 
  537 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  538   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  539 }
  540 
  541 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  542   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  543 }
  544 
  545 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  546   assert((mod & 3) != 0b11, "forbidden");
  547   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  548   emit_int8(modrm);
  549 }
  550 
  551 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  552                                         int disp) {
  553   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  554   emit_int16(modrm, disp & 0xFF);
  555 }
  556 
  557 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  558                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   int sib = sib_encoding(scale, index_enc, base_enc);
  561   emit_int16(modrm, sib);
  562 }
  563 
  564 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  565                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  566                                             int disp) {
  567   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  568   int sib = sib_encoding(scale, index_enc, base_enc);
  569   emit_int24(modrm, sib, disp & 0xFF);
  570 }
  571 
  572 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  573                                     Address::ScaleFactor scale, int disp,
  574                                     RelocationHolder const& rspec,
  575                                     int rip_relative_correction) {
  576   bool no_relocation = (rspec.type() == relocInfo::none);
  577 
  578   if (is_valid_encoding(base_enc)) {
  579     if (is_valid_encoding(index_enc)) {
  580       assert(scale != Address::no_scale, "inconsistent address");
  581       // [base + index*scale + disp]
  582       if (disp == 0 && no_relocation &&
  583           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  584         // [base + index*scale]
  585         // [00 reg 100][ss index base]
  586         emit_modrm_sib(0b00, reg_enc, 0b100,
  587                        scale, index_enc, base_enc);
  588       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  589         // [base + index*scale + imm8]
  590         // [01 reg 100][ss index base] imm8
  591         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  592                              scale, index_enc, base_enc,
  593                              disp);
  594       } else {
  595         // [base + index*scale + disp32]
  596         // [10 reg 100][ss index base] disp32
  597         emit_modrm_sib(0b10, reg_enc, 0b100,
  598                        scale, index_enc, base_enc);
  599         emit_data(disp, rspec, disp32_operand);
  600       }
  601     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  602       // [rsp + disp]
  603       if (disp == 0 && no_relocation) {
  604         // [rsp]
  605         // [00 reg 100][00 100 100]
  606         emit_modrm_sib(0b00, reg_enc, 0b100,
  607                        Address::times_1, 0b100, 0b100);
  608       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  609         // [rsp + imm8]
  610         // [01 reg 100][00 100 100] disp8
  611         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  612                              Address::times_1, 0b100, 0b100,
  613                              disp);
  614       } else {
  615         // [rsp + imm32]
  616         // [10 reg 100][00 100 100] disp32
  617         emit_modrm_sib(0b10, reg_enc, 0b100,
  618                        Address::times_1, 0b100, 0b100);
  619         emit_data(disp, rspec, disp32_operand);
  620       }
  621     } else {
  622       // [base + disp]
  623       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  624       if (disp == 0 && no_relocation &&
  625           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  626         // [base]
  627         // [00 reg base]
  628         emit_modrm(0, reg_enc, base_enc);
  629       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  630         // [base + disp8]
  631         // [01 reg base] disp8
  632         emit_modrm_disp8(0b01, reg_enc, base_enc,
  633                          disp);
  634       } else {
  635         // [base + disp32]
  636         // [10 reg base] disp32
  637         emit_modrm(0b10, reg_enc, base_enc);
  638         emit_data(disp, rspec, disp32_operand);
  639       }
  640     }
  641   } else {
  642     if (is_valid_encoding(index_enc)) {
  643       assert(scale != Address::no_scale, "inconsistent address");
  644       // base == noreg
  645       // [index*scale + disp]
  646       // [00 reg 100][ss index 101] disp32
  647       emit_modrm_sib(0b00, reg_enc, 0b100,
  648                      scale, index_enc, 0b101 /* no base */);
  649       emit_data(disp, rspec, disp32_operand);
  650     } else if (!no_relocation) {
  651       // base == noreg, index == noreg
  652       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  653       // [00 reg 101] disp32
  654 
  655       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  656       // Note that the RIP-rel. correction applies to the generated
  657       // disp field, but _not_ to the target address in the rspec.
  658 
  659       // disp was created by converting the target address minus the pc
  660       // at the start of the instruction. That needs more correction here.
  661       // intptr_t disp = target - next_ip;
  662       assert(inst_mark() != NULL, "must be inside InstructionMark");
  663       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  664       int64_t adjusted = disp;
  665       // Do rip-rel adjustment for 64bit
  666       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  667       assert(is_simm32(adjusted),
  668              "must be 32bit offset (RIP relative address)");
  669       emit_data((int32_t) adjusted, rspec, disp32_operand);
  670 
  671     } else {
  672       // base == noreg, index == noreg, no_relocation == true
  673       // 32bit never did this, did everything as the rip-rel/disp code above
  674       // [disp] ABSOLUTE
  675       // [00 reg 100][00 100 101] disp32
  676       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  677                      Address::times_1, 0b100, 0b101);
  678       emit_data(disp, rspec, disp32_operand);
  679     }
  680   }
  681 }
  682 
  683 void Assembler::emit_operand(Register reg, Register base, Register index,
  684                              Address::ScaleFactor scale, int disp,
  685                              RelocationHolder const& rspec,
  686                              int rip_relative_correction) {
  687   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  688   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  689                       scale, disp, rspec, rip_relative_correction);
  690 
  691 }
  692 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  693                              Address::ScaleFactor scale, int disp,
  694                              RelocationHolder const& rspec) {
  695   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  696   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  697   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  698                       scale, disp, rspec);
  699 }
  700 
  701 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  702                              Address::ScaleFactor scale, int disp,
  703                              RelocationHolder const& rspec) {
  704   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  705   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  706   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  707                       scale, disp, rspec, /* rip_relative_correction */ 0);
  708 }
  709 
  710 // Secret local extension to Assembler::WhichOperand:
  711 #define end_pc_operand (_WhichOperand_limit)
  712 
  713 address Assembler::locate_operand(address inst, WhichOperand which) {
  714   // Decode the given instruction, and return the address of
  715   // an embedded 32-bit operand word.
  716 
  717   // If "which" is disp32_operand, selects the displacement portion
  718   // of an effective address specifier.
  719   // If "which" is imm64_operand, selects the trailing immediate constant.
  720   // If "which" is call32_operand, selects the displacement of a call or jump.
  721   // Caller is responsible for ensuring that there is such an operand,
  722   // and that it is 32/64 bits wide.
  723 
  724   // If "which" is end_pc_operand, find the end of the instruction.
  725 
  726   address ip = inst;
  727   bool is_64bit = false;
  728 
  729   debug_only(bool has_disp32 = false);
  730   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  731 
  732   again_after_prefix:
  733   switch (0xFF & *ip++) {
  734 
  735   // These convenience macros generate groups of "case" labels for the switch.
  736 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  737 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  738              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  739 #define REP16(x) REP8((x)+0): \
  740               case REP8((x)+8)
  741 
  742   case CS_segment:
  743   case SS_segment:
  744   case DS_segment:
  745   case ES_segment:
  746   case FS_segment:
  747   case GS_segment:
  748     // Seems dubious
  749     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  750     assert(ip == inst+1, "only one prefix allowed");
  751     goto again_after_prefix;
  752 
  753   case 0x67:
  754   case REX:
  755   case REX_B:
  756   case REX_X:
  757   case REX_XB:
  758   case REX_R:
  759   case REX_RB:
  760   case REX_RX:
  761   case REX_RXB:
  762     NOT_LP64(assert(false, "64bit prefixes"));
  763     goto again_after_prefix;
  764 
  765   case REX_W:
  766   case REX_WB:
  767   case REX_WX:
  768   case REX_WXB:
  769   case REX_WR:
  770   case REX_WRB:
  771   case REX_WRX:
  772   case REX_WRXB:
  773     NOT_LP64(assert(false, "64bit prefixes"));
  774     is_64bit = true;
  775     goto again_after_prefix;
  776 
  777   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  778   case 0x88: // movb a, r
  779   case 0x89: // movl a, r
  780   case 0x8A: // movb r, a
  781   case 0x8B: // movl r, a
  782   case 0x8F: // popl a
  783     debug_only(has_disp32 = true);
  784     break;
  785 
  786   case 0x68: // pushq #32
  787     if (which == end_pc_operand) {
  788       return ip + 4;
  789     }
  790     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  791     return ip;                  // not produced by emit_operand
  792 
  793   case 0x66: // movw ... (size prefix)
  794     again_after_size_prefix2:
  795     switch (0xFF & *ip++) {
  796     case REX:
  797     case REX_B:
  798     case REX_X:
  799     case REX_XB:
  800     case REX_R:
  801     case REX_RB:
  802     case REX_RX:
  803     case REX_RXB:
  804     case REX_W:
  805     case REX_WB:
  806     case REX_WX:
  807     case REX_WXB:
  808     case REX_WR:
  809     case REX_WRB:
  810     case REX_WRX:
  811     case REX_WRXB:
  812       NOT_LP64(assert(false, "64bit prefix found"));
  813       goto again_after_size_prefix2;
  814     case 0x8B: // movw r, a
  815     case 0x89: // movw a, r
  816       debug_only(has_disp32 = true);
  817       break;
  818     case 0xC7: // movw a, #16
  819       debug_only(has_disp32 = true);
  820       tail_size = 2;  // the imm16
  821       break;
  822     case 0x0F: // several SSE/SSE2 variants
  823       ip--;    // reparse the 0x0F
  824       goto again_after_prefix;
  825     default:
  826       ShouldNotReachHere();
  827     }
  828     break;
  829 
  830   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  831     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  832     // these asserts are somewhat nonsensical
  833 #ifndef _LP64
  834     assert(which == imm_operand || which == disp32_operand,
  835            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  836 #else
  837     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  838            which == narrow_oop_operand && !is_64bit,
  839            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  840 #endif // _LP64
  841     return ip;
  842 
  843   case 0x69: // imul r, a, #32
  844   case 0xC7: // movl a, #32(oop?)
  845     tail_size = 4;
  846     debug_only(has_disp32 = true); // has both kinds of operands!
  847     break;
  848 
  849   case 0x0F: // movx..., etc.
  850     switch (0xFF & *ip++) {
  851     case 0x3A: // pcmpestri
  852       tail_size = 1;
  853     case 0x38: // ptest, pmovzxbw
  854       ip++; // skip opcode
  855       debug_only(has_disp32 = true); // has both kinds of operands!
  856       break;
  857 
  858     case 0x70: // pshufd r, r/a, #8
  859       debug_only(has_disp32 = true); // has both kinds of operands!
  860     case 0x73: // psrldq r, #8
  861       tail_size = 1;
  862       break;
  863 
  864     case 0x12: // movlps
  865     case 0x28: // movaps
  866     case 0x2E: // ucomiss
  867     case 0x2F: // comiss
  868     case 0x54: // andps
  869     case 0x55: // andnps
  870     case 0x56: // orps
  871     case 0x57: // xorps
  872     case 0x58: // addpd
  873     case 0x59: // mulpd
  874     case 0x6E: // movd
  875     case 0x7E: // movd
  876     case 0x6F: // movdq
  877     case 0x7F: // movdq
  878     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  879     case 0xFE: // paddd
  880       debug_only(has_disp32 = true);
  881       break;
  882 
  883     case 0xAD: // shrd r, a, %cl
  884     case 0xAF: // imul r, a
  885     case 0xBE: // movsbl r, a (movsxb)
  886     case 0xBF: // movswl r, a (movsxw)
  887     case 0xB6: // movzbl r, a (movzxb)
  888     case 0xB7: // movzwl r, a (movzxw)
  889     case REP16(0x40): // cmovl cc, r, a
  890     case 0xB0: // cmpxchgb
  891     case 0xB1: // cmpxchg
  892     case 0xC1: // xaddl
  893     case 0xC7: // cmpxchg8
  894     case REP16(0x90): // setcc a
  895       debug_only(has_disp32 = true);
  896       // fall out of the switch to decode the address
  897       break;
  898 
  899     case 0xC4: // pinsrw r, a, #8
  900       debug_only(has_disp32 = true);
  901     case 0xC5: // pextrw r, r, #8
  902       tail_size = 1;  // the imm8
  903       break;
  904 
  905     case 0xAC: // shrd r, a, #8
  906       debug_only(has_disp32 = true);
  907       tail_size = 1;  // the imm8
  908       break;
  909 
  910     case REP16(0x80): // jcc rdisp32
  911       if (which == end_pc_operand)  return ip + 4;
  912       assert(which == call32_operand, "jcc has no disp32 or imm");
  913       return ip;
  914     default:
  915       ShouldNotReachHere();
  916     }
  917     break;
  918 
  919   case 0x81: // addl a, #32; addl r, #32
  920     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  921     // on 32bit in the case of cmpl, the imm might be an oop
  922     tail_size = 4;
  923     debug_only(has_disp32 = true); // has both kinds of operands!
  924     break;
  925 
  926   case 0x83: // addl a, #8; addl r, #8
  927     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  928     debug_only(has_disp32 = true); // has both kinds of operands!
  929     tail_size = 1;
  930     break;
  931 
  932   case 0x9B:
  933     switch (0xFF & *ip++) {
  934     case 0xD9: // fnstcw a
  935       debug_only(has_disp32 = true);
  936       break;
  937     default:
  938       ShouldNotReachHere();
  939     }
  940     break;
  941 
  942   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  943   case REP4(0x10): // adc...
  944   case REP4(0x20): // and...
  945   case REP4(0x30): // xor...
  946   case REP4(0x08): // or...
  947   case REP4(0x18): // sbb...
  948   case REP4(0x28): // sub...
  949   case 0xF7: // mull a
  950   case 0x8D: // lea r, a
  951   case 0x87: // xchg r, a
  952   case REP4(0x38): // cmp...
  953   case 0x85: // test r, a
  954     debug_only(has_disp32 = true); // has both kinds of operands!
  955     break;
  956 
  957   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  958   case 0xC6: // movb a, #8
  959   case 0x80: // cmpb a, #8
  960   case 0x6B: // imul r, a, #8
  961     debug_only(has_disp32 = true); // has both kinds of operands!
  962     tail_size = 1; // the imm8
  963     break;
  964 
  965   case 0xC4: // VEX_3bytes
  966   case 0xC5: // VEX_2bytes
  967     assert((UseAVX > 0), "shouldn't have VEX prefix");
  968     assert(ip == inst+1, "no prefixes allowed");
  969     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  970     // but they have prefix 0x0F and processed when 0x0F processed above.
  971     //
  972     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
  973     // instructions (these instructions are not supported in 64-bit mode).
  974     // To distinguish them bits [7:6] are set in the VEX second byte since
  975     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
  976     // those VEX bits REX and vvvv bits are inverted.
  977     //
  978     // Fortunately C2 doesn't generate these instructions so we don't need
  979     // to check for them in product version.
  980 
  981     // Check second byte
  982     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
  983 
  984     int vex_opcode;
  985     // First byte
  986     if ((0xFF & *inst) == VEX_3bytes) {
  987       vex_opcode = VEX_OPCODE_MASK & *ip;
  988       ip++; // third byte
  989       is_64bit = ((VEX_W & *ip) == VEX_W);
  990     } else {
  991       vex_opcode = VEX_OPCODE_0F;
  992     }
  993     ip++; // opcode
  994     // To find the end of instruction (which == end_pc_operand).
  995     switch (vex_opcode) {
  996       case VEX_OPCODE_0F:
  997         switch (0xFF & *ip) {
  998         case 0x70: // pshufd r, r/a, #8
  999         case 0x71: // ps[rl|ra|ll]w r, #8
 1000         case 0x72: // ps[rl|ra|ll]d r, #8
 1001         case 0x73: // ps[rl|ra|ll]q r, #8
 1002         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1003         case 0xC4: // pinsrw r, r, r/a, #8
 1004         case 0xC5: // pextrw r/a, r, #8
 1005         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1006           tail_size = 1;  // the imm8
 1007           break;
 1008         }
 1009         break;
 1010       case VEX_OPCODE_0F_3A:
 1011         tail_size = 1;
 1012         break;
 1013     }
 1014     ip++; // skip opcode
 1015     debug_only(has_disp32 = true); // has both kinds of operands!
 1016     break;
 1017 
 1018   case 0x62: // EVEX_4bytes
 1019     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1020     assert(ip == inst+1, "no prefixes allowed");
 1021     // no EVEX collisions, all instructions that have 0x62 opcodes
 1022     // have EVEX versions and are subopcodes of 0x66
 1023     ip++; // skip P0 and exmaine W in P1
 1024     is_64bit = ((VEX_W & *ip) == VEX_W);
 1025     ip++; // move to P2
 1026     ip++; // skip P2, move to opcode
 1027     // To find the end of instruction (which == end_pc_operand).
 1028     switch (0xFF & *ip) {
 1029     case 0x22: // pinsrd r, r/a, #8
 1030     case 0x61: // pcmpestri r, r/a, #8
 1031     case 0x70: // pshufd r, r/a, #8
 1032     case 0x73: // psrldq r, #8
 1033     case 0x1f: // evpcmpd/evpcmpq
 1034     case 0x3f: // evpcmpb/evpcmpw
 1035       tail_size = 1;  // the imm8
 1036       break;
 1037     default:
 1038       break;
 1039     }
 1040     ip++; // skip opcode
 1041     debug_only(has_disp32 = true); // has both kinds of operands!
 1042     break;
 1043 
 1044   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1045   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1046   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1047   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1048   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1049   case 0xDF: // fild_d a; fistp_d a
 1050   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1051   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1052   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1053     debug_only(has_disp32 = true);
 1054     break;
 1055 
 1056   case 0xE8: // call rdisp32
 1057   case 0xE9: // jmp  rdisp32
 1058     if (which == end_pc_operand)  return ip + 4;
 1059     assert(which == call32_operand, "call has no disp32 or imm");
 1060     return ip;
 1061 
 1062   case 0xF0:                    // Lock
 1063     goto again_after_prefix;
 1064 
 1065   case 0xF3:                    // For SSE
 1066   case 0xF2:                    // For SSE2
 1067     switch (0xFF & *ip++) {
 1068     case REX:
 1069     case REX_B:
 1070     case REX_X:
 1071     case REX_XB:
 1072     case REX_R:
 1073     case REX_RB:
 1074     case REX_RX:
 1075     case REX_RXB:
 1076     case REX_W:
 1077     case REX_WB:
 1078     case REX_WX:
 1079     case REX_WXB:
 1080     case REX_WR:
 1081     case REX_WRB:
 1082     case REX_WRX:
 1083     case REX_WRXB:
 1084       NOT_LP64(assert(false, "found 64bit prefix"));
 1085       ip++;
 1086     default:
 1087       ip++;
 1088     }
 1089     debug_only(has_disp32 = true); // has both kinds of operands!
 1090     break;
 1091 
 1092   default:
 1093     ShouldNotReachHere();
 1094 
 1095 #undef REP8
 1096 #undef REP16
 1097   }
 1098 
 1099   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1100 #ifdef _LP64
 1101   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1102 #else
 1103   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1104   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1105 #endif // LP64
 1106   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1107 
 1108   // parse the output of emit_operand
 1109   int op2 = 0xFF & *ip++;
 1110   int base = op2 & 0x07;
 1111   int op3 = -1;
 1112   const int b100 = 4;
 1113   const int b101 = 5;
 1114   if (base == b100 && (op2 >> 6) != 3) {
 1115     op3 = 0xFF & *ip++;
 1116     base = op3 & 0x07;   // refetch the base
 1117   }
 1118   // now ip points at the disp (if any)
 1119 
 1120   switch (op2 >> 6) {
 1121   case 0:
 1122     // [00 reg  100][ss index base]
 1123     // [00 reg  100][00   100  esp]
 1124     // [00 reg base]
 1125     // [00 reg  100][ss index  101][disp32]
 1126     // [00 reg  101]               [disp32]
 1127 
 1128     if (base == b101) {
 1129       if (which == disp32_operand)
 1130         return ip;              // caller wants the disp32
 1131       ip += 4;                  // skip the disp32
 1132     }
 1133     break;
 1134 
 1135   case 1:
 1136     // [01 reg  100][ss index base][disp8]
 1137     // [01 reg  100][00   100  esp][disp8]
 1138     // [01 reg base]               [disp8]
 1139     ip += 1;                    // skip the disp8
 1140     break;
 1141 
 1142   case 2:
 1143     // [10 reg  100][ss index base][disp32]
 1144     // [10 reg  100][00   100  esp][disp32]
 1145     // [10 reg base]               [disp32]
 1146     if (which == disp32_operand)
 1147       return ip;                // caller wants the disp32
 1148     ip += 4;                    // skip the disp32
 1149     break;
 1150 
 1151   case 3:
 1152     // [11 reg base]  (not a memory addressing mode)
 1153     break;
 1154   }
 1155 
 1156   if (which == end_pc_operand) {
 1157     return ip + tail_size;
 1158   }
 1159 
 1160 #ifdef _LP64
 1161   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1162 #else
 1163   assert(which == imm_operand, "instruction has only an imm field");
 1164 #endif // LP64
 1165   return ip;
 1166 }
 1167 
 1168 address Assembler::locate_next_instruction(address inst) {
 1169   // Secretly share code with locate_operand:
 1170   return locate_operand(inst, end_pc_operand);
 1171 }
 1172 
 1173 
 1174 #ifdef ASSERT
 1175 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1176   address inst = inst_mark();
 1177   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1178   address opnd;
 1179 
 1180   Relocation* r = rspec.reloc();
 1181   if (r->type() == relocInfo::none) {
 1182     return;
 1183   } else if (r->is_call() || format == call32_operand) {
 1184     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1185     opnd = locate_operand(inst, call32_operand);
 1186   } else if (r->is_data()) {
 1187     assert(format == imm_operand || format == disp32_operand
 1188            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1189     opnd = locate_operand(inst, (WhichOperand)format);
 1190   } else {
 1191     assert(format == imm_operand, "cannot specify a format");
 1192     return;
 1193   }
 1194   assert(opnd == pc(), "must put operand where relocs can find it");
 1195 }
 1196 #endif // ASSERT
 1197 
 1198 void Assembler::emit_operand(Register reg, Address adr,
 1199                              int rip_relative_correction) {
 1200   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1201                adr._rspec,
 1202                rip_relative_correction);
 1203 }
 1204 
 1205 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1206     if (adr.isxmmindex()) {
 1207        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1208     } else {
 1209        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1210        adr._rspec);
 1211     }
 1212 }
 1213 
 1214 // Now the Assembler instructions (identical for 32/64 bits)
 1215 
 1216 void Assembler::adcl(Address dst, int32_t imm32) {
 1217   InstructionMark im(this);
 1218   prefix(dst);
 1219   emit_arith_operand(0x81, rdx, dst, imm32);
 1220 }
 1221 
 1222 void Assembler::adcl(Address dst, Register src) {
 1223   InstructionMark im(this);
 1224   prefix(dst, src);
 1225   emit_int8(0x11);
 1226   emit_operand(src, dst);
 1227 }
 1228 
 1229 void Assembler::adcl(Register dst, int32_t imm32) {
 1230   prefix(dst);
 1231   emit_arith(0x81, 0xD0, dst, imm32);
 1232 }
 1233 
 1234 void Assembler::adcl(Register dst, Address src) {
 1235   InstructionMark im(this);
 1236   prefix(src, dst);
 1237   emit_int8(0x13);
 1238   emit_operand(dst, src);
 1239 }
 1240 
 1241 void Assembler::adcl(Register dst, Register src) {
 1242   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1243   emit_arith(0x13, 0xC0, dst, src);
 1244 }
 1245 
 1246 void Assembler::addl(Address dst, int32_t imm32) {
 1247   InstructionMark im(this);
 1248   prefix(dst);
 1249   emit_arith_operand(0x81, rax, dst, imm32);
 1250 }
 1251 
 1252 void Assembler::addb(Address dst, int imm8) {
 1253   InstructionMark im(this);
 1254   prefix(dst);
 1255   emit_int8((unsigned char)0x80);
 1256   emit_operand(rax, dst, 1);
 1257   emit_int8(imm8);
 1258 }
 1259 
 1260 void Assembler::addw(Register dst, Register src) {
 1261   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1262   emit_arith(0x03, 0xC0, dst, src);
 1263 }
 1264 
 1265 void Assembler::addw(Address dst, int imm16) {
 1266   InstructionMark im(this);
 1267   emit_int8(0x66);
 1268   prefix(dst);
 1269   emit_int8((unsigned char)0x81);
 1270   emit_operand(rax, dst, 2);
 1271   emit_int16(imm16);
 1272 }
 1273 
 1274 void Assembler::addl(Address dst, Register src) {
 1275   InstructionMark im(this);
 1276   prefix(dst, src);
 1277   emit_int8(0x01);
 1278   emit_operand(src, dst);
 1279 }
 1280 
 1281 void Assembler::addl(Register dst, int32_t imm32) {
 1282   prefix(dst);
 1283   emit_arith(0x81, 0xC0, dst, imm32);
 1284 }
 1285 
 1286 void Assembler::addl(Register dst, Address src) {
 1287   InstructionMark im(this);
 1288   prefix(src, dst);
 1289   emit_int8(0x03);
 1290   emit_operand(dst, src);
 1291 }
 1292 
 1293 void Assembler::addl(Register dst, Register src) {
 1294   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1295   emit_arith(0x03, 0xC0, dst, src);
 1296 }
 1297 
 1298 void Assembler::addr_nop_4() {
 1299   assert(UseAddressNop, "no CPU support");
 1300   // 4 bytes: NOP DWORD PTR [EAX+0]
 1301   emit_int32(0x0F,
 1302              0x1F,
 1303              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1304              0);   // 8-bits offset (1 byte)
 1305 }
 1306 
 1307 void Assembler::addr_nop_5() {
 1308   assert(UseAddressNop, "no CPU support");
 1309   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1310   emit_int32(0x0F,
 1311              0x1F,
 1312              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1313              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1314   emit_int8(0);     // 8-bits offset (1 byte)
 1315 }
 1316 
 1317 void Assembler::addr_nop_7() {
 1318   assert(UseAddressNop, "no CPU support");
 1319   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1320   emit_int24(0x0F,
 1321              0x1F,
 1322              (unsigned char)0x80);
 1323                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1324   emit_int32(0);   // 32-bits offset (4 bytes)
 1325 }
 1326 
 1327 void Assembler::addr_nop_8() {
 1328   assert(UseAddressNop, "no CPU support");
 1329   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1330   emit_int32(0x0F,
 1331              0x1F,
 1332              (unsigned char)0x84,
 1333                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1334              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1335   emit_int32(0);    // 32-bits offset (4 bytes)
 1336 }
 1337 
 1338 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1341   attributes.set_rex_vex_w_reverted();
 1342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1343   emit_int16(0x58, (0xC0 | encode));
 1344 }
 1345 
 1346 void Assembler::addsd(XMMRegister dst, Address src) {
 1347   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1348   InstructionMark im(this);
 1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1350   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1351   attributes.set_rex_vex_w_reverted();
 1352   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1353   emit_int8(0x58);
 1354   emit_operand(dst, src);
 1355 }
 1356 
 1357 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1358   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1360   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1361   emit_int16(0x58, (0xC0 | encode));
 1362 }
 1363 
 1364 void Assembler::addss(XMMRegister dst, Address src) {
 1365   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1366   InstructionMark im(this);
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1369   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1370   emit_int8(0x58);
 1371   emit_operand(dst, src);
 1372 }
 1373 
 1374 void Assembler::aesdec(XMMRegister dst, Address src) {
 1375   assert(VM_Version::supports_aes(), "");
 1376   InstructionMark im(this);
 1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1378   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1379   emit_int8((unsigned char)0xDE);
 1380   emit_operand(dst, src);
 1381 }
 1382 
 1383 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1384   assert(VM_Version::supports_aes(), "");
 1385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1386   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1387   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1388 }
 1389 
 1390 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1391   assert(VM_Version::supports_avx512_vaes(), "");
 1392   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1393   attributes.set_is_evex_instruction();
 1394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1395   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1396 }
 1397 
 1398 
 1399 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1400   assert(VM_Version::supports_aes(), "");
 1401   InstructionMark im(this);
 1402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1403   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1404   emit_int8((unsigned char)0xDF);
 1405   emit_operand(dst, src);
 1406 }
 1407 
 1408 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1409   assert(VM_Version::supports_aes(), "");
 1410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1411   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1412   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1413 }
 1414 
 1415 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1416   assert(VM_Version::supports_avx512_vaes(), "");
 1417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1418   attributes.set_is_evex_instruction();
 1419   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1420   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1421 }
 1422 
 1423 void Assembler::aesenc(XMMRegister dst, Address src) {
 1424   assert(VM_Version::supports_aes(), "");
 1425   InstructionMark im(this);
 1426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1427   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1428   emit_int8((unsigned char)0xDC);
 1429   emit_operand(dst, src);
 1430 }
 1431 
 1432 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1433   assert(VM_Version::supports_aes(), "");
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1437 }
 1438 
 1439 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1440   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1442   attributes.set_is_evex_instruction();
 1443   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1448   assert(VM_Version::supports_aes(), "");
 1449   InstructionMark im(this);
 1450   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1451   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int8((unsigned char)0xDD);
 1453   emit_operand(dst, src);
 1454 }
 1455 
 1456 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1460   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1461 }
 1462 
 1463 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1464   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1465   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1466   attributes.set_is_evex_instruction();
 1467   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1468   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1469 }
 1470 
 1471 void Assembler::andb(Address dst, Register src) {
 1472   InstructionMark im(this);
 1473   prefix(dst, src, true);
 1474   emit_int8(0x20);
 1475   emit_operand(src, dst);
 1476 }
 1477 
 1478 void Assembler::andw(Register dst, Register src) {
 1479   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1480   emit_arith(0x23, 0xC0, dst, src);
 1481 }
 1482 
 1483 void Assembler::andl(Address dst, int32_t imm32) {
 1484   InstructionMark im(this);
 1485   prefix(dst);
 1486   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1487 }
 1488 
 1489 void Assembler::andl(Register dst, int32_t imm32) {
 1490   prefix(dst);
 1491   emit_arith(0x81, 0xE0, dst, imm32);
 1492 }
 1493 
 1494 void Assembler::andl(Address dst, Register src) {
 1495   InstructionMark im(this);
 1496   prefix(dst, src);
 1497   emit_int8(0x21);
 1498   emit_operand(src, dst);
 1499 }
 1500 
 1501 void Assembler::andl(Register dst, Address src) {
 1502   InstructionMark im(this);
 1503   prefix(src, dst);
 1504   emit_int8(0x23);
 1505   emit_operand(dst, src);
 1506 }
 1507 
 1508 void Assembler::andl(Register dst, Register src) {
 1509   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1510   emit_arith(0x23, 0xC0, dst, src);
 1511 }
 1512 
 1513 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1514   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1521   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1522   InstructionMark im(this);
 1523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1524   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int8((unsigned char)0xF2);
 1526   emit_operand(dst, src2);
 1527 }
 1528 
 1529 void Assembler::bsfl(Register dst, Register src) {
 1530   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1531   emit_int24(0x0F,
 1532              (unsigned char)0xBC,
 1533              0xC0 | encode);
 1534 }
 1535 
 1536 void Assembler::bsrl(Register dst, Register src) {
 1537   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1538   emit_int24(0x0F,
 1539              (unsigned char)0xBD,
 1540              0xC0 | encode);
 1541 }
 1542 
 1543 void Assembler::bswapl(Register reg) { // bswap
 1544   int encode = prefix_and_encode(reg->encoding());
 1545   emit_int16(0x0F, (0xC8 | encode));
 1546 }
 1547 
 1548 void Assembler::blsil(Register dst, Register src) {
 1549   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1550   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1553 }
 1554 
 1555 void Assembler::blsil(Register dst, Address src) {
 1556   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1557   InstructionMark im(this);
 1558   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1559   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1560   emit_int8((unsigned char)0xF3);
 1561   emit_operand(rbx, src);
 1562 }
 1563 
 1564 void Assembler::blsmskl(Register dst, Register src) {
 1565   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1566   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1567   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1568   emit_int16((unsigned char)0xF3,
 1569              0xC0 | encode);
 1570 }
 1571 
 1572 void Assembler::blsmskl(Register dst, Address src) {
 1573   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1574   InstructionMark im(this);
 1575   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1576   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1577   emit_int8((unsigned char)0xF3);
 1578   emit_operand(rdx, src);
 1579 }
 1580 
 1581 void Assembler::blsrl(Register dst, Register src) {
 1582   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1583   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1584   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1585   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1586 }
 1587 
 1588 void Assembler::blsrl(Register dst, Address src) {
 1589   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1590   InstructionMark im(this);
 1591   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1592   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1593   emit_int8((unsigned char)0xF3);
 1594   emit_operand(rcx, src);
 1595 }
 1596 
 1597 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1598   // suspect disp32 is always good
 1599   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1600 
 1601   if (L.is_bound()) {
 1602     const int long_size = 5;
 1603     int offs = (int)( target(L) - pc() );
 1604     assert(offs <= 0, "assembler error");
 1605     InstructionMark im(this);
 1606     // 1110 1000 #32-bit disp
 1607     emit_int8((unsigned char)0xE8);
 1608     emit_data(offs - long_size, rtype, operand);
 1609   } else {
 1610     InstructionMark im(this);
 1611     // 1110 1000 #32-bit disp
 1612     L.add_patch_at(code(), locator());
 1613 
 1614     emit_int8((unsigned char)0xE8);
 1615     emit_data(int(0), rtype, operand);
 1616   }
 1617 }
 1618 
 1619 void Assembler::call(Register dst) {
 1620   int encode = prefix_and_encode(dst->encoding());
 1621   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1622 }
 1623 
 1624 
 1625 void Assembler::call(Address adr) {
 1626   InstructionMark im(this);
 1627   prefix(adr);
 1628   emit_int8((unsigned char)0xFF);
 1629   emit_operand(rdx, adr);
 1630 }
 1631 
 1632 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1633   InstructionMark im(this);
 1634   emit_int8((unsigned char)0xE8);
 1635   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1636   // Entry is NULL in case of a scratch emit.
 1637   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1638   // Technically, should use call32_operand, but this format is
 1639   // implied by the fact that we're emitting a call instruction.
 1640 
 1641   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1642   emit_data((int) disp, rspec, operand);
 1643 }
 1644 
 1645 void Assembler::cdql() {
 1646   emit_int8((unsigned char)0x99);
 1647 }
 1648 
 1649 void Assembler::cld() {
 1650   emit_int8((unsigned char)0xFC);
 1651 }
 1652 
 1653 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1654   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1655   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1656   emit_int24(0x0F,
 1657              0x40 | cc,
 1658              0xC0 | encode);
 1659 }
 1660 
 1661 
 1662 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1663   InstructionMark im(this);
 1664   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1665   prefix(src, dst);
 1666   emit_int16(0x0F, (0x40 | cc));
 1667   emit_operand(dst, src);
 1668 }
 1669 
 1670 void Assembler::cmpb(Address dst, int imm8) {
 1671   InstructionMark im(this);
 1672   prefix(dst);
 1673   emit_int8((unsigned char)0x80);
 1674   emit_operand(rdi, dst, 1);
 1675   emit_int8(imm8);
 1676 }
 1677 
 1678 void Assembler::cmpl(Address dst, int32_t imm32) {
 1679   InstructionMark im(this);
 1680   prefix(dst);
 1681   emit_int8((unsigned char)0x81);
 1682   emit_operand(rdi, dst, 4);
 1683   emit_int32(imm32);
 1684 }
 1685 
 1686 void Assembler::cmp(Register dst, int32_t imm32) {
 1687   prefix(dst);
 1688   emit_int8((unsigned char)0x3D);
 1689   emit_int32(imm32);
 1690 }
 1691 
 1692 void Assembler::cmpl(Register dst, int32_t imm32) {
 1693   prefix(dst);
 1694   emit_arith(0x81, 0xF8, dst, imm32);
 1695 }
 1696 
 1697 void Assembler::cmpl(Register dst, Register src) {
 1698   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1699   emit_arith(0x3B, 0xC0, dst, src);
 1700 }
 1701 
 1702 void Assembler::cmpl(Register dst, Address  src) {
 1703   InstructionMark im(this);
 1704   prefix(src, dst);
 1705   emit_int8(0x3B);
 1706   emit_operand(dst, src);
 1707 }
 1708 
 1709 void Assembler::cmpw(Address dst, int imm16) {
 1710   InstructionMark im(this);
 1711   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1712   emit_int16(0x66, (unsigned char)0x81);
 1713   emit_operand(rdi, dst, 2);
 1714   emit_int16(imm16);
 1715 }
 1716 
 1717 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1718 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1719 // The ZF is set if the compared values were equal, and cleared otherwise.
 1720 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1721   InstructionMark im(this);
 1722   prefix(adr, reg);
 1723   emit_int16(0x0F, (unsigned char)0xB1);
 1724   emit_operand(reg, adr);
 1725 }
 1726 
 1727 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1728   InstructionMark im(this);
 1729   size_prefix();
 1730   prefix(adr, reg);
 1731   emit_int16(0x0F, (unsigned char)0xB1);
 1732   emit_operand(reg, adr);
 1733 }
 1734 
 1735 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1736 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1737 // The ZF is set if the compared values were equal, and cleared otherwise.
 1738 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1739   InstructionMark im(this);
 1740   prefix(adr, reg, true);
 1741   emit_int16(0x0F, (unsigned char)0xB0);
 1742   emit_operand(reg, adr);
 1743 }
 1744 
 1745 void Assembler::comisd(XMMRegister dst, Address src) {
 1746   // NOTE: dbx seems to decode this as comiss even though the
 1747   // 0x66 is there. Strangly ucomisd comes out correct
 1748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1749   InstructionMark im(this);
 1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1751   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1752   attributes.set_rex_vex_w_reverted();
 1753   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1754   emit_int8(0x2F);
 1755   emit_operand(dst, src);
 1756 }
 1757 
 1758 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1760   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1761   attributes.set_rex_vex_w_reverted();
 1762   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1763   emit_int16(0x2F, (0xC0 | encode));
 1764 }
 1765 
 1766 void Assembler::comiss(XMMRegister dst, Address src) {
 1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1768   InstructionMark im(this);
 1769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1770   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1771   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1772   emit_int8(0x2F);
 1773   emit_operand(dst, src);
 1774 }
 1775 
 1776 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1777   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1779   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1780   emit_int16(0x2F, (0xC0 | encode));
 1781 }
 1782 
 1783 void Assembler::cpuid() {
 1784   emit_int16(0x0F, (unsigned char)0xA2);
 1785 }
 1786 
 1787 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1788 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1789 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1790 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1791 //
 1792 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1793 //
 1794 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1795 //
 1796 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1797 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1798   assert(VM_Version::supports_sse4_2(), "");
 1799   int8_t w = 0x01;
 1800   Prefix p = Prefix_EMPTY;
 1801 
 1802   emit_int8((unsigned char)0xF2);
 1803   switch (sizeInBytes) {
 1804   case 1:
 1805     w = 0;
 1806     break;
 1807   case 2:
 1808   case 4:
 1809     break;
 1810   LP64_ONLY(case 8:)
 1811     // This instruction is not valid in 32 bits
 1812     // Note:
 1813     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1814     //
 1815     // Page B - 72   Vol. 2C says
 1816     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1817     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1818     //                                                                            F0!!!
 1819     // while 3 - 208 Vol. 2A
 1820     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1821     //
 1822     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1823     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1824     p = REX_W;
 1825     break;
 1826   default:
 1827     assert(0, "Unsupported value for a sizeInBytes argument");
 1828     break;
 1829   }
 1830   LP64_ONLY(prefix(crc, v, p);)
 1831   emit_int32(0x0F,
 1832              0x38,
 1833              0xF0 | w,
 1834              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1835 }
 1836 
 1837 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1838   assert(VM_Version::supports_sse4_2(), "");
 1839   InstructionMark im(this);
 1840   int8_t w = 0x01;
 1841   Prefix p = Prefix_EMPTY;
 1842 
 1843   emit_int8((int8_t)0xF2);
 1844   switch (sizeInBytes) {
 1845   case 1:
 1846     w = 0;
 1847     break;
 1848   case 2:
 1849   case 4:
 1850     break;
 1851   LP64_ONLY(case 8:)
 1852     // This instruction is not valid in 32 bits
 1853     p = REX_W;
 1854     break;
 1855   default:
 1856     assert(0, "Unsupported value for a sizeInBytes argument");
 1857     break;
 1858   }
 1859   LP64_ONLY(prefix(crc, adr, p);)
 1860   emit_int24(0x0F, 0x38, (0xF0 | w));
 1861   emit_operand(crc, adr);
 1862 }
 1863 
 1864 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1868   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1869 }
 1870 
 1871 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1872   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1875   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1876 }
 1877 
 1878 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1882   emit_int16(0x5B, (0xC0 | encode));
 1883 }
 1884 
 1885 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1886   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1887   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1889   emit_int16(0x5B, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1893   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1894   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1895   attributes.set_rex_vex_w_reverted();
 1896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1897   emit_int16(0x5A, (0xC0 | encode));
 1898 }
 1899 
 1900 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1901   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1902   InstructionMark im(this);
 1903   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1904   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1905   attributes.set_rex_vex_w_reverted();
 1906   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1907   emit_int8(0x5A);
 1908   emit_operand(dst, src);
 1909 }
 1910 
 1911 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1914   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1915   emit_int16(0x2A, (0xC0 | encode));
 1916 }
 1917 
 1918 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1919   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1920   InstructionMark im(this);
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1922   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1923   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1924   emit_int8(0x2A);
 1925   emit_operand(dst, src);
 1926 }
 1927 
 1928 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1929   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1931   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1932   emit_int16(0x2A, (0xC0 | encode));
 1933 }
 1934 
 1935 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1937   InstructionMark im(this);
 1938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1939   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1940   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1941   emit_int8(0x2A);
 1942   emit_operand(dst, src);
 1943 }
 1944 
 1945 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1946   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1947   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1948   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1949   emit_int16(0x2A, (0xC0 | encode));
 1950 }
 1951 
 1952 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1953   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1955   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1956   emit_int16(0x5A, (0xC0 | encode));
 1957 }
 1958 
 1959 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1961   InstructionMark im(this);
 1962   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1963   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1964   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1965   emit_int8(0x5A);
 1966   emit_operand(dst, src);
 1967 }
 1968 
 1969 
 1970 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1971   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1973   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1974   emit_int16(0x2C, (0xC0 | encode));
 1975 }
 1976 
 1977 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 1978   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1980   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1981   emit_int16(0x2C, (0xC0 | encode));
 1982 }
 1983 
 1984 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 1985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1986   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 1987   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1988   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1989   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1990 }
 1991 
 1992 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 1993   assert(VM_Version::supports_ssse3(), "");
 1994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 1995   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1996   emit_int16(0x1C, (0xC0 | encode));
 1997 }
 1998 
 1999 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2000   assert(VM_Version::supports_ssse3(), "");
 2001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2002   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2003   emit_int16(0x1D, (0xC0 | encode));
 2004 }
 2005 
 2006 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2007   assert(VM_Version::supports_ssse3(), "");
 2008   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2009   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2010   emit_int16(0x1E, (0xC0 | encode));
 2011 }
 2012 
 2013 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2014   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2015          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2016          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2017   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2018   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2019   emit_int16(0x1C, (0xC0 | encode));
 2020 }
 2021 
 2022 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2023   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2024          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2025          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2026   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2027   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2028   emit_int16(0x1D, (0xC0 | encode));
 2029 }
 2030 
 2031 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2032   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2033   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2034   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2035   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2037   emit_int16(0x1E, (0xC0 | encode));
 2038 }
 2039 
 2040 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2041   assert(UseAVX > 2, "");
 2042   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2043   attributes.set_is_evex_instruction();
 2044   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2045   emit_int16(0x1F, (0xC0 | encode));
 2046 }
 2047 
 2048 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2049   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2050   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2052   emit_int16(0x5A, (0xC0 | encode));
 2053 }
 2054 
 2055 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2056   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2057   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2058   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2059   attributes.set_rex_vex_w_reverted();
 2060   emit_int16(0x5A, (0xC0 | encode));
 2061 }
 2062 
 2063 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2064   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2065   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2066   attributes.set_is_evex_instruction();
 2067   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2068   emit_int16(0x5B, (0xC0 | encode));
 2069 }
 2070 
 2071 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2072   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2073   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2074   attributes.set_is_evex_instruction();
 2075   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2076   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2077 }
 2078 
 2079 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2080   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2082   attributes.set_is_evex_instruction();
 2083   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2084   emit_int16(0x30, (0xC0 | encode));
 2085 }
 2086 
 2087 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2088   assert(UseAVX > 2, "");
 2089   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2090   attributes.set_is_evex_instruction();
 2091   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2092   emit_int16(0x33, (0xC0 | encode));
 2093 }
 2094 
 2095 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2096   assert(UseAVX > 2, "");
 2097   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2098   attributes.set_is_evex_instruction();
 2099   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x31, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(UseAVX > 2, "");
 2105   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2106   attributes.set_is_evex_instruction();
 2107   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2108   emit_int16(0x35, (0xC0 | encode));
 2109 }
 2110 
 2111 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2112   assert(UseAVX > 2, "");
 2113   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2114   attributes.set_is_evex_instruction();
 2115   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2116   emit_int16(0x32, (0xC0 | encode));
 2117 }
 2118 
 2119 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2120   assert(UseAVX > 2, "");
 2121   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2122   attributes.set_is_evex_instruction();
 2123   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2124   emit_int16(0x34, (0xC0 | encode));
 2125 }
 2126 
 2127 void Assembler::decl(Address dst) {
 2128   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2129   InstructionMark im(this);
 2130   prefix(dst);
 2131   emit_int8((unsigned char)0xFF);
 2132   emit_operand(rcx, dst);
 2133 }
 2134 
 2135 void Assembler::divsd(XMMRegister dst, Address src) {
 2136   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2137   InstructionMark im(this);
 2138   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2139   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2140   attributes.set_rex_vex_w_reverted();
 2141   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2142   emit_int8(0x5E);
 2143   emit_operand(dst, src);
 2144 }
 2145 
 2146 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2147   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2148   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2149   attributes.set_rex_vex_w_reverted();
 2150   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2151   emit_int16(0x5E, (0xC0 | encode));
 2152 }
 2153 
 2154 void Assembler::divss(XMMRegister dst, Address src) {
 2155   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2156   InstructionMark im(this);
 2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2158   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2159   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2160   emit_int8(0x5E);
 2161   emit_operand(dst, src);
 2162 }
 2163 
 2164 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2168   emit_int16(0x5E, (0xC0 | encode));
 2169 }
 2170 
 2171 void Assembler::hlt() {
 2172   emit_int8((unsigned char)0xF4);
 2173 }
 2174 
 2175 void Assembler::idivl(Register src) {
 2176   int encode = prefix_and_encode(src->encoding());
 2177   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2178 }
 2179 
 2180 void Assembler::divl(Register src) { // Unsigned
 2181   int encode = prefix_and_encode(src->encoding());
 2182   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2183 }
 2184 
 2185 void Assembler::imull(Register src) {
 2186   int encode = prefix_and_encode(src->encoding());
 2187   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2188 }
 2189 
 2190 void Assembler::imull(Register dst, Register src) {
 2191   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2192   emit_int24(0x0F,
 2193              (unsigned char)0xAF,
 2194              (0xC0 | encode));
 2195 }
 2196 
 2197 void Assembler::imull(Register dst, Address src, int32_t value) {
 2198   InstructionMark im(this);
 2199   prefix(src, dst);
 2200   if (is8bit(value)) {
 2201     emit_int8((unsigned char)0x6B);
 2202     emit_operand(dst, src);
 2203     emit_int8(value);
 2204   } else {
 2205     emit_int8((unsigned char)0x69);
 2206     emit_operand(dst, src);
 2207     emit_int32(value);
 2208   }
 2209 }
 2210 
 2211 void Assembler::imull(Register dst, Register src, int value) {
 2212   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2213   if (is8bit(value)) {
 2214     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2215   } else {
 2216     emit_int16(0x69, (0xC0 | encode));
 2217     emit_int32(value);
 2218   }
 2219 }
 2220 
 2221 void Assembler::imull(Register dst, Address src) {
 2222   InstructionMark im(this);
 2223   prefix(src, dst);
 2224   emit_int16(0x0F, (unsigned char)0xAF);
 2225   emit_operand(dst, src);
 2226 }
 2227 
 2228 
 2229 void Assembler::incl(Address dst) {
 2230   // Don't use it directly. Use MacroAssembler::increment() instead.
 2231   InstructionMark im(this);
 2232   prefix(dst);
 2233   emit_int8((unsigned char)0xFF);
 2234   emit_operand(rax, dst);
 2235 }
 2236 
 2237 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2238   InstructionMark im(this);
 2239   assert((0 <= cc) && (cc < 16), "illegal cc");
 2240   if (L.is_bound()) {
 2241     address dst = target(L);
 2242     assert(dst != NULL, "jcc most probably wrong");
 2243 
 2244     const int short_size = 2;
 2245     const int long_size = 6;
 2246     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2247     if (maybe_short && is8bit(offs - short_size)) {
 2248       // 0111 tttn #8-bit disp
 2249       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2250     } else {
 2251       // 0000 1111 1000 tttn #32-bit disp
 2252       assert(is_simm32(offs - long_size),
 2253              "must be 32bit offset (call4)");
 2254       emit_int16(0x0F, (0x80 | cc));
 2255       emit_int32(offs - long_size);
 2256     }
 2257   } else {
 2258     // Note: could eliminate cond. jumps to this jump if condition
 2259     //       is the same however, seems to be rather unlikely case.
 2260     // Note: use jccb() if label to be bound is very close to get
 2261     //       an 8-bit displacement
 2262     L.add_patch_at(code(), locator());
 2263     emit_int16(0x0F, (0x80 | cc));
 2264     emit_int32(0);
 2265   }
 2266 }
 2267 
 2268 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2269   if (L.is_bound()) {
 2270     const int short_size = 2;
 2271     address entry = target(L);
 2272 #ifdef ASSERT
 2273     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2274     intptr_t delta = short_branch_delta();
 2275     if (delta != 0) {
 2276       dist += (dist < 0 ? (-delta) :delta);
 2277     }
 2278     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2279 #endif
 2280     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2281     // 0111 tttn #8-bit disp
 2282     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2283   } else {
 2284     InstructionMark im(this);
 2285     L.add_patch_at(code(), locator(), file, line);
 2286     emit_int16(0x70 | cc, 0);
 2287   }
 2288 }
 2289 
 2290 void Assembler::jmp(Address adr) {
 2291   InstructionMark im(this);
 2292   prefix(adr);
 2293   emit_int8((unsigned char)0xFF);
 2294   emit_operand(rsp, adr);
 2295 }
 2296 
 2297 void Assembler::jmp(Label& L, bool maybe_short) {
 2298   if (L.is_bound()) {
 2299     address entry = target(L);
 2300     assert(entry != NULL, "jmp most probably wrong");
 2301     InstructionMark im(this);
 2302     const int short_size = 2;
 2303     const int long_size = 5;
 2304     intptr_t offs = entry - pc();
 2305     if (maybe_short && is8bit(offs - short_size)) {
 2306       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2307     } else {
 2308       emit_int8((unsigned char)0xE9);
 2309       emit_int32(offs - long_size);
 2310     }
 2311   } else {
 2312     // By default, forward jumps are always 32-bit displacements, since
 2313     // we can't yet know where the label will be bound.  If you're sure that
 2314     // the forward jump will not run beyond 256 bytes, use jmpb to
 2315     // force an 8-bit displacement.
 2316     InstructionMark im(this);
 2317     L.add_patch_at(code(), locator());
 2318     emit_int8((unsigned char)0xE9);
 2319     emit_int32(0);
 2320   }
 2321 }
 2322 
 2323 void Assembler::jmp(Register entry) {
 2324   int encode = prefix_and_encode(entry->encoding());
 2325   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2326 }
 2327 
 2328 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2329   InstructionMark im(this);
 2330   emit_int8((unsigned char)0xE9);
 2331   assert(dest != NULL, "must have a target");
 2332   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2333   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2334   emit_data(disp, rspec.reloc(), call32_operand);
 2335 }
 2336 
 2337 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2338   if (L.is_bound()) {
 2339     const int short_size = 2;
 2340     address entry = target(L);
 2341     assert(entry != NULL, "jmp most probably wrong");
 2342 #ifdef ASSERT
 2343     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2344     intptr_t delta = short_branch_delta();
 2345     if (delta != 0) {
 2346       dist += (dist < 0 ? (-delta) :delta);
 2347     }
 2348     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2349 #endif
 2350     intptr_t offs = entry - pc();
 2351     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2352   } else {
 2353     InstructionMark im(this);
 2354     L.add_patch_at(code(), locator(), file, line);
 2355     emit_int16((unsigned char)0xEB, 0);
 2356   }
 2357 }
 2358 
 2359 void Assembler::ldmxcsr( Address src) {
 2360   if (UseAVX > 0 ) {
 2361     InstructionMark im(this);
 2362     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2363     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2364     emit_int8((unsigned char)0xAE);
 2365     emit_operand(as_Register(2), src);
 2366   } else {
 2367     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2368     InstructionMark im(this);
 2369     prefix(src);
 2370     emit_int16(0x0F, (unsigned char)0xAE);
 2371     emit_operand(as_Register(2), src);
 2372   }
 2373 }
 2374 
 2375 void Assembler::leal(Register dst, Address src) {
 2376   InstructionMark im(this);
 2377 #ifdef _LP64
 2378   emit_int8(0x67); // addr32
 2379   prefix(src, dst);
 2380 #endif // LP64
 2381   emit_int8((unsigned char)0x8D);
 2382   emit_operand(dst, src);
 2383 }
 2384 
 2385 void Assembler::lfence() {
 2386   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2387 }
 2388 
 2389 void Assembler::lock() {
 2390   emit_int8((unsigned char)0xF0);
 2391 }
 2392 
 2393 void Assembler::size_prefix() {
 2394   emit_int8(0x66);
 2395 }
 2396 
 2397 void Assembler::lzcntl(Register dst, Register src) {
 2398   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2399   emit_int8((unsigned char)0xF3);
 2400   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2401   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2402 }
 2403 
 2404 // Emit mfence instruction
 2405 void Assembler::mfence() {
 2406   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2407   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2408 }
 2409 
 2410 // Emit sfence instruction
 2411 void Assembler::sfence() {
 2412   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2413   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2414 }
 2415 
 2416 void Assembler::mov(Register dst, Register src) {
 2417   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2418 }
 2419 
 2420 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2422   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2423   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2424   attributes.set_rex_vex_w_reverted();
 2425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2426   emit_int16(0x28, (0xC0 | encode));
 2427 }
 2428 
 2429 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2430   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2431   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2432   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2433   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2434   emit_int16(0x28, (0xC0 | encode));
 2435 }
 2436 
 2437 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2438   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2440   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2441   emit_int16(0x16, (0xC0 | encode));
 2442 }
 2443 
 2444 void Assembler::movb(Register dst, Address src) {
 2445   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2446   InstructionMark im(this);
 2447   prefix(src, dst, true);
 2448   emit_int8((unsigned char)0x8A);
 2449   emit_operand(dst, src);
 2450 }
 2451 
 2452 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2453   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2454   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2455   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2456   attributes.set_rex_vex_w_reverted();
 2457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2458   emit_int16(0x12, 0xC0 | encode);
 2459 }
 2460 
 2461 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2462   assert(VM_Version::supports_avx512dq(), "");
 2463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2465   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2466 }
 2467 
 2468 void Assembler::kmovbl(KRegister dst, Register src) {
 2469   assert(VM_Version::supports_avx512dq(), "");
 2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2472   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2473 }
 2474 
 2475 void Assembler::kmovbl(Register dst, KRegister src) {
 2476   assert(VM_Version::supports_avx512dq(), "");
 2477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2478   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2479   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2480 }
 2481 
 2482 void Assembler::kmovwl(KRegister dst, Register src) {
 2483   assert(VM_Version::supports_evex(), "");
 2484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2485   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2486   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2487 }
 2488 
 2489 void Assembler::kmovwl(Register dst, KRegister src) {
 2490   assert(VM_Version::supports_evex(), "");
 2491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2492   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2493   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2494 }
 2495 
 2496 void Assembler::kmovwl(KRegister dst, Address src) {
 2497   assert(VM_Version::supports_evex(), "");
 2498   InstructionMark im(this);
 2499   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2500   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2501   emit_int8((unsigned char)0x90);
 2502   emit_operand((Register)dst, src);
 2503 }
 2504 
 2505 void Assembler::kmovwl(Address dst, KRegister src) {
 2506   assert(VM_Version::supports_evex(), "");
 2507   InstructionMark im(this);
 2508   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2509   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2510   emit_int8((unsigned char)0x91);
 2511   emit_operand((Register)src, dst);
 2512 }
 2513 
 2514 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2515   assert(VM_Version::supports_evex(), "");
 2516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2517   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2518   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2519 }
 2520 
 2521 void Assembler::kmovdl(KRegister dst, Register src) {
 2522   assert(VM_Version::supports_avx512bw(), "");
 2523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2524   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2525   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2526 }
 2527 
 2528 void Assembler::kmovdl(Register dst, KRegister src) {
 2529   assert(VM_Version::supports_avx512bw(), "");
 2530   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2531   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2532   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2533 }
 2534 
 2535 void Assembler::kmovql(KRegister dst, KRegister src) {
 2536   assert(VM_Version::supports_avx512bw(), "");
 2537   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2538   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2539   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2540 }
 2541 
 2542 void Assembler::kmovql(KRegister dst, Address src) {
 2543   assert(VM_Version::supports_avx512bw(), "");
 2544   InstructionMark im(this);
 2545   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2546   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2547   emit_int8((unsigned char)0x90);
 2548   emit_operand((Register)dst, src);
 2549 }
 2550 
 2551 void Assembler::kmovql(Address dst, KRegister src) {
 2552   assert(VM_Version::supports_avx512bw(), "");
 2553   InstructionMark im(this);
 2554   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2555   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2556   emit_int8((unsigned char)0x91);
 2557   emit_operand((Register)src, dst);
 2558 }
 2559 
 2560 void Assembler::kmovql(KRegister dst, Register src) {
 2561   assert(VM_Version::supports_avx512bw(), "");
 2562   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2563   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2564   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2565 }
 2566 
 2567 void Assembler::kmovql(Register dst, KRegister src) {
 2568   assert(VM_Version::supports_avx512bw(), "");
 2569   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2570   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2571   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2572 }
 2573 
 2574 void Assembler::knotwl(KRegister dst, KRegister src) {
 2575   assert(VM_Version::supports_evex(), "");
 2576   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2578   emit_int16(0x44, (0xC0 | encode));
 2579 }
 2580 
 2581 void Assembler::knotbl(KRegister dst, KRegister src) {
 2582   assert(VM_Version::supports_evex(), "");
 2583   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2584   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2585   emit_int16(0x44, (0xC0 | encode));
 2586 }
 2587 
 2588 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2589   assert(VM_Version::supports_avx512dq(), "");
 2590   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2591   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2592   emit_int16(0x45, (0xC0 | encode));
 2593 }
 2594 
 2595 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2596   assert(VM_Version::supports_evex(), "");
 2597   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2598   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2599   emit_int16(0x45, (0xC0 | encode));
 2600 }
 2601 
 2602 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2603   assert(VM_Version::supports_avx512bw(), "");
 2604   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2605   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2606   emit_int16(0x45, (0xC0 | encode));
 2607 }
 2608 
 2609 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2610   assert(VM_Version::supports_avx512bw(), "");
 2611   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2612   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2613   emit_int16(0x45, (0xC0 | encode));
 2614 }
 2615 
 2616 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2617   assert(VM_Version::supports_avx512dq(), "");
 2618   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2619   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2620   emit_int16(0x47, (0xC0 | encode));
 2621 }
 2622 
 2623 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2624   assert(VM_Version::supports_evex(), "");
 2625   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2626   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2627   emit_int16(0x47, (0xC0 | encode));
 2628 }
 2629 
 2630 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2631   assert(VM_Version::supports_avx512bw(), "");
 2632   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2633   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2634   emit_int16(0x47, (0xC0 | encode));
 2635 }
 2636 
 2637 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2638   assert(VM_Version::supports_avx512bw(), "");
 2639   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2640   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2641   emit_int16(0x47, (0xC0 | encode));
 2642 }
 2643 
 2644 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2645   assert(VM_Version::supports_avx512dq(), "");
 2646   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2647   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2648   emit_int16(0x41, (0xC0 | encode));
 2649 }
 2650 
 2651 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2652   assert(VM_Version::supports_evex(), "");
 2653   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2654   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2655   emit_int16(0x41, (0xC0 | encode));
 2656 }
 2657 
 2658 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2659   assert(VM_Version::supports_avx512bw(), "");
 2660   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2661   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2662   emit_int16(0x41, (0xC0 | encode));
 2663 }
 2664 
 2665 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2666   assert(VM_Version::supports_avx512bw(), "");
 2667   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2668   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2669   emit_int16(0x41, (0xC0 | encode));
 2670 }
 2671 
 2672 void Assembler::knotdl(KRegister dst, KRegister src) {
 2673   assert(VM_Version::supports_avx512bw(), "");
 2674   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2675   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2676   emit_int16(0x44, (0xC0 | encode));
 2677 }
 2678 
 2679 void Assembler::knotql(KRegister dst, KRegister src) {
 2680   assert(VM_Version::supports_avx512bw(), "");
 2681   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2682   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2683   emit_int16(0x44, (0xC0 | encode));
 2684 }
 2685 
 2686 // This instruction produces ZF or CF flags
 2687 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2688   assert(VM_Version::supports_avx512dq(), "");
 2689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2690   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2691   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2692 }
 2693 
 2694 // This instruction produces ZF or CF flags
 2695 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2696   assert(VM_Version::supports_evex(), "");
 2697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2698   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2699   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2700 }
 2701 
 2702 // This instruction produces ZF or CF flags
 2703 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2704   assert(VM_Version::supports_avx512bw(), "");
 2705   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2706   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2707   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2708 }
 2709 
 2710 // This instruction produces ZF or CF flags
 2711 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2712   assert(VM_Version::supports_avx512bw(), "");
 2713   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2714   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2715   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2716 }
 2717 
 2718 // This instruction produces ZF or CF flags
 2719 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2720   assert(VM_Version::supports_avx512bw(), "");
 2721   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2722   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2723   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2724 }
 2725 
 2726 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2727   assert(VM_Version::supports_avx512bw(), "");
 2728   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2729   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2730   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2731 }
 2732 
 2733 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2734   assert(VM_Version::supports_avx512dq(), "");
 2735   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2736   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2737   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2738 }
 2739 
 2740 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2741   assert(VM_Version::supports_avx512dq(), "");
 2742   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2743   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2744   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2745 }
 2746 
 2747 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2748   assert(VM_Version::supports_avx512bw(), "");
 2749   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2750   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2751   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2752 }
 2753 
 2754 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2755   assert(VM_Version::supports_avx512bw(), "");
 2756   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2757   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2758   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2759 }
 2760 
 2761 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2762   assert(VM_Version::supports_avx512dq(), "");
 2763   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2764   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2765   emit_int16(0x46, (0xC0 | encode));
 2766 }
 2767 
 2768 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2769   assert(VM_Version::supports_avx512dq(), "");
 2770   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2771   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2772   emit_int16(0x32, (0xC0 | encode));
 2773   emit_int8(imm8);
 2774 }
 2775 
 2776 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2777   assert(VM_Version::supports_avx512dq(), "");
 2778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2779   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2780   emit_int16(0x30, (0xC0 | encode));
 2781 }
 2782 
 2783 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2784   assert(VM_Version::supports_evex(), "");
 2785   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2786   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2787   emit_int16(0x30, (0xC0 | encode));
 2788   emit_int8(imm8);
 2789 }
 2790 
 2791 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2792   assert(VM_Version::supports_avx512bw(), "");
 2793   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2794   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2795   emit_int16(0x31, (0xC0 | encode));
 2796   emit_int8(imm8);
 2797 }
 2798 
 2799 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2800   assert(VM_Version::supports_avx512bw(), "");
 2801   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2802   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2803   emit_int16(0x31, (0xC0 | encode));
 2804   emit_int8(imm8);
 2805 }
 2806 
 2807 void Assembler::movb(Address dst, int imm8) {
 2808   InstructionMark im(this);
 2809    prefix(dst);
 2810   emit_int8((unsigned char)0xC6);
 2811   emit_operand(rax, dst, 1);
 2812   emit_int8(imm8);
 2813 }
 2814 
 2815 
 2816 void Assembler::movb(Address dst, Register src) {
 2817   assert(src->has_byte_register(), "must have byte register");
 2818   InstructionMark im(this);
 2819   prefix(dst, src, true);
 2820   emit_int8((unsigned char)0x88);
 2821   emit_operand(src, dst);
 2822 }
 2823 
 2824 void Assembler::movdl(XMMRegister dst, Register src) {
 2825   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2826   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2827   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2828   emit_int16(0x6E, (0xC0 | encode));
 2829 }
 2830 
 2831 void Assembler::movdl(Register dst, XMMRegister src) {
 2832   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2833   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2834   // swap src/dst to get correct prefix
 2835   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2836   emit_int16(0x7E, (0xC0 | encode));
 2837 }
 2838 
 2839 void Assembler::movdl(XMMRegister dst, Address src) {
 2840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2841   InstructionMark im(this);
 2842   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2843   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2844   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2845   emit_int8(0x6E);
 2846   emit_operand(dst, src);
 2847 }
 2848 
 2849 void Assembler::movdl(Address dst, XMMRegister src) {
 2850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2851   InstructionMark im(this);
 2852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2853   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2854   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2855   emit_int8(0x7E);
 2856   emit_operand(src, dst);
 2857 }
 2858 
 2859 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2861   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2862   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2863   emit_int16(0x6F, (0xC0 | encode));
 2864 }
 2865 
 2866 void Assembler::movdqa(XMMRegister dst, Address src) {
 2867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2868   InstructionMark im(this);
 2869   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2870   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2871   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2872   emit_int8(0x6F);
 2873   emit_operand(dst, src);
 2874 }
 2875 
 2876 void Assembler::movdqu(XMMRegister dst, Address src) {
 2877   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2878   InstructionMark im(this);
 2879   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2880   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2881   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2882   emit_int8(0x6F);
 2883   emit_operand(dst, src);
 2884 }
 2885 
 2886 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2887   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2888   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2889   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2890   emit_int16(0x6F, (0xC0 | encode));
 2891 }
 2892 
 2893 void Assembler::movdqu(Address dst, XMMRegister src) {
 2894   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2895   InstructionMark im(this);
 2896   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2897   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2898   attributes.reset_is_clear_context();
 2899   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2900   emit_int8(0x7F);
 2901   emit_operand(src, dst);
 2902 }
 2903 
 2904 // Move Unaligned 256bit Vector
 2905 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2906   assert(UseAVX > 0, "");
 2907   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2908   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2909   emit_int16(0x6F, (0xC0 | encode));
 2910 }
 2911 
 2912 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 2913   assert(UseAVX > 0, "");
 2914   InstructionMark im(this);
 2915   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2916   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2917   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2918   emit_int8(0x6F);
 2919   emit_operand(dst, src);
 2920 }
 2921 
 2922 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 2923   assert(UseAVX > 0, "");
 2924   InstructionMark im(this);
 2925   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2926   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2927   attributes.reset_is_clear_context();
 2928   // swap src<->dst for encoding
 2929   assert(src != xnoreg, "sanity");
 2930   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2931   emit_int8(0x7F);
 2932   emit_operand(src, dst);
 2933 }
 2934 
 2935 // Move Aligned 256bit Vector
 2936 
 2937 void Assembler::vmovdqa(XMMRegister dst, Address src) {
 2938   assert(UseAVX > 0, "");
 2939   InstructionMark im(this);
 2940   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2941   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2942   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2943   emit_int8(0x6F);
 2944   emit_operand(dst, src);
 2945 }
 2946 
 2947 void Assembler::vmovdqa(Address dst, XMMRegister src) {
 2948   assert(UseAVX > 0, "");
 2949   InstructionMark im(this);
 2950   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2951   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2952   attributes.reset_is_clear_context();
 2953   // swap src<->dst for encoding
 2954   assert(src != xnoreg, "sanity");
 2955   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2956   emit_int8(0x7F);
 2957   emit_operand(src, dst);
 2958 }
 2959 
 2960 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 2961 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 2962   assert(VM_Version::supports_evex(), "");
 2963   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2964   attributes.set_is_evex_instruction();
 2965   if (merge) {
 2966     attributes.reset_is_clear_context();
 2967   }
 2968   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2969   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2970   emit_int16(0x6F, (0xC0 | encode));
 2971 }
 2972 
 2973 void Assembler::evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) {
 2974   assert(VM_Version::supports_evex(), "");
 2975   InstructionMark im(this);
 2976   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2977   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2978   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2979   attributes.set_is_evex_instruction();
 2980   if (merge) {
 2981     attributes.reset_is_clear_context();
 2982   }
 2983   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2984   emit_int8(0x6F);
 2985   emit_operand(dst, src);
 2986 }
 2987 
 2988 void Assembler::evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) {
 2989   assert(VM_Version::supports_evex(), "");
 2990   assert(src != xnoreg, "sanity");
 2991   InstructionMark im(this);
 2992   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2993   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2994   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2995   attributes.set_is_evex_instruction();
 2996   if (merge) {
 2997     attributes.reset_is_clear_context();
 2998   }
 2999   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3000   emit_int8(0x7F);
 3001   emit_operand(src, dst);
 3002 }
 3003 
 3004 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3005   assert(VM_Version::supports_avx512vlbw(), "");
 3006   InstructionMark im(this);
 3007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3008   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3009   attributes.set_embedded_opmask_register_specifier(mask);
 3010   attributes.set_is_evex_instruction();
 3011   if (merge) {
 3012     attributes.reset_is_clear_context();
 3013   }
 3014   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3015   emit_int8(0x6F);
 3016   emit_operand(dst, src);
 3017 }
 3018 
 3019 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3020   assert(VM_Version::supports_avx512vlbw(), "");
 3021   assert(src != xnoreg, "sanity");
 3022   InstructionMark im(this);
 3023   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3024   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3025   attributes.set_embedded_opmask_register_specifier(mask);
 3026   attributes.set_is_evex_instruction();
 3027   if (merge) {
 3028     attributes.reset_is_clear_context();
 3029   }
 3030   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3031   emit_int8(0x7F);
 3032   emit_operand(src, dst);
 3033 }
 3034 
 3035 void Assembler::evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) {
 3036   assert(VM_Version::supports_evex(), "");
 3037   InstructionMark im(this);
 3038   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3039   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3040   attributes.set_is_evex_instruction();
 3041   if (merge) {
 3042     attributes.reset_is_clear_context();
 3043   }
 3044   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3045   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3046   emit_int8(0x6F);
 3047   emit_operand(dst, src);
 3048 }
 3049 
 3050 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3051   assert(VM_Version::supports_avx512vlbw(), "");
 3052   InstructionMark im(this);
 3053   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3054   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3055   attributes.set_embedded_opmask_register_specifier(mask);
 3056   attributes.set_is_evex_instruction();
 3057   if (merge) {
 3058     attributes.reset_is_clear_context();
 3059   }
 3060   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3061   emit_int8(0x6F);
 3062   emit_operand(dst, src);
 3063 }
 3064 
 3065 void Assembler::evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) {
 3066   assert(VM_Version::supports_evex(), "");
 3067   assert(src != xnoreg, "sanity");
 3068   InstructionMark im(this);
 3069   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3070   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3071   attributes.set_is_evex_instruction();
 3072   if (merge) {
 3073     attributes.reset_is_clear_context();
 3074   }
 3075   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3076   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3077   emit_int8(0x7F);
 3078   emit_operand(src, dst);
 3079 }
 3080 
 3081 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3082   assert(VM_Version::supports_avx512vlbw(), "");
 3083   assert(src != xnoreg, "sanity");
 3084   InstructionMark im(this);
 3085   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3086   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3087   attributes.set_embedded_opmask_register_specifier(mask);
 3088   attributes.set_is_evex_instruction();
 3089   if (merge) {
 3090     attributes.reset_is_clear_context();
 3091   }
 3092   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3093   emit_int8(0x7F);
 3094   emit_operand(src, dst);
 3095 }
 3096 
 3097 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3098   // Unmasked instruction
 3099   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3100 }
 3101 
 3102 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3103   assert(VM_Version::supports_evex(), "");
 3104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3105   attributes.set_embedded_opmask_register_specifier(mask);
 3106   attributes.set_is_evex_instruction();
 3107   if (merge) {
 3108     attributes.reset_is_clear_context();
 3109   }
 3110   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3111   emit_int16(0x6F, (0xC0 | encode));
 3112 }
 3113 
 3114 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3115   // Unmasked instruction
 3116   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3117 }
 3118 
 3119 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3120   assert(VM_Version::supports_evex(), "");
 3121   InstructionMark im(this);
 3122   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3123   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3124   attributes.set_embedded_opmask_register_specifier(mask);
 3125   attributes.set_is_evex_instruction();
 3126   if (merge) {
 3127     attributes.reset_is_clear_context();
 3128   }
 3129   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3130   emit_int8(0x6F);
 3131   emit_operand(dst, src);
 3132 }
 3133 
 3134 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3135   // Unmasked isntruction
 3136   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3137 }
 3138 
 3139 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3140   assert(VM_Version::supports_evex(), "");
 3141   assert(src != xnoreg, "sanity");
 3142   InstructionMark im(this);
 3143   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3144   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3145   attributes.set_embedded_opmask_register_specifier(mask);
 3146   attributes.set_is_evex_instruction();
 3147   if (merge) {
 3148     attributes.reset_is_clear_context();
 3149   }
 3150   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3151   emit_int8(0x7F);
 3152   emit_operand(src, dst);
 3153 }
 3154 
 3155 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3156   // Unmasked instruction
 3157   if (dst->encoding() == src->encoding()) return;
 3158   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3159 }
 3160 
 3161 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3162   assert(VM_Version::supports_evex(), "");
 3163   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3164   attributes.set_embedded_opmask_register_specifier(mask);
 3165   attributes.set_is_evex_instruction();
 3166   if (merge) {
 3167     attributes.reset_is_clear_context();
 3168   }
 3169   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3170   emit_int16(0x6F, (0xC0 | encode));
 3171 }
 3172 
 3173 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3174   // Unmasked instruction
 3175   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3176 }
 3177 
 3178 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3179   assert(VM_Version::supports_evex(), "");
 3180   InstructionMark im(this);
 3181   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3182   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3183   attributes.set_embedded_opmask_register_specifier(mask);
 3184   attributes.set_is_evex_instruction();
 3185   if (merge) {
 3186     attributes.reset_is_clear_context();
 3187   }
 3188   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3189   emit_int8(0x6F);
 3190   emit_operand(dst, src);
 3191 }
 3192 
 3193 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3194   // Unmasked instruction
 3195   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3196 }
 3197 
 3198 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3199   assert(VM_Version::supports_evex(), "");
 3200   assert(src != xnoreg, "sanity");
 3201   InstructionMark im(this);
 3202   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3203   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3204   attributes.set_embedded_opmask_register_specifier(mask);
 3205   if (merge) {
 3206     attributes.reset_is_clear_context();
 3207   }
 3208   attributes.set_is_evex_instruction();
 3209   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3210   emit_int8(0x7F);
 3211   emit_operand(src, dst);
 3212 }
 3213 
 3214 // Move Aligned EVEX enabled Vector (programmable : 8,16,32,64)
 3215 void Assembler::evmovdqab(XMMRegister dst, Address src, int vector_len) {
 3216   assert(VM_Version::supports_evex(), "");
 3217   InstructionMark im(this);
 3218   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3219   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3220   attributes.set_is_evex_instruction();
 3221   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3222   emit_int8(0x6F);
 3223   emit_operand(dst, src);
 3224 }
 3225 
 3226 void Assembler::evmovdqab(Address dst, XMMRegister src, int vector_len) {
 3227   assert(VM_Version::supports_evex(), "");
 3228   assert(src != xnoreg, "sanity");
 3229   InstructionMark im(this);
 3230   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3231   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3232   attributes.set_is_evex_instruction();
 3233   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3234   emit_int8(0x7F);
 3235   emit_operand(src, dst);
 3236 }
 3237 
 3238 void Assembler::evmovdqab(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 3239   assert(VM_Version::supports_avx512vlbw(), "");
 3240   InstructionMark im(this);
 3241   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3242   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3243   attributes.set_embedded_opmask_register_specifier(mask);
 3244   attributes.set_is_evex_instruction();
 3245   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3246   emit_int8(0x6F);
 3247   emit_operand(dst, src);
 3248 }
 3249 
 3250 void Assembler::evmovdqaw(XMMRegister dst, Address src, int vector_len) {
 3251   assert(VM_Version::supports_evex(), "");
 3252   InstructionMark im(this);
 3253   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3254   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3255   attributes.set_is_evex_instruction();
 3256   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3257   emit_int8(0x6F);
 3258   emit_operand(dst, src);
 3259 }
 3260 
 3261 void Assembler::evmovdqaw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 3262   assert(VM_Version::supports_avx512vlbw(), "");
 3263   InstructionMark im(this);
 3264   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3265   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3266   attributes.set_embedded_opmask_register_specifier(mask);
 3267   attributes.set_is_evex_instruction();
 3268   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3269   emit_int8(0x6F);
 3270   emit_operand(dst, src);
 3271 }
 3272 
 3273 void Assembler::evmovdqaw(Address dst, XMMRegister src, int vector_len) {
 3274   assert(VM_Version::supports_evex(), "");
 3275   assert(src != xnoreg, "sanity");
 3276   InstructionMark im(this);
 3277   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3278   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3279   attributes.set_is_evex_instruction();
 3280   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3281   emit_int8(0x7F);
 3282   emit_operand(src, dst);
 3283 }
 3284 
 3285 void Assembler::evmovdqaw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 3286   assert(VM_Version::supports_avx512vlbw(), "");
 3287   assert(src != xnoreg, "sanity");
 3288   InstructionMark im(this);
 3289   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3290   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3291   attributes.reset_is_clear_context();
 3292   attributes.set_embedded_opmask_register_specifier(mask);
 3293   attributes.set_is_evex_instruction();
 3294   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3295   emit_int8(0x7F);
 3296   emit_operand(src, dst);
 3297 }
 3298 
 3299 void Assembler::evmovdqal(XMMRegister dst, Address src, int vector_len) {
 3300   assert(VM_Version::supports_evex(), "");
 3301   InstructionMark im(this);
 3302   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ true);
 3303   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3304   attributes.set_is_evex_instruction();
 3305   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3306   emit_int8(0x6F);
 3307   emit_operand(dst, src);
 3308 }
 3309 
 3310 void Assembler::evmovdqal(Address dst, XMMRegister src, int vector_len) {
 3311   assert(VM_Version::supports_evex(), "");
 3312   assert(src != xnoreg, "sanity");
 3313   InstructionMark im(this);
 3314   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3315   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3316   attributes.reset_is_clear_context();
 3317   attributes.set_is_evex_instruction();
 3318   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3319   emit_int8(0x7F);
 3320   emit_operand(src, dst);
 3321 }
 3322 
 3323 void Assembler::evmovdqaq(XMMRegister dst, Address src, int vector_len) {
 3324   assert(VM_Version::supports_evex(), "");
 3325   InstructionMark im(this);
 3326   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3327   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3328   attributes.set_is_evex_instruction();
 3329   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3330   emit_int8(0x6F);
 3331   emit_operand(dst, src);
 3332 }
 3333 
 3334 void Assembler::evmovdqaq(Address dst, XMMRegister src, int vector_len) {
 3335   assert(VM_Version::supports_evex(), "");
 3336   assert(src != xnoreg, "sanity");
 3337   InstructionMark im(this);
 3338   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3339   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3340   attributes.reset_is_clear_context();
 3341   attributes.set_is_evex_instruction();
 3342   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3343   emit_int8(0x7F);
 3344   emit_operand(src, dst);
 3345 }
 3346 
 3347 // Uses zero extension on 64bit
 3348 
 3349 void Assembler::movl(Register dst, int32_t imm32) {
 3350   int encode = prefix_and_encode(dst->encoding());
 3351   emit_int8(0xB8 | encode);
 3352   emit_int32(imm32);
 3353 }
 3354 
 3355 void Assembler::movl(Register dst, Register src) {
 3356   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3357   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3358 }
 3359 
 3360 void Assembler::movl(Register dst, Address src) {
 3361   InstructionMark im(this);
 3362   prefix(src, dst);
 3363   emit_int8((unsigned char)0x8B);
 3364   emit_operand(dst, src);
 3365 }
 3366 
 3367 void Assembler::movl(Address dst, int32_t imm32) {
 3368   InstructionMark im(this);
 3369   prefix(dst);
 3370   emit_int8((unsigned char)0xC7);
 3371   emit_operand(rax, dst, 4);
 3372   emit_int32(imm32);
 3373 }
 3374 
 3375 void Assembler::movl(Address dst, Register src) {
 3376   InstructionMark im(this);
 3377   prefix(dst, src);
 3378   emit_int8((unsigned char)0x89);
 3379   emit_operand(src, dst);
 3380 }
 3381 
 3382 // New cpus require to use movsd and movss to avoid partial register stall
 3383 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3384 // The selection is done in MacroAssembler::movdbl() and movflt().
 3385 void Assembler::movlpd(XMMRegister dst, Address src) {
 3386   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3387   InstructionMark im(this);
 3388   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3389   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3390   attributes.set_rex_vex_w_reverted();
 3391   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3392   emit_int8(0x12);
 3393   emit_operand(dst, src);
 3394 }
 3395 
 3396 void Assembler::movntq(Address dst, Register src) { // uses the MOVNTI operation
 3397   InstructionMark im(this);
 3398   prefixq(dst);
 3399   emit_int8(0x0F);
 3400   emit_int8((unsigned char)0xC3);
 3401   emit_operand(src, dst);
 3402 }
 3403 
 3404 void Assembler::movntdq(Address dst, XMMRegister src) {
 3405   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3406   InstructionMark im(this);
 3407   emit_int8((unsigned char)0x66);
 3408   emit_int8(0x0F);
 3409   emit_int8((unsigned char)0xE7);
 3410   emit_operand(src, dst);
 3411 }
 3412 
 3413 void Assembler::movntdqa(XMMRegister dst, Address src) {
 3414   NOT_LP64(assert(VM_Version::supports_sse4_1(), ""));
 3415   emit_int8((unsigned char)0x66);
 3416   emit_int8(0x0F);
 3417   emit_int8(0x38);
 3418   emit_int8((unsigned char)0x2A);
 3419   emit_operand(dst, src);
 3420 
 3421   // Same thing, with AVX:
 3422   // assert(UseAVX > 0, "");
 3423   // InstructionMark im(this);
 3424   // InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3425   // simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3426   // emit_int8((unsigned char)0x2A);
 3427   // emit_operand(dst, src);
 3428 }
 3429 
 3430 void Assembler::vmovntdqa(XMMRegister dst, Address src) {
 3431   assert(UseAVX > 0, "");
 3432   InstructionMark im(this);
 3433   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3434   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3435   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3436   emit_int8(0x2A);
 3437   emit_operand(dst, src);
 3438 }
 3439 
 3440 void Assembler::evmovntdqa(XMMRegister dst, Address src, int vector_len) {
 3441   assert(VM_Version::supports_evex(), "");
 3442   InstructionMark im(this);
 3443   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3444   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3445   attributes.set_is_evex_instruction();
 3446   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3447   emit_int8(0x2A);
 3448   emit_operand(dst, src);
 3449 }
 3450 
 3451 void Assembler::vmovntdq(Address dst, XMMRegister src) {
 3452   assert(UseAVX > 0, "");
 3453   InstructionMark im(this);
 3454   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3455   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3456   attributes.reset_is_clear_context();
 3457   // swap src<->dst for encoding
 3458   assert(src != xnoreg, "sanity");
 3459   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3460   emit_int8((unsigned char)0xE7);
 3461   emit_operand(src, dst);
 3462 }
 3463 
 3464 void Assembler::evmovntdq(Address dst, XMMRegister src, int vector_len) {
 3465   assert(VM_Version::supports_evex(), "");
 3466   assert(src != xnoreg, "sanity");
 3467   InstructionMark im(this);
 3468   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3469   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3470   attributes.reset_is_clear_context();
 3471   attributes.set_is_evex_instruction();
 3472   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3473   emit_int8((unsigned char)0xE7);
 3474   emit_operand(src, dst);
 3475 }
 3476 
 3477 void Assembler::movq(XMMRegister dst, Address src) {
 3478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3479   InstructionMark im(this);
 3480   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3482   attributes.set_rex_vex_w_reverted();
 3483   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3484   emit_int8(0x7E);
 3485   emit_operand(dst, src);
 3486 }
 3487 
 3488 void Assembler::movq(Address dst, XMMRegister src) {
 3489   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3490   InstructionMark im(this);
 3491   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3492   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3493   attributes.set_rex_vex_w_reverted();
 3494   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3495   emit_int8((unsigned char)0xD6);
 3496   emit_operand(src, dst);
 3497 }
 3498 
 3499 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3500   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3501   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3502   attributes.set_rex_vex_w_reverted();
 3503   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3504   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3505 }
 3506 
 3507 void Assembler::movq(Register dst, XMMRegister src) {
 3508   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3509   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3510   // swap src/dst to get correct prefix
 3511   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3512   emit_int16(0x7E, (0xC0 | encode));
 3513 }
 3514 
 3515 void Assembler::movq(XMMRegister dst, Register src) {
 3516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3517   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3518   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3519   emit_int16(0x6E, (0xC0 | encode));
 3520 }
 3521 
 3522 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3523   InstructionMark im(this);
 3524   prefix(src, dst);
 3525   emit_int16(0x0F, (unsigned char)0xBE);
 3526   emit_operand(dst, src);
 3527 }
 3528 
 3529 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3530   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3531   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3532   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3533 }
 3534 
 3535 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3536   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3537   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3538   attributes.set_rex_vex_w_reverted();
 3539   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3540   emit_int16(0x10, (0xC0 | encode));
 3541 }
 3542 
 3543 void Assembler::movsd(XMMRegister dst, Address src) {
 3544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3545   InstructionMark im(this);
 3546   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3547   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3548   attributes.set_rex_vex_w_reverted();
 3549   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3550   emit_int8(0x10);
 3551   emit_operand(dst, src);
 3552 }
 3553 
 3554 void Assembler::movsd(Address dst, XMMRegister src) {
 3555   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3556   InstructionMark im(this);
 3557   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3558   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3559   attributes.reset_is_clear_context();
 3560   attributes.set_rex_vex_w_reverted();
 3561   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3562   emit_int8(0x11);
 3563   emit_operand(src, dst);
 3564 }
 3565 
 3566 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3567   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3568   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3569   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3570   emit_int16(0x10, (0xC0 | encode));
 3571 }
 3572 
 3573 void Assembler::movss(XMMRegister dst, Address src) {
 3574   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3575   InstructionMark im(this);
 3576   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3577   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3578   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3579   emit_int8(0x10);
 3580   emit_operand(dst, src);
 3581 }
 3582 
 3583 void Assembler::movss(Address dst, XMMRegister src) {
 3584   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3585   InstructionMark im(this);
 3586   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3587   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3588   attributes.reset_is_clear_context();
 3589   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3590   emit_int8(0x11);
 3591   emit_operand(src, dst);
 3592 }
 3593 
 3594 void Assembler::movswl(Register dst, Address src) { // movsxw
 3595   InstructionMark im(this);
 3596   prefix(src, dst);
 3597   emit_int16(0x0F, (unsigned char)0xBF);
 3598   emit_operand(dst, src);
 3599 }
 3600 
 3601 void Assembler::movswl(Register dst, Register src) { // movsxw
 3602   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3603   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3604 }
 3605 
 3606 void Assembler::movw(Address dst, int imm16) {
 3607   InstructionMark im(this);
 3608 
 3609   emit_int8(0x66); // switch to 16-bit mode
 3610   prefix(dst);
 3611   emit_int8((unsigned char)0xC7);
 3612   emit_operand(rax, dst, 2);
 3613   emit_int16(imm16);
 3614 }
 3615 
 3616 void Assembler::movw(Register dst, Address src) {
 3617   InstructionMark im(this);
 3618   emit_int8(0x66);
 3619   prefix(src, dst);
 3620   emit_int8((unsigned char)0x8B);
 3621   emit_operand(dst, src);
 3622 }
 3623 
 3624 void Assembler::movw(Address dst, Register src) {
 3625   InstructionMark im(this);
 3626   emit_int8(0x66);
 3627   prefix(dst, src);
 3628   emit_int8((unsigned char)0x89);
 3629   emit_operand(src, dst);
 3630 }
 3631 
 3632 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3633   InstructionMark im(this);
 3634   prefix(src, dst);
 3635   emit_int16(0x0F, (unsigned char)0xB6);
 3636   emit_operand(dst, src);
 3637 }
 3638 
 3639 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3640   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3641   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3642   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3643 }
 3644 
 3645 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3646   InstructionMark im(this);
 3647   prefix(src, dst);
 3648   emit_int16(0x0F, (unsigned char)0xB7);
 3649   emit_operand(dst, src);
 3650 }
 3651 
 3652 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3653   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3654   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3655 }
 3656 
 3657 void Assembler::mull(Address src) {
 3658   InstructionMark im(this);
 3659   prefix(src);
 3660   emit_int8((unsigned char)0xF7);
 3661   emit_operand(rsp, src);
 3662 }
 3663 
 3664 void Assembler::mull(Register src) {
 3665   int encode = prefix_and_encode(src->encoding());
 3666   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3667 }
 3668 
 3669 void Assembler::mulsd(XMMRegister dst, Address src) {
 3670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3671   InstructionMark im(this);
 3672   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3673   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3674   attributes.set_rex_vex_w_reverted();
 3675   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3676   emit_int8(0x59);
 3677   emit_operand(dst, src);
 3678 }
 3679 
 3680 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3681   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3682   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3683   attributes.set_rex_vex_w_reverted();
 3684   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3685   emit_int16(0x59, (0xC0 | encode));
 3686 }
 3687 
 3688 void Assembler::mulss(XMMRegister dst, Address src) {
 3689   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3690   InstructionMark im(this);
 3691   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3692   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3693   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3694   emit_int8(0x59);
 3695   emit_operand(dst, src);
 3696 }
 3697 
 3698 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3699   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3700   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3701   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3702   emit_int16(0x59, (0xC0 | encode));
 3703 }
 3704 
 3705 void Assembler::negl(Register dst) {
 3706   int encode = prefix_and_encode(dst->encoding());
 3707   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3708 }
 3709 
 3710 void Assembler::negl(Address dst) {
 3711   InstructionMark im(this);
 3712   prefix(dst);
 3713   emit_int8((unsigned char)0xF7);
 3714   emit_operand(as_Register(3), dst);
 3715 }
 3716 
 3717 void Assembler::nop(int i) {
 3718 #ifdef ASSERT
 3719   assert(i > 0, " ");
 3720   // The fancy nops aren't currently recognized by debuggers making it a
 3721   // pain to disassemble code while debugging. If asserts are on clearly
 3722   // speed is not an issue so simply use the single byte traditional nop
 3723   // to do alignment.
 3724 
 3725   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3726   return;
 3727 
 3728 #endif // ASSERT
 3729 
 3730   if (UseAddressNop && VM_Version::is_intel()) {
 3731     //
 3732     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3733     //  1: 0x90
 3734     //  2: 0x66 0x90
 3735     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3736     //  4: 0x0F 0x1F 0x40 0x00
 3737     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3738     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3739     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3740     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3741     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3742     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3743     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3744 
 3745     // The rest coding is Intel specific - don't use consecutive address nops
 3746 
 3747     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3748     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3749     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3750     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3751 
 3752     while(i >= 15) {
 3753       // For Intel don't generate consecutive addess nops (mix with regular nops)
 3754       i -= 15;
 3755       emit_int24(0x66, 0x66, 0x66);
 3756       addr_nop_8();
 3757       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3758     }
 3759     switch (i) {
 3760       case 14:
 3761         emit_int8(0x66); // size prefix
 3762       case 13:
 3763         emit_int8(0x66); // size prefix
 3764       case 12:
 3765         addr_nop_8();
 3766         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3767         break;
 3768       case 11:
 3769         emit_int8(0x66); // size prefix
 3770       case 10:
 3771         emit_int8(0x66); // size prefix
 3772       case 9:
 3773         emit_int8(0x66); // size prefix
 3774       case 8:
 3775         addr_nop_8();
 3776         break;
 3777       case 7:
 3778         addr_nop_7();
 3779         break;
 3780       case 6:
 3781         emit_int8(0x66); // size prefix
 3782       case 5:
 3783         addr_nop_5();
 3784         break;
 3785       case 4:
 3786         addr_nop_4();
 3787         break;
 3788       case 3:
 3789         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3790         emit_int8(0x66); // size prefix
 3791       case 2:
 3792         emit_int8(0x66); // size prefix
 3793       case 1:
 3794         emit_int8((unsigned char)0x90);
 3795                          // nop
 3796         break;
 3797       default:
 3798         assert(i == 0, " ");
 3799     }
 3800     return;
 3801   }
 3802   if (UseAddressNop && VM_Version::is_amd_family()) {
 3803     //
 3804     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3805     //  1: 0x90
 3806     //  2: 0x66 0x90
 3807     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3808     //  4: 0x0F 0x1F 0x40 0x00
 3809     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3810     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3811     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3812     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3813     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3814     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3815     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3816 
 3817     // The rest coding is AMD specific - use consecutive address nops
 3818 
 3819     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3820     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3821     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3822     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3823     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3824     //     Size prefixes (0x66) are added for larger sizes
 3825 
 3826     while(i >= 22) {
 3827       i -= 11;
 3828       emit_int24(0x66, 0x66, 0x66);
 3829       addr_nop_8();
 3830     }
 3831     // Generate first nop for size between 21-12
 3832     switch (i) {
 3833       case 21:
 3834         i -= 1;
 3835         emit_int8(0x66); // size prefix
 3836       case 20:
 3837       case 19:
 3838         i -= 1;
 3839         emit_int8(0x66); // size prefix
 3840       case 18:
 3841       case 17:
 3842         i -= 1;
 3843         emit_int8(0x66); // size prefix
 3844       case 16:
 3845       case 15:
 3846         i -= 8;
 3847         addr_nop_8();
 3848         break;
 3849       case 14:
 3850       case 13:
 3851         i -= 7;
 3852         addr_nop_7();
 3853         break;
 3854       case 12:
 3855         i -= 6;
 3856         emit_int8(0x66); // size prefix
 3857         addr_nop_5();
 3858         break;
 3859       default:
 3860         assert(i < 12, " ");
 3861     }
 3862 
 3863     // Generate second nop for size between 11-1
 3864     switch (i) {
 3865       case 11:
 3866         emit_int8(0x66); // size prefix
 3867       case 10:
 3868         emit_int8(0x66); // size prefix
 3869       case 9:
 3870         emit_int8(0x66); // size prefix
 3871       case 8:
 3872         addr_nop_8();
 3873         break;
 3874       case 7:
 3875         addr_nop_7();
 3876         break;
 3877       case 6:
 3878         emit_int8(0x66); // size prefix
 3879       case 5:
 3880         addr_nop_5();
 3881         break;
 3882       case 4:
 3883         addr_nop_4();
 3884         break;
 3885       case 3:
 3886         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3887         emit_int8(0x66); // size prefix
 3888       case 2:
 3889         emit_int8(0x66); // size prefix
 3890       case 1:
 3891         emit_int8((unsigned char)0x90);
 3892                          // nop
 3893         break;
 3894       default:
 3895         assert(i == 0, " ");
 3896     }
 3897     return;
 3898   }
 3899 
 3900   if (UseAddressNop && VM_Version::is_zx()) {
 3901     //
 3902     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3903     //  1: 0x90
 3904     //  2: 0x66 0x90
 3905     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3906     //  4: 0x0F 0x1F 0x40 0x00
 3907     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3908     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3909     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3910     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3911     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3912     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3913     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3914 
 3915     // The rest coding is ZX specific - don't use consecutive address nops
 3916 
 3917     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3918     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3919     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3920     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3921 
 3922     while (i >= 15) {
 3923       // For ZX don't generate consecutive addess nops (mix with regular nops)
 3924       i -= 15;
 3925       emit_int24(0x66, 0x66, 0x66);
 3926       addr_nop_8();
 3927       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3928     }
 3929     switch (i) {
 3930       case 14:
 3931         emit_int8(0x66); // size prefix
 3932       case 13:
 3933         emit_int8(0x66); // size prefix
 3934       case 12:
 3935         addr_nop_8();
 3936         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3937         break;
 3938       case 11:
 3939         emit_int8(0x66); // size prefix
 3940       case 10:
 3941         emit_int8(0x66); // size prefix
 3942       case 9:
 3943         emit_int8(0x66); // size prefix
 3944       case 8:
 3945         addr_nop_8();
 3946         break;
 3947       case 7:
 3948         addr_nop_7();
 3949         break;
 3950       case 6:
 3951         emit_int8(0x66); // size prefix
 3952       case 5:
 3953         addr_nop_5();
 3954         break;
 3955       case 4:
 3956         addr_nop_4();
 3957         break;
 3958       case 3:
 3959         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3960         emit_int8(0x66); // size prefix
 3961       case 2:
 3962         emit_int8(0x66); // size prefix
 3963       case 1:
 3964         emit_int8((unsigned char)0x90);
 3965                          // nop
 3966         break;
 3967       default:
 3968         assert(i == 0, " ");
 3969     }
 3970     return;
 3971   }
 3972 
 3973   // Using nops with size prefixes "0x66 0x90".
 3974   // From AMD Optimization Guide:
 3975   //  1: 0x90
 3976   //  2: 0x66 0x90
 3977   //  3: 0x66 0x66 0x90
 3978   //  4: 0x66 0x66 0x66 0x90
 3979   //  5: 0x66 0x66 0x90 0x66 0x90
 3980   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3981   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3982   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3983   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3984   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3985   //
 3986   while (i > 12) {
 3987     i -= 4;
 3988     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3989   }
 3990   // 1 - 12 nops
 3991   if (i > 8) {
 3992     if (i > 9) {
 3993       i -= 1;
 3994       emit_int8(0x66);
 3995     }
 3996     i -= 3;
 3997     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3998   }
 3999   // 1 - 8 nops
 4000   if (i > 4) {
 4001     if (i > 6) {
 4002       i -= 1;
 4003       emit_int8(0x66);
 4004     }
 4005     i -= 3;
 4006     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4007   }
 4008   switch (i) {
 4009     case 4:
 4010       emit_int8(0x66);
 4011     case 3:
 4012       emit_int8(0x66);
 4013     case 2:
 4014       emit_int8(0x66);
 4015     case 1:
 4016       emit_int8((unsigned char)0x90);
 4017       break;
 4018     default:
 4019       assert(i == 0, " ");
 4020   }
 4021 }
 4022 
 4023 void Assembler::notl(Register dst) {
 4024   int encode = prefix_and_encode(dst->encoding());
 4025   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 4026 }
 4027 
 4028 void Assembler::orw(Register dst, Register src) {
 4029   (void)prefix_and_encode(dst->encoding(), src->encoding());
 4030   emit_arith(0x0B, 0xC0, dst, src);
 4031 }
 4032 
 4033 void Assembler::orl(Address dst, int32_t imm32) {
 4034   InstructionMark im(this);
 4035   prefix(dst);
 4036   emit_arith_operand(0x81, rcx, dst, imm32);
 4037 }
 4038 
 4039 void Assembler::orl(Register dst, int32_t imm32) {
 4040   prefix(dst);
 4041   emit_arith(0x81, 0xC8, dst, imm32);
 4042 }
 4043 
 4044 void Assembler::orl(Register dst, Address src) {
 4045   InstructionMark im(this);
 4046   prefix(src, dst);
 4047   emit_int8(0x0B);
 4048   emit_operand(dst, src);
 4049 }
 4050 
 4051 void Assembler::orl(Register dst, Register src) {
 4052   (void) prefix_and_encode(dst->encoding(), src->encoding());
 4053   emit_arith(0x0B, 0xC0, dst, src);
 4054 }
 4055 
 4056 void Assembler::orl(Address dst, Register src) {
 4057   InstructionMark im(this);
 4058   prefix(dst, src);
 4059   emit_int8(0x09);
 4060   emit_operand(src, dst);
 4061 }
 4062 
 4063 void Assembler::orb(Address dst, int imm8) {
 4064   InstructionMark im(this);
 4065   prefix(dst);
 4066   emit_int8((unsigned char)0x80);
 4067   emit_operand(rcx, dst, 1);
 4068   emit_int8(imm8);
 4069 }
 4070 
 4071 void Assembler::orb(Address dst, Register src) {
 4072   InstructionMark im(this);
 4073   prefix(dst, src, true);
 4074   emit_int8(0x08);
 4075   emit_operand(src, dst);
 4076 }
 4077 
 4078 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 4079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4080   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4081   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4082   emit_int16(0x63, (0xC0 | encode));
 4083 }
 4084 
 4085 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4086   assert(UseAVX > 0, "some form of AVX must be enabled");
 4087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4088   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4089   emit_int16(0x63, (0xC0 | encode));
 4090 }
 4091 
 4092 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 4093   assert(VM_Version::supports_sse2(), "");
 4094   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4095   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4096   emit_int16(0x6B, (0xC0 | encode));
 4097 }
 4098 
 4099 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4100   assert(UseAVX > 0, "some form of AVX must be enabled");
 4101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4102   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4103   emit_int16(0x6B, (0xC0 | encode));
 4104 }
 4105 
 4106 void Assembler::packuswb(XMMRegister dst, Address src) {
 4107   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4108   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4109   InstructionMark im(this);
 4110   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4111   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4112   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4113   emit_int8(0x67);
 4114   emit_operand(dst, src);
 4115 }
 4116 
 4117 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 4118   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4119   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4120   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4121   emit_int16(0x67, (0xC0 | encode));
 4122 }
 4123 
 4124 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4125   assert(UseAVX > 0, "some form of AVX must be enabled");
 4126   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4127   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4128   emit_int16(0x67, (0xC0 | encode));
 4129 }
 4130 
 4131 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4132   assert(VM_Version::supports_sse4_1(), "");
 4133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4134   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4135   emit_int16(0x2B, (0xC0 | encode));
 4136 }
 4137 
 4138 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4139   assert(UseAVX > 0, "some form of AVX must be enabled");
 4140   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4141   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4142   emit_int16(0x2B, (0xC0 | encode));
 4143 }
 4144 
 4145 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4146   assert(VM_Version::supports_avx2(), "");
 4147   assert(vector_len != AVX_128bit, "");
 4148   // VEX.256.66.0F3A.W1 00 /r ib
 4149   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4150   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4151   emit_int24(0x00, (0xC0 | encode), imm8);
 4152 }
 4153 
 4154 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4155   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4156          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4157   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4158   attributes.set_is_evex_instruction();
 4159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4160   emit_int16(0x36, (0xC0 | encode));
 4161 }
 4162 
 4163 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4164   assert(VM_Version::supports_avx512_vbmi(), "");
 4165   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4166   attributes.set_is_evex_instruction();
 4167   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4168   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4169 }
 4170 
 4171 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4172   assert(VM_Version::supports_avx512_vbmi(), "");
 4173   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4174   attributes.set_is_evex_instruction();
 4175   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4176   emit_int8((unsigned char)0x8D);
 4177   emit_operand(dst, src);
 4178 }
 4179 
 4180 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4181   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4182          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4183          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4184   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4185   attributes.set_is_evex_instruction();
 4186   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4187   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4188 }
 4189 
 4190 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4191   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4192   // VEX.NDS.256.66.0F38.W0 36 /r
 4193   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4194   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4195   emit_int16(0x36, (0xC0 | encode));
 4196 }
 4197 
 4198 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4199   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4200   // VEX.NDS.256.66.0F38.W0 36 /r
 4201   InstructionMark im(this);
 4202   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4203   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4204   emit_int8(0x36);
 4205   emit_operand(dst, src);
 4206 }
 4207 
 4208 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4209   assert(VM_Version::supports_avx2(), "");
 4210   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4211   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4212   emit_int24(0x46, (0xC0 | encode), imm8);
 4213 }
 4214 
 4215 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4216   assert(VM_Version::supports_avx(), "");
 4217   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4218   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4219   emit_int24(0x06, (0xC0 | encode), imm8);
 4220 }
 4221 
 4222 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4223   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4224   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4225   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4226   emit_int24(0x04, (0xC0 | encode), imm8);
 4227 }
 4228 
 4229 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4230   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4231   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4232   attributes.set_rex_vex_w_reverted();
 4233   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4234   emit_int24(0x05, (0xC0 | encode), imm8);
 4235 }
 4236 
 4237 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4238   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4239   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4240   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4241   emit_int24(0x01, (0xC0 | encode), imm8);
 4242 }
 4243 
 4244 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4245   assert(VM_Version::supports_evex(), "");
 4246   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4247   attributes.set_is_evex_instruction();
 4248   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4249   emit_int16(0x76, (0xC0 | encode));
 4250 }
 4251 
 4252 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4253   assert(VM_Version::supports_avx512_vbmi(), "");
 4254   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4255   attributes.set_is_evex_instruction();
 4256   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4257   emit_int16(0x7D, (0xC0 | encode));
 4258 }
 4259 
 4260 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4261   assert(VM_Version::supports_avx512_vbmi(), "");
 4262   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4263   attributes.set_is_evex_instruction();
 4264   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4265   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4266 }
 4267 
 4268 void Assembler::pause() {
 4269   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4270 }
 4271 
 4272 void Assembler::ud2() {
 4273   emit_int16(0x0F, 0x0B);
 4274 }
 4275 
 4276 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4277   assert(VM_Version::supports_sse4_2(), "");
 4278   InstructionMark im(this);
 4279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4280   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4281   emit_int8(0x61);
 4282   emit_operand(dst, src);
 4283   emit_int8(imm8);
 4284 }
 4285 
 4286 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4287   assert(VM_Version::supports_sse4_2(), "");
 4288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4289   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4290   emit_int24(0x61, (0xC0 | encode), imm8);
 4291 }
 4292 
 4293 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4294 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4295   assert(VM_Version::supports_sse2(), "");
 4296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4298   emit_int16(0x74, (0xC0 | encode));
 4299 }
 4300 
 4301 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4302   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4303   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4304   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4305   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4306   emit_int16(cond_encoding, (0xC0 | encode));
 4307 }
 4308 
 4309 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4310 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4311   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4312   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4313   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4314   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4315   emit_int16(0x74, (0xC0 | encode));
 4316 }
 4317 
 4318 // In this context, kdst is written the mask used to process the equal components
 4319 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4320   assert(VM_Version::supports_avx512bw(), "");
 4321   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4322   attributes.set_is_evex_instruction();
 4323   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4324   emit_int16(0x74, (0xC0 | encode));
 4325 }
 4326 
 4327 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4328   assert(VM_Version::supports_avx512vlbw(), "");
 4329   InstructionMark im(this);
 4330   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4331   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4332   attributes.set_is_evex_instruction();
 4333   int dst_enc = kdst->encoding();
 4334   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4335   emit_int8(0x64);
 4336   emit_operand(as_Register(dst_enc), src);
 4337 }
 4338 
 4339 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4340   assert(VM_Version::supports_avx512vlbw(), "");
 4341   InstructionMark im(this);
 4342   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4343   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4344   attributes.reset_is_clear_context();
 4345   attributes.set_embedded_opmask_register_specifier(mask);
 4346   attributes.set_is_evex_instruction();
 4347   int dst_enc = kdst->encoding();
 4348   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4349   emit_int8(0x64);
 4350   emit_operand(as_Register(dst_enc), src);
 4351 }
 4352 
 4353 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4354   assert(VM_Version::supports_avx512vlbw(), "");
 4355   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4356   attributes.set_is_evex_instruction();
 4357   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4358   emit_int24(0x3E, (0xC0 | encode), vcc);
 4359 }
 4360 
 4361 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4362   assert(VM_Version::supports_avx512vlbw(), "");
 4363   InstructionMark im(this);
 4364   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4365   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4366   attributes.set_is_evex_instruction();
 4367   int dst_enc = kdst->encoding();
 4368   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4369   emit_int8(0x3E);
 4370   emit_operand(as_Register(dst_enc), src);
 4371   emit_int8(vcc);
 4372 }
 4373 
 4374 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4375   assert(VM_Version::supports_avx512bw(), "");
 4376   InstructionMark im(this);
 4377   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4378   attributes.set_is_evex_instruction();
 4379   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4380   int dst_enc = kdst->encoding();
 4381   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4382   emit_int8(0x74);
 4383   emit_operand(as_Register(dst_enc), src);
 4384 }
 4385 
 4386 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4387   assert(VM_Version::supports_avx512vlbw(), "");
 4388   InstructionMark im(this);
 4389   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4390   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4391   attributes.reset_is_clear_context();
 4392   attributes.set_embedded_opmask_register_specifier(mask);
 4393   attributes.set_is_evex_instruction();
 4394   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4395   emit_int8(0x74);
 4396   emit_operand(as_Register(kdst->encoding()), src);
 4397 }
 4398 
 4399 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4400 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4401   assert(VM_Version::supports_sse2(), "");
 4402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4403   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4404   emit_int16(0x75, (0xC0 | encode));
 4405 }
 4406 
 4407 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4408 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4409   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4410   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4411   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4412   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4413   emit_int16(0x75, (0xC0 | encode));
 4414 }
 4415 
 4416 // In this context, kdst is written the mask used to process the equal components
 4417 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4418   assert(VM_Version::supports_avx512bw(), "");
 4419   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4420   attributes.set_is_evex_instruction();
 4421   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4422   emit_int16(0x75, (0xC0 | encode));
 4423 }
 4424 
 4425 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4426   assert(VM_Version::supports_avx512bw(), "");
 4427   InstructionMark im(this);
 4428   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4429   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4430   attributes.set_is_evex_instruction();
 4431   int dst_enc = kdst->encoding();
 4432   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4433   emit_int8(0x75);
 4434   emit_operand(as_Register(dst_enc), src);
 4435 }
 4436 
 4437 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4438 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4439   assert(VM_Version::supports_sse2(), "");
 4440   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4441   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4442   emit_int16(0x76, (0xC0 | encode));
 4443 }
 4444 
 4445 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4446 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4447   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4448   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4449   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4450   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4451   emit_int16(0x76, (0xC0 | encode));
 4452 }
 4453 
 4454 // In this context, kdst is written the mask used to process the equal components
 4455 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4456   assert(VM_Version::supports_evex(), "");
 4457   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4458   attributes.set_is_evex_instruction();
 4459   attributes.reset_is_clear_context();
 4460   attributes.set_embedded_opmask_register_specifier(mask);
 4461   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4462   emit_int16(0x76, (0xC0 | encode));
 4463 }
 4464 
 4465 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4466   assert(VM_Version::supports_evex(), "");
 4467   InstructionMark im(this);
 4468   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4469   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4470   attributes.set_is_evex_instruction();
 4471   attributes.reset_is_clear_context();
 4472   attributes.set_embedded_opmask_register_specifier(mask);
 4473   int dst_enc = kdst->encoding();
 4474   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4475   emit_int8(0x76);
 4476   emit_operand(as_Register(dst_enc), src);
 4477 }
 4478 
 4479 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4480 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4481   assert(VM_Version::supports_sse4_1(), "");
 4482   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4483   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4484   emit_int16(0x29, (0xC0 | encode));
 4485 }
 4486 
 4487 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4488   assert(VM_Version::supports_avx(), "");
 4489   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4490   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4491   emit_int16(cond_encoding, (0xC0 | encode));
 4492 }
 4493 
 4494 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4495 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4496   assert(VM_Version::supports_avx(), "");
 4497   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4498   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4499   emit_int16(0x29, (0xC0 | encode));
 4500 }
 4501 
 4502 // In this context, kdst is written the mask used to process the equal components
 4503 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4504   assert(VM_Version::supports_evex(), "");
 4505   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4506   attributes.reset_is_clear_context();
 4507   attributes.set_is_evex_instruction();
 4508   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4509   emit_int16(0x29, (0xC0 | encode));
 4510 }
 4511 
 4512 // In this context, kdst is written the mask used to process the equal components
 4513 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4514   assert(VM_Version::supports_evex(), "");
 4515   InstructionMark im(this);
 4516   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4517   attributes.reset_is_clear_context();
 4518   attributes.set_is_evex_instruction();
 4519   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4520   int dst_enc = kdst->encoding();
 4521   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4522   emit_int8(0x29);
 4523   emit_operand(as_Register(dst_enc), src);
 4524 }
 4525 
 4526 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4527   assert(VM_Version::supports_sse4_1(), "");
 4528   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4529   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4530   emit_int16(0x37, (0xC0 | encode));
 4531 }
 4532 
 4533 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4534   assert(VM_Version::supports_sse2(), "");
 4535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4536   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4537   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4538 }
 4539 
 4540 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4541   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4542          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4543   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4544   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4545   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4546 }
 4547 
 4548 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4549   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 4550   InstructionMark im(this);
 4551   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 4552   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4553   emit_int8((unsigned char)0x8C);
 4554   emit_operand(dst, src);
 4555 }
 4556 
 4557 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4558   assert(VM_Version::supports_sse4_1(), "");
 4559   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4560   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4561   emit_int24(0x16, (0xC0 | encode), imm8);
 4562 }
 4563 
 4564 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4565   assert(VM_Version::supports_sse4_1(), "");
 4566   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4567   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4568   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4569   emit_int8(0x16);
 4570   emit_operand(src, dst);
 4571   emit_int8(imm8);
 4572 }
 4573 
 4574 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4575   assert(VM_Version::supports_sse4_1(), "");
 4576   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4577   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4578   emit_int24(0x16, (0xC0 | encode), imm8);
 4579 }
 4580 
 4581 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4582   assert(VM_Version::supports_sse4_1(), "");
 4583   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4584   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4585   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4586   emit_int8(0x16);
 4587   emit_operand(src, dst);
 4588   emit_int8(imm8);
 4589 }
 4590 
 4591 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4592   assert(VM_Version::supports_sse2(), "");
 4593   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4594   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4595   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4596 }
 4597 
 4598 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4599   assert(VM_Version::supports_sse4_1(), "");
 4600   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4601   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4602   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4603   emit_int8(0x15);
 4604   emit_operand(src, dst);
 4605   emit_int8(imm8);
 4606 }
 4607 
 4608 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4609   assert(VM_Version::supports_sse4_1(), "");
 4610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4611   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4612   emit_int24(0x14, (0xC0 | encode), imm8);
 4613 }
 4614 
 4615 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4616   assert(VM_Version::supports_sse4_1(), "");
 4617   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4618   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4619   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4620   emit_int8(0x14);
 4621   emit_operand(src, dst);
 4622   emit_int8(imm8);
 4623 }
 4624 
 4625 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4626   assert(VM_Version::supports_sse4_1(), "");
 4627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4628   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4629   emit_int24(0x22, (0xC0 | encode), imm8);
 4630 }
 4631 
 4632 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4633   assert(VM_Version::supports_sse4_1(), "");
 4634   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4635   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4636   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4637   emit_int8(0x22);
 4638   emit_operand(dst,src);
 4639   emit_int8(imm8);
 4640 }
 4641 
 4642 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4643   assert(VM_Version::supports_avx(), "");
 4644   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4645   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4646   emit_int24(0x22, (0xC0 | encode), imm8);
 4647 }
 4648 
 4649 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4650   assert(VM_Version::supports_sse4_1(), "");
 4651   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4652   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4653   emit_int24(0x22, (0xC0 | encode), imm8);
 4654 }
 4655 
 4656 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4657   assert(VM_Version::supports_sse4_1(), "");
 4658   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4659   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4660   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4661   emit_int8(0x22);
 4662   emit_operand(dst, src);
 4663   emit_int8(imm8);
 4664 }
 4665 
 4666 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4667   assert(VM_Version::supports_avx(), "");
 4668   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4670   emit_int24(0x22, (0xC0 | encode), imm8);
 4671 }
 4672 
 4673 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4674   assert(VM_Version::supports_sse2(), "");
 4675   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4676   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4677   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4678 }
 4679 
 4680 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4681   assert(VM_Version::supports_sse2(), "");
 4682   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4683   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4684   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4685   emit_int8((unsigned char)0xC4);
 4686   emit_operand(dst, src);
 4687   emit_int8(imm8);
 4688 }
 4689 
 4690 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4691   assert(VM_Version::supports_avx(), "");
 4692   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4693   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4694   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4695 }
 4696 
 4697 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4698   assert(VM_Version::supports_sse4_1(), "");
 4699   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4700   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4701   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4702   emit_int8(0x20);
 4703   emit_operand(dst, src);
 4704   emit_int8(imm8);
 4705 }
 4706 
 4707 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4708   assert(VM_Version::supports_sse4_1(), "");
 4709   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4710   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4711   emit_int24(0x20, (0xC0 | encode), imm8);
 4712 }
 4713 
 4714 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4715   assert(VM_Version::supports_avx(), "");
 4716   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4717   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4718   emit_int24(0x20, (0xC0 | encode), imm8);
 4719 }
 4720 
 4721 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4722   assert(VM_Version::supports_sse4_1(), "");
 4723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4724   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4725   emit_int24(0x21, (0xC0 | encode), imm8);
 4726 }
 4727 
 4728 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4729   assert(VM_Version::supports_avx(), "");
 4730   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4731   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4732   emit_int24(0x21, (0xC0 | encode), imm8);
 4733 }
 4734 
 4735 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4736   assert(VM_Version::supports_sse4_1(), "");
 4737   InstructionMark im(this);
 4738   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4739   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4740   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4741   emit_int8(0x30);
 4742   emit_operand(dst, src);
 4743 }
 4744 
 4745 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4746   assert(VM_Version::supports_sse4_1(), "");
 4747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4748   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4749   emit_int16(0x30, (0xC0 | encode));
 4750 }
 4751 
 4752 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4753   assert(VM_Version::supports_sse4_1(), "");
 4754   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4755   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4756   emit_int16(0x20, (0xC0 | encode));
 4757 }
 4758 
 4759 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4760   assert(VM_Version::supports_sse4_1(), "");
 4761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4762   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4763   emit_int16(0x35, (0xC0 | encode));
 4764 }
 4765 
 4766 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4767   assert(VM_Version::supports_sse4_1(), "");
 4768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4769   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4770   emit_int16(0x21, (0xC0 | encode));
 4771 }
 4772 
 4773 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4774   assert(VM_Version::supports_sse4_1(), "");
 4775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4776   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4777   emit_int16(0x31, (0xC0 | encode));
 4778 }
 4779 
 4780 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4781   assert(VM_Version::supports_sse4_1(), "");
 4782   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4783   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4784   emit_int16(0x22, (0xC0 | encode));
 4785 }
 4786 
 4787 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4788   assert(VM_Version::supports_sse4_1(), "");
 4789   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4790   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4791   emit_int16(0x23, (0xC0 | encode));
 4792 }
 4793 
 4794 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4795   assert(VM_Version::supports_avx(), "");
 4796   InstructionMark im(this);
 4797   assert(dst != xnoreg, "sanity");
 4798   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4799   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4800   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4801   emit_int8(0x30);
 4802   emit_operand(dst, src);
 4803 }
 4804 
 4805 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4806   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4807   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4808   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4809   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4810   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4811   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4812 }
 4813 
 4814 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4815   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4816   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4817   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4818   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4819   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4820   emit_int16(0x20, (0xC0 | encode));
 4821 }
 4822 
 4823 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4824   assert(VM_Version::supports_avx512vlbw(), "");
 4825   assert(dst != xnoreg, "sanity");
 4826   InstructionMark im(this);
 4827   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4828   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4829   attributes.set_embedded_opmask_register_specifier(mask);
 4830   attributes.set_is_evex_instruction();
 4831   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4832   emit_int8(0x30);
 4833   emit_operand(dst, src);
 4834 }
 4835 
 4836 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4837   assert(VM_Version::supports_evex(), "");
 4838   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4839   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4840   attributes.set_is_evex_instruction();
 4841   attributes.set_embedded_opmask_register_specifier(mask);
 4842   if (merge) {
 4843     attributes.reset_is_clear_context();
 4844   }
 4845   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4846   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4847 }
 4848 
 4849 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4850   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4851   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4852   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4853   emit_int16(0x35, (0xC0 | encode));
 4854 }
 4855 
 4856 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4857   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4858   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4859   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4860   emit_int16(0x31, (0xC0 | encode));
 4861 }
 4862 
 4863 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4864   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4865   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4866   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4867   emit_int16(0x32, (0xC0 | encode));
 4868 }
 4869 
 4870 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4871   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4872          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4873              VM_Version::supports_evex(), "");
 4874   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4875   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4876   emit_int16(0x21, (0xC0 | encode));
 4877 }
 4878 
 4879 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4880   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4881          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4882              VM_Version::supports_evex(), "");
 4883   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4884   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4885   emit_int16(0x22, (0xC0 | encode));
 4886 }
 4887 
 4888 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4889   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4890          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4891              VM_Version::supports_evex(), "");
 4892   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4893   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4894   emit_int16(0x23, (0xC0 | encode));
 4895 }
 4896 
 4897 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4898   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4899          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4900              VM_Version::supports_evex(), "");
 4901   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4902   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4903   emit_int16(0x24, (0xC0 | encode));
 4904 }
 4905 
 4906 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4907   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4908          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4909              VM_Version::supports_evex(), "");
 4910   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4911   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4912   emit_int16(0x25, (0xC0 | encode));
 4913 }
 4914 
 4915 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4916   assert(VM_Version::supports_avx512vlbw(), "");
 4917   assert(src != xnoreg, "sanity");
 4918   InstructionMark im(this);
 4919   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4920   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4921   attributes.set_is_evex_instruction();
 4922   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4923   emit_int8(0x30);
 4924   emit_operand(src, dst);
 4925 }
 4926 
 4927 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4928   assert(VM_Version::supports_avx512vlbw(), "");
 4929   assert(src != xnoreg, "sanity");
 4930   InstructionMark im(this);
 4931   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4932   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4933   attributes.reset_is_clear_context();
 4934   attributes.set_embedded_opmask_register_specifier(mask);
 4935   attributes.set_is_evex_instruction();
 4936   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4937   emit_int8(0x30);
 4938   emit_operand(src, dst);
 4939 }
 4940 
 4941 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4942   assert(VM_Version::supports_evex(), "");
 4943   assert(src != xnoreg, "sanity");
 4944   InstructionMark im(this);
 4945   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4946   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4947   attributes.set_is_evex_instruction();
 4948   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4949   emit_int8(0x31);
 4950   emit_operand(src, dst);
 4951 }
 4952 
 4953 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4954   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4955   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4956   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4957   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4958   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4959   emit_int16(0x33, (0xC0 | encode));
 4960 }
 4961 
 4962 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4963   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4964   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4965   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4966   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4967 }
 4968 
 4969 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4970   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4971     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4972     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4973   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4974   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4975   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4976 }
 4977 
 4978 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4979 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4980        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4981        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4982   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4983   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4984   emit_int16(0x04, (0xC0 | encode));
 4985 }
 4986 
 4987 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4988   assert(VM_Version::supports_evex(), "");
 4989   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4990   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4991   attributes.set_is_evex_instruction();
 4992   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4993   emit_int16(0x52, (0xC0 | encode));
 4994 }
 4995 
 4996 // generic
 4997 void Assembler::pop(Register dst) {
 4998   int encode = prefix_and_encode(dst->encoding());
 4999   emit_int8(0x58 | encode);
 5000 }
 5001 
 5002 void Assembler::popcntl(Register dst, Address src) {
 5003   assert(VM_Version::supports_popcnt(), "must support");
 5004   InstructionMark im(this);
 5005   emit_int8((unsigned char)0xF3);
 5006   prefix(src, dst);
 5007   emit_int16(0x0F, (unsigned char)0xB8);
 5008   emit_operand(dst, src);
 5009 }
 5010 
 5011 void Assembler::popcntl(Register dst, Register src) {
 5012   assert(VM_Version::supports_popcnt(), "must support");
 5013   emit_int8((unsigned char)0xF3);
 5014   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5015   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 5016 }
 5017 
 5018 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
 5019   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5020   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5021   attributes.set_is_evex_instruction();
 5022   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5023   emit_int16(0x55, (0xC0 | encode));
 5024 }
 5025 
 5026 void Assembler::popf() {
 5027   emit_int8((unsigned char)0x9D);
 5028 }
 5029 
 5030 #ifndef _LP64 // no 32bit push/pop on amd64
 5031 void Assembler::popl(Address dst) {
 5032   // NOTE: this will adjust stack by 8byte on 64bits
 5033   InstructionMark im(this);
 5034   prefix(dst);
 5035   emit_int8((unsigned char)0x8F);
 5036   emit_operand(rax, dst);
 5037 }
 5038 #endif
 5039 
 5040 void Assembler::prefetchnta(Address src) {
 5041   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5042   InstructionMark im(this);
 5043   prefix(src);
 5044   emit_int16(0x0F, 0x18);
 5045   emit_operand(rax, src); // 0, src
 5046 }
 5047 
 5048 void Assembler::prefetchr(Address src) {
 5049   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5050   InstructionMark im(this);
 5051   prefix(src);
 5052   emit_int16(0x0F, 0x0D);
 5053   emit_operand(rax, src); // 0, src
 5054 }
 5055 
 5056 void Assembler::prefetcht0(Address src) {
 5057   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5058   InstructionMark im(this);
 5059   prefix(src);
 5060   emit_int16(0x0F, 0x18);
 5061   emit_operand(rcx, src); // 1, src
 5062 }
 5063 
 5064 void Assembler::prefetcht1(Address src) {
 5065   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5066   InstructionMark im(this);
 5067   prefix(src);
 5068   emit_int16(0x0F, 0x18);
 5069   emit_operand(rdx, src); // 2, src
 5070 }
 5071 
 5072 void Assembler::prefetcht2(Address src) {
 5073   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5074   InstructionMark im(this);
 5075   prefix(src);
 5076   emit_int16(0x0F, 0x18);
 5077   emit_operand(rbx, src); // 3, src
 5078 }
 5079 
 5080 void Assembler::prefetchw(Address src) {
 5081   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5082   InstructionMark im(this);
 5083   prefix(src);
 5084   emit_int16(0x0F, 0x0D);
 5085   emit_operand(rcx, src); // 1, src
 5086 }
 5087 
 5088 void Assembler::prefix(Prefix p) {
 5089   emit_int8(p);
 5090 }
 5091 
 5092 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5093   assert(VM_Version::supports_ssse3(), "");
 5094   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5095   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5096   emit_int16(0x00, (0xC0 | encode));
 5097 }
 5098 
 5099 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5100   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5101          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5102          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5103   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5104   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5105   emit_int16(0x00, (0xC0 | encode));
 5106 }
 5107 
 5108 void Assembler::pshufb(XMMRegister dst, Address src) {
 5109   assert(VM_Version::supports_ssse3(), "");
 5110   InstructionMark im(this);
 5111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5112   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5113   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5114   emit_int8(0x00);
 5115   emit_operand(dst, src);
 5116 }
 5117 
 5118 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5119   assert(isByte(mode), "invalid value");
 5120   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5121   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5122   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5123   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5124   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5125 }
 5126 
 5127 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5128   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5129          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5130          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5131   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5132   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5133   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5134   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5135 }
 5136 
 5137 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5138   assert(isByte(mode), "invalid value");
 5139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5140   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5141   InstructionMark im(this);
 5142   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5143   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5144   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5145   emit_int8(0x70);
 5146   emit_operand(dst, src);
 5147   emit_int8(mode & 0xFF);
 5148 }
 5149 
 5150 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5151   assert(isByte(mode), "invalid value");
 5152   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5153   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5154   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5155   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5156 }
 5157 
 5158 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5159   assert(isByte(mode), "invalid value");
 5160   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5161   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5162   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5163   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5164 }
 5165 
 5166 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5167   assert(isByte(mode), "invalid value");
 5168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5169   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5170   InstructionMark im(this);
 5171   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5172   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5173   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5174   emit_int8(0x70);
 5175   emit_operand(dst, src);
 5176   emit_int8(mode & 0xFF);
 5177 }
 5178 
 5179 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5180   assert(VM_Version::supports_evex(), "requires EVEX support");
 5181   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5182   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5183   attributes.set_is_evex_instruction();
 5184   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5185   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5186 }
 5187 
 5188 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5189   assert(isByte(imm8), "invalid value");
 5190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5192   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5193   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5194 }
 5195 
 5196 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5197   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5198   attributes.set_rex_vex_w_reverted();
 5199   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5200   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5201 }
 5202 
 5203 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 5204   assert(isByte(imm8), "invalid value");
 5205   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5206   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5207   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5208   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5209 }
 5210 
 5211 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5212   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5213   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5214   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5215 }
 5216 
 5217 void Assembler::psrldq(XMMRegister dst, int shift) {
 5218   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5219   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5220   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5221   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5222   emit_int24(0x73, (0xC0 | encode), shift);
 5223 }
 5224 
 5225 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5226   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5227          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5228          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5229   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5230   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5231   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5232 }
 5233 
 5234 void Assembler::pslldq(XMMRegister dst, int shift) {
 5235   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5236   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5237   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5238   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5239   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5240   emit_int24(0x73, (0xC0 | encode), shift);
 5241 }
 5242 
 5243 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5244   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5245          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5246          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5247   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5248   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5249   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5250 }
 5251 
 5252 void Assembler::ptest(XMMRegister dst, Address src) {
 5253   assert(VM_Version::supports_sse4_1(), "");
 5254   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5255   InstructionMark im(this);
 5256   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5257   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5258   emit_int8(0x17);
 5259   emit_operand(dst, src);
 5260 }
 5261 
 5262 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5263   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5264   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5265   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5266   emit_int8(0x17);
 5267   emit_int8((0xC0 | encode));
 5268 }
 5269 
 5270 void Assembler::vptest(XMMRegister dst, Address src) {
 5271   assert(VM_Version::supports_avx(), "");
 5272   InstructionMark im(this);
 5273   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5274   assert(dst != xnoreg, "sanity");
 5275   // swap src<->dst for encoding
 5276   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5277   emit_int8(0x17);
 5278   emit_operand(dst, src);
 5279 }
 5280 
 5281 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5282   assert(VM_Version::supports_avx(), "");
 5283   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5284   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5285   emit_int16(0x17, (0xC0 | encode));
 5286 }
 5287 
 5288 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5289   assert(VM_Version::supports_avx(), "");
 5290   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5291   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5292   emit_int16(0x17, (0xC0 | encode));
 5293 }
 5294 
 5295 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5296   assert(VM_Version::supports_avx512vlbw(), "");
 5297   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5299   attributes.set_is_evex_instruction();
 5300   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5301   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5302 }
 5303 
 5304 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5305   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5306   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5307   InstructionMark im(this);
 5308   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5309   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5310   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5311   emit_int8(0x60);
 5312   emit_operand(dst, src);
 5313 }
 5314 
 5315 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5318   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5319   emit_int16(0x60, (0xC0 | encode));
 5320 }
 5321 
 5322 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5323   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5324   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5325   InstructionMark im(this);
 5326   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5327   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5328   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5329   emit_int8(0x62);
 5330   emit_operand(dst, src);
 5331 }
 5332 
 5333 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5337   emit_int16(0x62, (0xC0 | encode));
 5338 }
 5339 
 5340 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5341   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5342   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5343   attributes.set_rex_vex_w_reverted();
 5344   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5345   emit_int16(0x6C, (0xC0 | encode));
 5346 }
 5347 
 5348 void Assembler::push(int32_t imm32) {
 5349   // in 64bits we push 64bits onto the stack but only
 5350   // take a 32bit immediate
 5351   emit_int8(0x68);
 5352   emit_int32(imm32);
 5353 }
 5354 
 5355 void Assembler::push(Register src) {
 5356   int encode = prefix_and_encode(src->encoding());
 5357   emit_int8(0x50 | encode);
 5358 }
 5359 
 5360 void Assembler::pushf() {
 5361   emit_int8((unsigned char)0x9C);
 5362 }
 5363 
 5364 #ifndef _LP64 // no 32bit push/pop on amd64
 5365 void Assembler::pushl(Address src) {
 5366   // Note this will push 64bit on 64bit
 5367   InstructionMark im(this);
 5368   prefix(src);
 5369   emit_int8((unsigned char)0xFF);
 5370   emit_operand(rsi, src);
 5371 }
 5372 #endif
 5373 
 5374 void Assembler::rcll(Register dst, int imm8) {
 5375   assert(isShiftCount(imm8), "illegal shift count");
 5376   int encode = prefix_and_encode(dst->encoding());
 5377   if (imm8 == 1) {
 5378     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5379   } else {
 5380     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5381   }
 5382 }
 5383 
 5384 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5385   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5387   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5388   emit_int16(0x53, (0xC0 | encode));
 5389 }
 5390 
 5391 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5392   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5393   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5394   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5395   emit_int16(0x53, (0xC0 | encode));
 5396 }
 5397 
 5398 void Assembler::rdtsc() {
 5399   emit_int16(0x0F, 0x31);
 5400 }
 5401 
 5402 // copies data from [esi] to [edi] using rcx pointer sized words
 5403 // generic
 5404 void Assembler::rep_mov() {
 5405   // REP
 5406   // MOVSQ
 5407   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5408   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5409 }
 5410 
 5411 // sets rcx bytes with rax, value at [edi]
 5412 void Assembler::rep_stosb() {
 5413   // REP
 5414   // STOSB
 5415   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5416   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5417 }
 5418 
 5419 // sets rcx pointer sized words with rax, value at [edi]
 5420 // generic
 5421 void Assembler::rep_stos() {
 5422   // REP
 5423   // LP64:STOSQ, LP32:STOSD
 5424   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5425   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5426 }
 5427 
 5428 // scans rcx pointer sized words at [edi] for occurance of rax,
 5429 // generic
 5430 void Assembler::repne_scan() { // repne_scan
 5431   // SCASQ
 5432   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5433   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5434 }
 5435 
 5436 #ifdef _LP64
 5437 // scans rcx 4 byte words at [edi] for occurance of rax,
 5438 // generic
 5439 void Assembler::repne_scanl() { // repne_scan
 5440   // SCASL
 5441   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5442 }
 5443 #endif
 5444 
 5445 void Assembler::ret(int imm16) {
 5446   if (imm16 == 0) {
 5447     emit_int8((unsigned char)0xC3);
 5448   } else {
 5449     emit_int8((unsigned char)0xC2);
 5450     emit_int16(imm16);
 5451   }
 5452 }
 5453 
 5454 void Assembler::roll(Register dst, int imm8) {
 5455   assert(isShiftCount(imm8), "illegal shift count");
 5456   int encode = prefix_and_encode(dst->encoding());
 5457   if (imm8 == 1) {
 5458     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5459   } else {
 5460     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5461   }
 5462 }
 5463 
 5464 void Assembler::roll(Register dst) {
 5465   int encode = prefix_and_encode(dst->encoding());
 5466   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5467 }
 5468 
 5469 void Assembler::rorl(Register dst, int imm8) {
 5470   assert(isShiftCount(imm8), "illegal shift count");
 5471   int encode = prefix_and_encode(dst->encoding());
 5472   if (imm8 == 1) {
 5473     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5474   } else {
 5475     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5476   }
 5477 }
 5478 
 5479 void Assembler::rorl(Register dst) {
 5480   int encode = prefix_and_encode(dst->encoding());
 5481   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5482 }
 5483 
 5484 #ifdef _LP64
 5485 void Assembler::rorq(Register dst) {
 5486   int encode = prefixq_and_encode(dst->encoding());
 5487   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5488 }
 5489 
 5490 void Assembler::rorq(Register dst, int imm8) {
 5491   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5492   int encode = prefixq_and_encode(dst->encoding());
 5493   if (imm8 == 1) {
 5494     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5495   } else {
 5496     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5497   }
 5498 }
 5499 
 5500 void Assembler::rolq(Register dst) {
 5501   int encode = prefixq_and_encode(dst->encoding());
 5502   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5503 }
 5504 
 5505 void Assembler::rolq(Register dst, int imm8) {
 5506   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5507   int encode = prefixq_and_encode(dst->encoding());
 5508   if (imm8 == 1) {
 5509     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5510   } else {
 5511     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5512   }
 5513 }
 5514 #endif
 5515 
 5516 void Assembler::sahf() {
 5517 #ifdef _LP64
 5518   // Not supported in 64bit mode
 5519   ShouldNotReachHere();
 5520 #endif
 5521   emit_int8((unsigned char)0x9E);
 5522 }
 5523 
 5524 void Assembler::sall(Address dst, int imm8) {
 5525   InstructionMark im(this);
 5526   assert(isShiftCount(imm8), "illegal shift count");
 5527   prefix(dst);
 5528   if (imm8 == 1) {
 5529     emit_int8((unsigned char)0xD1);
 5530     emit_operand(as_Register(4), dst);
 5531   }
 5532   else {
 5533     emit_int8((unsigned char)0xC1);
 5534     emit_operand(as_Register(4), dst);
 5535     emit_int8(imm8);
 5536   }
 5537 }
 5538 
 5539 void Assembler::sall(Address dst) {
 5540   InstructionMark im(this);
 5541   prefix(dst);
 5542   emit_int8((unsigned char)0xD3);
 5543   emit_operand(as_Register(4), dst);
 5544 }
 5545 
 5546 void Assembler::sall(Register dst, int imm8) {
 5547   assert(isShiftCount(imm8), "illegal shift count");
 5548   int encode = prefix_and_encode(dst->encoding());
 5549   if (imm8 == 1) {
 5550     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5551   } else {
 5552     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5553   }
 5554 }
 5555 
 5556 void Assembler::sall(Register dst) {
 5557   int encode = prefix_and_encode(dst->encoding());
 5558   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5559 }
 5560 
 5561 void Assembler::sarl(Address dst, int imm8) {
 5562   assert(isShiftCount(imm8), "illegal shift count");
 5563   InstructionMark im(this);
 5564   prefix(dst);
 5565   if (imm8 == 1) {
 5566     emit_int8((unsigned char)0xD1);
 5567     emit_operand(as_Register(7), dst);
 5568   }
 5569   else {
 5570     emit_int8((unsigned char)0xC1);
 5571     emit_operand(as_Register(7), dst);
 5572     emit_int8(imm8);
 5573   }
 5574 }
 5575 
 5576 void Assembler::sarl(Address dst) {
 5577   InstructionMark im(this);
 5578   prefix(dst);
 5579   emit_int8((unsigned char)0xD3);
 5580   emit_operand(as_Register(7), dst);
 5581 }
 5582 
 5583 void Assembler::sarl(Register dst, int imm8) {
 5584   int encode = prefix_and_encode(dst->encoding());
 5585   assert(isShiftCount(imm8), "illegal shift count");
 5586   if (imm8 == 1) {
 5587     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5588   } else {
 5589     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5590   }
 5591 }
 5592 
 5593 void Assembler::sarl(Register dst) {
 5594   int encode = prefix_and_encode(dst->encoding());
 5595   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5596 }
 5597 
 5598 void Assembler::sbbl(Address dst, int32_t imm32) {
 5599   InstructionMark im(this);
 5600   prefix(dst);
 5601   emit_arith_operand(0x81, rbx, dst, imm32);
 5602 }
 5603 
 5604 void Assembler::sbbl(Register dst, int32_t imm32) {
 5605   prefix(dst);
 5606   emit_arith(0x81, 0xD8, dst, imm32);
 5607 }
 5608 
 5609 
 5610 void Assembler::sbbl(Register dst, Address src) {
 5611   InstructionMark im(this);
 5612   prefix(src, dst);
 5613   emit_int8(0x1B);
 5614   emit_operand(dst, src);
 5615 }
 5616 
 5617 void Assembler::sbbl(Register dst, Register src) {
 5618   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5619   emit_arith(0x1B, 0xC0, dst, src);
 5620 }
 5621 
 5622 void Assembler::setb(Condition cc, Register dst) {
 5623   assert(0 <= cc && cc < 16, "illegal cc");
 5624   int encode = prefix_and_encode(dst->encoding(), true);
 5625   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5626 }
 5627 
 5628 void Assembler::sete(Register dst) {
 5629   int encode = prefix_and_encode(dst->encoding(), true);
 5630   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5631 }
 5632 
 5633 void Assembler::setl(Register dst) {
 5634   int encode = prefix_and_encode(dst->encoding(), true);
 5635   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5636 }
 5637 
 5638 void Assembler::setne(Register dst) {
 5639   int encode = prefix_and_encode(dst->encoding(), true);
 5640   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5641 }
 5642 
 5643 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5644   assert(VM_Version::supports_ssse3(), "");
 5645   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5646   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5647   emit_int24(0x0F, (0xC0 | encode), imm8);
 5648 }
 5649 
 5650 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5651   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5652          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5653          0, "");
 5654   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5655   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5656   emit_int24(0x0F, (0xC0 | encode), imm8);
 5657 }
 5658 
 5659 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5660   assert(VM_Version::supports_evex(), "");
 5661   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5662   attributes.set_is_evex_instruction();
 5663   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5664   emit_int24(0x3, (0xC0 | encode), imm8);
 5665 }
 5666 
 5667 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5668   assert(VM_Version::supports_sse4_1(), "");
 5669   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5670   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5671   emit_int24(0x0E, (0xC0 | encode), imm8);
 5672 }
 5673 
 5674 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5675   assert(VM_Version::supports_sha(), "");
 5676   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5677   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5678 }
 5679 
 5680 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5681   assert(VM_Version::supports_sha(), "");
 5682   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5683   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5684 }
 5685 
 5686 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5687   assert(VM_Version::supports_sha(), "");
 5688   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5689   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5690 }
 5691 
 5692 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5693   assert(VM_Version::supports_sha(), "");
 5694   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5695   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5696 }
 5697 
 5698 // xmm0 is implicit additional source to this instruction.
 5699 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5700   assert(VM_Version::supports_sha(), "");
 5701   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5702   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5703 }
 5704 
 5705 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5706   assert(VM_Version::supports_sha(), "");
 5707   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5708   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5709 }
 5710 
 5711 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5712   assert(VM_Version::supports_sha(), "");
 5713   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5714   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5715 }
 5716 
 5717 
 5718 void Assembler::shll(Register dst, int imm8) {
 5719   assert(isShiftCount(imm8), "illegal shift count");
 5720   int encode = prefix_and_encode(dst->encoding());
 5721   if (imm8 == 1 ) {
 5722     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5723   } else {
 5724     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5725   }
 5726 }
 5727 
 5728 void Assembler::shll(Register dst) {
 5729   int encode = prefix_and_encode(dst->encoding());
 5730   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5731 }
 5732 
 5733 void Assembler::shrl(Register dst, int imm8) {
 5734   assert(isShiftCount(imm8), "illegal shift count");
 5735   int encode = prefix_and_encode(dst->encoding());
 5736   if (imm8 == 1) {
 5737     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5738   }
 5739   else {
 5740     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5741   }
 5742 }
 5743 
 5744 void Assembler::shrl(Register dst) {
 5745   int encode = prefix_and_encode(dst->encoding());
 5746   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5747 }
 5748 
 5749 void Assembler::shrl(Address dst) {
 5750   InstructionMark im(this);
 5751   prefix(dst);
 5752   emit_int8((unsigned char)0xD3);
 5753   emit_operand(as_Register(5), dst);
 5754 }
 5755 
 5756 void Assembler::shrl(Address dst, int imm8) {
 5757   InstructionMark im(this);
 5758   assert(isShiftCount(imm8), "illegal shift count");
 5759   prefix(dst);
 5760   if (imm8 == 1) {
 5761     emit_int8((unsigned char)0xD1);
 5762     emit_operand(as_Register(5), dst);
 5763   }
 5764   else {
 5765     emit_int8((unsigned char)0xC1);
 5766     emit_operand(as_Register(5), dst);
 5767     emit_int8(imm8);
 5768   }
 5769 }
 5770 
 5771 
 5772 void Assembler::shldl(Register dst, Register src) {
 5773   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5774   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5775 }
 5776 
 5777 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5778   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5779   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5780 }
 5781 
 5782 void Assembler::shrdl(Register dst, Register src) {
 5783   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5784   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5785 }
 5786 
 5787 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5788   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5789   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5790 }
 5791 
 5792 // copies a single word from [esi] to [edi]
 5793 void Assembler::smovl() {
 5794   emit_int8((unsigned char)0xA5);
 5795 }
 5796 
 5797 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5798   assert(VM_Version::supports_sse4_1(), "");
 5799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5800   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5801   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5802 }
 5803 
 5804 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5805   assert(VM_Version::supports_sse4_1(), "");
 5806   InstructionMark im(this);
 5807   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5808   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5809   emit_int8(0x0B);
 5810   emit_operand(dst, src);
 5811   emit_int8((unsigned char)rmode);
 5812 }
 5813 
 5814 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5815   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5816   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5817   attributes.set_rex_vex_w_reverted();
 5818   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5819   emit_int16(0x51, (0xC0 | encode));
 5820 }
 5821 
 5822 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5823   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5824   InstructionMark im(this);
 5825   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5826   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5827   attributes.set_rex_vex_w_reverted();
 5828   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5829   emit_int8(0x51);
 5830   emit_operand(dst, src);
 5831 }
 5832 
 5833 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5834   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5835   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5836   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5837   emit_int16(0x51, (0xC0 | encode));
 5838 }
 5839 
 5840 void Assembler::std() {
 5841   emit_int8((unsigned char)0xFD);
 5842 }
 5843 
 5844 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5845   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5846   InstructionMark im(this);
 5847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5848   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5849   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5850   emit_int8(0x51);
 5851   emit_operand(dst, src);
 5852 }
 5853 
 5854 void Assembler::stmxcsr( Address dst) {
 5855   if (UseAVX > 0 ) {
 5856     assert(VM_Version::supports_avx(), "");
 5857     InstructionMark im(this);
 5858     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5859     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5860     emit_int8((unsigned char)0xAE);
 5861     emit_operand(as_Register(3), dst);
 5862   } else {
 5863     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5864     InstructionMark im(this);
 5865     prefix(dst);
 5866     emit_int16(0x0F, (unsigned char)0xAE);
 5867     emit_operand(as_Register(3), dst);
 5868   }
 5869 }
 5870 
 5871 void Assembler::subl(Address dst, int32_t imm32) {
 5872   InstructionMark im(this);
 5873   prefix(dst);
 5874   emit_arith_operand(0x81, rbp, dst, imm32);
 5875 }
 5876 
 5877 void Assembler::subl(Address dst, Register src) {
 5878   InstructionMark im(this);
 5879   prefix(dst, src);
 5880   emit_int8(0x29);
 5881   emit_operand(src, dst);
 5882 }
 5883 
 5884 void Assembler::subl(Register dst, int32_t imm32) {
 5885   prefix(dst);
 5886   emit_arith(0x81, 0xE8, dst, imm32);
 5887 }
 5888 
 5889 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5890 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5891   prefix(dst);
 5892   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5893 }
 5894 
 5895 void Assembler::subl(Register dst, Address src) {
 5896   InstructionMark im(this);
 5897   prefix(src, dst);
 5898   emit_int8(0x2B);
 5899   emit_operand(dst, src);
 5900 }
 5901 
 5902 void Assembler::subl(Register dst, Register src) {
 5903   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5904   emit_arith(0x2B, 0xC0, dst, src);
 5905 }
 5906 
 5907 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5908   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5909   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5910   attributes.set_rex_vex_w_reverted();
 5911   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5912   emit_int16(0x5C, (0xC0 | encode));
 5913 }
 5914 
 5915 void Assembler::subsd(XMMRegister dst, Address src) {
 5916   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5917   InstructionMark im(this);
 5918   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5919   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5920   attributes.set_rex_vex_w_reverted();
 5921   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5922   emit_int8(0x5C);
 5923   emit_operand(dst, src);
 5924 }
 5925 
 5926 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5927   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5928   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5929   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5930   emit_int16(0x5C, (0xC0 | encode));
 5931 }
 5932 
 5933 void Assembler::subss(XMMRegister dst, Address src) {
 5934   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5935   InstructionMark im(this);
 5936   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5937   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5938   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5939   emit_int8(0x5C);
 5940   emit_operand(dst, src);
 5941 }
 5942 
 5943 void Assembler::testb(Register dst, int imm8) {
 5944   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5945   (void) prefix_and_encode(dst->encoding(), true);
 5946   emit_arith_b(0xF6, 0xC0, dst, imm8);
 5947 }
 5948 
 5949 void Assembler::testb(Address dst, int imm8) {
 5950   InstructionMark im(this);
 5951   prefix(dst);
 5952   emit_int8((unsigned char)0xF6);
 5953   emit_operand(rax, dst, 1);
 5954   emit_int8(imm8);
 5955 }
 5956 
 5957 void Assembler::testl(Register dst, int32_t imm32) {
 5958   // not using emit_arith because test
 5959   // doesn't support sign-extension of
 5960   // 8bit operands
 5961   int encode = dst->encoding();
 5962   encode = prefix_and_encode(encode);
 5963   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5964   emit_int32(imm32);
 5965 }
 5966 
 5967 void Assembler::testl(Register dst, Register src) {
 5968   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5969   emit_arith(0x85, 0xC0, dst, src);
 5970 }
 5971 
 5972 void Assembler::testl(Register dst, Address src) {
 5973   InstructionMark im(this);
 5974   prefix(src, dst);
 5975   emit_int8((unsigned char)0x85);
 5976   emit_operand(dst, src);
 5977 }
 5978 
 5979 void Assembler::tzcntl(Register dst, Register src) {
 5980   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5981   emit_int8((unsigned char)0xF3);
 5982   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5983   emit_int24(0x0F,
 5984              (unsigned char)0xBC,
 5985              0xC0 | encode);
 5986 }
 5987 
 5988 void Assembler::tzcntq(Register dst, Register src) {
 5989   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5990   emit_int8((unsigned char)0xF3);
 5991   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5992   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5993 }
 5994 
 5995 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5996   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5997   InstructionMark im(this);
 5998   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5999   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6000   attributes.set_rex_vex_w_reverted();
 6001   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6002   emit_int8(0x2E);
 6003   emit_operand(dst, src);
 6004 }
 6005 
 6006 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6007   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6008   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6009   attributes.set_rex_vex_w_reverted();
 6010   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6011   emit_int16(0x2E, (0xC0 | encode));
 6012 }
 6013 
 6014 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6015   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6016   InstructionMark im(this);
 6017   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6018   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6019   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6020   emit_int8(0x2E);
 6021   emit_operand(dst, src);
 6022 }
 6023 
 6024 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6025   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6026   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6027   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6028   emit_int16(0x2E, (0xC0 | encode));
 6029 }
 6030 
 6031 void Assembler::xabort(int8_t imm8) {
 6032   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6033 }
 6034 
 6035 void Assembler::xaddb(Address dst, Register src) {
 6036   InstructionMark im(this);
 6037   prefix(dst, src, true);
 6038   emit_int16(0x0F, (unsigned char)0xC0);
 6039   emit_operand(src, dst);
 6040 }
 6041 
 6042 void Assembler::xaddw(Address dst, Register src) {
 6043   InstructionMark im(this);
 6044   emit_int8(0x66);
 6045   prefix(dst, src);
 6046   emit_int16(0x0F, (unsigned char)0xC1);
 6047   emit_operand(src, dst);
 6048 }
 6049 
 6050 void Assembler::xaddl(Address dst, Register src) {
 6051   InstructionMark im(this);
 6052   prefix(dst, src);
 6053   emit_int16(0x0F, (unsigned char)0xC1);
 6054   emit_operand(src, dst);
 6055 }
 6056 
 6057 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6058   InstructionMark im(this);
 6059   relocate(rtype);
 6060   if (abort.is_bound()) {
 6061     address entry = target(abort);
 6062     assert(entry != NULL, "abort entry NULL");
 6063     intptr_t offset = entry - pc();
 6064     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6065     emit_int32(offset - 6); // 2 opcode + 4 address
 6066   } else {
 6067     abort.add_patch_at(code(), locator());
 6068     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6069     emit_int32(0);
 6070   }
 6071 }
 6072 
 6073 void Assembler::xchgb(Register dst, Address src) { // xchg
 6074   InstructionMark im(this);
 6075   prefix(src, dst, true);
 6076   emit_int8((unsigned char)0x86);
 6077   emit_operand(dst, src);
 6078 }
 6079 
 6080 void Assembler::xchgw(Register dst, Address src) { // xchg
 6081   InstructionMark im(this);
 6082   emit_int8(0x66);
 6083   prefix(src, dst);
 6084   emit_int8((unsigned char)0x87);
 6085   emit_operand(dst, src);
 6086 }
 6087 
 6088 void Assembler::xchgl(Register dst, Address src) { // xchg
 6089   InstructionMark im(this);
 6090   prefix(src, dst);
 6091   emit_int8((unsigned char)0x87);
 6092   emit_operand(dst, src);
 6093 }
 6094 
 6095 void Assembler::xchgl(Register dst, Register src) {
 6096   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6097   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6098 }
 6099 
 6100 void Assembler::xend() {
 6101   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6102 }
 6103 
 6104 void Assembler::xgetbv() {
 6105   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6106 }
 6107 
 6108 void Assembler::xorl(Address dst, int32_t imm32) {
 6109   InstructionMark im(this);
 6110   prefix(dst);
 6111   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6112 }
 6113 
 6114 void Assembler::xorl(Register dst, int32_t imm32) {
 6115   prefix(dst);
 6116   emit_arith(0x81, 0xF0, dst, imm32);
 6117 }
 6118 
 6119 void Assembler::xorl(Register dst, Address src) {
 6120   InstructionMark im(this);
 6121   prefix(src, dst);
 6122   emit_int8(0x33);
 6123   emit_operand(dst, src);
 6124 }
 6125 
 6126 void Assembler::xorl(Register dst, Register src) {
 6127   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6128   emit_arith(0x33, 0xC0, dst, src);
 6129 }
 6130 
 6131 void Assembler::xorl(Address dst, Register src) {
 6132   InstructionMark im(this);
 6133   prefix(dst, src);
 6134   emit_int8(0x31);
 6135   emit_operand(src, dst);
 6136 }
 6137 
 6138 void Assembler::xorb(Register dst, Address src) {
 6139   InstructionMark im(this);
 6140   prefix(src, dst);
 6141   emit_int8(0x32);
 6142   emit_operand(dst, src);
 6143 }
 6144 
 6145 void Assembler::xorb(Address dst, Register src) {
 6146   InstructionMark im(this);
 6147   prefix(dst, src, true);
 6148   emit_int8(0x30);
 6149   emit_operand(src, dst);
 6150 }
 6151 
 6152 void Assembler::xorw(Register dst, Register src) {
 6153   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6154   emit_arith(0x33, 0xC0, dst, src);
 6155 }
 6156 
 6157 // AVX 3-operands scalar float-point arithmetic instructions
 6158 
 6159 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6160   assert(VM_Version::supports_avx(), "");
 6161   InstructionMark im(this);
 6162   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6163   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6164   attributes.set_rex_vex_w_reverted();
 6165   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6166   emit_int8(0x58);
 6167   emit_operand(dst, src);
 6168 }
 6169 
 6170 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6171   assert(VM_Version::supports_avx(), "");
 6172   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6173   attributes.set_rex_vex_w_reverted();
 6174   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6175   emit_int16(0x58, (0xC0 | encode));
 6176 }
 6177 
 6178 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6179   assert(VM_Version::supports_avx(), "");
 6180   InstructionMark im(this);
 6181   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6182   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6183   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6184   emit_int8(0x58);
 6185   emit_operand(dst, src);
 6186 }
 6187 
 6188 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6189   assert(VM_Version::supports_avx(), "");
 6190   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6191   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6192   emit_int16(0x58, (0xC0 | encode));
 6193 }
 6194 
 6195 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6196   assert(VM_Version::supports_avx(), "");
 6197   InstructionMark im(this);
 6198   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6199   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6200   attributes.set_rex_vex_w_reverted();
 6201   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6202   emit_int8(0x5E);
 6203   emit_operand(dst, src);
 6204 }
 6205 
 6206 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6207   assert(VM_Version::supports_avx(), "");
 6208   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6209   attributes.set_rex_vex_w_reverted();
 6210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6211   emit_int16(0x5E, (0xC0 | encode));
 6212 }
 6213 
 6214 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6215   assert(VM_Version::supports_avx(), "");
 6216   InstructionMark im(this);
 6217   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6218   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6219   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6220   emit_int8(0x5E);
 6221   emit_operand(dst, src);
 6222 }
 6223 
 6224 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6225   assert(VM_Version::supports_avx(), "");
 6226   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6227   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6228   emit_int16(0x5E, (0xC0 | encode));
 6229 }
 6230 
 6231 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6232   assert(VM_Version::supports_fma(), "");
 6233   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6234   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6235   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6236 }
 6237 
 6238 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6239   assert(VM_Version::supports_fma(), "");
 6240   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6241   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6242   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6243 }
 6244 
 6245 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6246   assert(VM_Version::supports_avx(), "");
 6247   InstructionMark im(this);
 6248   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6249   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6250   attributes.set_rex_vex_w_reverted();
 6251   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6252   emit_int8(0x59);
 6253   emit_operand(dst, src);
 6254 }
 6255 
 6256 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6257   assert(VM_Version::supports_avx(), "");
 6258   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6259   attributes.set_rex_vex_w_reverted();
 6260   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6261   emit_int16(0x59, (0xC0 | encode));
 6262 }
 6263 
 6264 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6265   assert(VM_Version::supports_avx(), "");
 6266   InstructionMark im(this);
 6267   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6268   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6269   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6270   emit_int8(0x59);
 6271   emit_operand(dst, src);
 6272 }
 6273 
 6274 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6275   assert(VM_Version::supports_avx(), "");
 6276   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6277   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6278   emit_int16(0x59, (0xC0 | encode));
 6279 }
 6280 
 6281 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6282   assert(VM_Version::supports_avx(), "");
 6283   InstructionMark im(this);
 6284   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6285   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6286   attributes.set_rex_vex_w_reverted();
 6287   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6288   emit_int8(0x5C);
 6289   emit_operand(dst, src);
 6290 }
 6291 
 6292 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6293   assert(VM_Version::supports_avx(), "");
 6294   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6295   attributes.set_rex_vex_w_reverted();
 6296   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6297   emit_int16(0x5C, (0xC0 | encode));
 6298 }
 6299 
 6300 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6301   assert(VM_Version::supports_avx(), "");
 6302   InstructionMark im(this);
 6303   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6304   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6305   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6306   emit_int8(0x5C);
 6307   emit_operand(dst, src);
 6308 }
 6309 
 6310 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6311   assert(VM_Version::supports_avx(), "");
 6312   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6313   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6314   emit_int16(0x5C, (0xC0 | encode));
 6315 }
 6316 
 6317 //====================VECTOR ARITHMETIC=====================================
 6318 
 6319 // Float-point vector arithmetic
 6320 
 6321 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6322   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6323   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6324   attributes.set_rex_vex_w_reverted();
 6325   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6326   emit_int16(0x58, (0xC0 | encode));
 6327 }
 6328 
 6329 void Assembler::addpd(XMMRegister dst, Address src) {
 6330   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6331   InstructionMark im(this);
 6332   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6333   attributes.set_rex_vex_w_reverted();
 6334   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6335   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6336   emit_int8(0x58);
 6337   emit_operand(dst, src);
 6338 }
 6339 
 6340 
 6341 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6342   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6343   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6344   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6345   emit_int16(0x58, (0xC0 | encode));
 6346 }
 6347 
 6348 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6349   assert(VM_Version::supports_avx(), "");
 6350   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6351   attributes.set_rex_vex_w_reverted();
 6352   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6353   emit_int16(0x58, (0xC0 | encode));
 6354 }
 6355 
 6356 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6357   assert(VM_Version::supports_avx(), "");
 6358   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6359   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6360   emit_int16(0x58, (0xC0 | encode));
 6361 }
 6362 
 6363 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6364   assert(VM_Version::supports_avx(), "");
 6365   InstructionMark im(this);
 6366   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6367   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6368   attributes.set_rex_vex_w_reverted();
 6369   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6370   emit_int8(0x58);
 6371   emit_operand(dst, src);
 6372 }
 6373 
 6374 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6375   assert(VM_Version::supports_avx(), "");
 6376   InstructionMark im(this);
 6377   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6378   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6379   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6380   emit_int8(0x58);
 6381   emit_operand(dst, src);
 6382 }
 6383 
 6384 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6386   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6387   attributes.set_rex_vex_w_reverted();
 6388   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6389   emit_int16(0x5C, (0xC0 | encode));
 6390 }
 6391 
 6392 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6395   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6396   emit_int16(0x5C, (0xC0 | encode));
 6397 }
 6398 
 6399 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6400   assert(VM_Version::supports_avx(), "");
 6401   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6402   attributes.set_rex_vex_w_reverted();
 6403   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6404   emit_int16(0x5C, (0xC0 | encode));
 6405 }
 6406 
 6407 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6408   assert(VM_Version::supports_avx(), "");
 6409   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6410   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6411   emit_int16(0x5C, (0xC0 | encode));
 6412 }
 6413 
 6414 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6415   assert(VM_Version::supports_avx(), "");
 6416   InstructionMark im(this);
 6417   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6418   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6419   attributes.set_rex_vex_w_reverted();
 6420   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6421   emit_int8(0x5C);
 6422   emit_operand(dst, src);
 6423 }
 6424 
 6425 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6426   assert(VM_Version::supports_avx(), "");
 6427   InstructionMark im(this);
 6428   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6429   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6430   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6431   emit_int8(0x5C);
 6432   emit_operand(dst, src);
 6433 }
 6434 
 6435 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6436   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6437   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6438   attributes.set_rex_vex_w_reverted();
 6439   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6440   emit_int16(0x59, (0xC0 | encode));
 6441 }
 6442 
 6443 void Assembler::mulpd(XMMRegister dst, Address src) {
 6444   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6445   InstructionMark im(this);
 6446   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6447   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6448   attributes.set_rex_vex_w_reverted();
 6449   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6450   emit_int8(0x59);
 6451   emit_operand(dst, src);
 6452 }
 6453 
 6454 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6455   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6457   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6458   emit_int16(0x59, (0xC0 | encode));
 6459 }
 6460 
 6461 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6462   assert(VM_Version::supports_avx(), "");
 6463   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6464   attributes.set_rex_vex_w_reverted();
 6465   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6466   emit_int16(0x59, (0xC0 | encode));
 6467 }
 6468 
 6469 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6470   assert(VM_Version::supports_avx(), "");
 6471   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6472   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6473   emit_int16(0x59, (0xC0 | encode));
 6474 }
 6475 
 6476 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6477   assert(VM_Version::supports_avx(), "");
 6478   InstructionMark im(this);
 6479   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6480   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6481   attributes.set_rex_vex_w_reverted();
 6482   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6483   emit_int8(0x59);
 6484   emit_operand(dst, src);
 6485 }
 6486 
 6487 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6488   assert(VM_Version::supports_avx(), "");
 6489   InstructionMark im(this);
 6490   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6491   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6492   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6493   emit_int8(0x59);
 6494   emit_operand(dst, src);
 6495 }
 6496 
 6497 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6498   assert(VM_Version::supports_fma(), "");
 6499   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6500   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6501   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6502 }
 6503 
 6504 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6505   assert(VM_Version::supports_fma(), "");
 6506   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6507   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6508   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6509 }
 6510 
 6511 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6512   assert(VM_Version::supports_fma(), "");
 6513   InstructionMark im(this);
 6514   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6515   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6516   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6517   emit_int8((unsigned char)0xB8);
 6518   emit_operand(dst, src2);
 6519 }
 6520 
 6521 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6522   assert(VM_Version::supports_fma(), "");
 6523   InstructionMark im(this);
 6524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6525   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6526   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6527   emit_int8((unsigned char)0xB8);
 6528   emit_operand(dst, src2);
 6529 }
 6530 
 6531 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6532   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6533   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6534   attributes.set_rex_vex_w_reverted();
 6535   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6536   emit_int16(0x5E, (0xC0 | encode));
 6537 }
 6538 
 6539 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6540   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6541   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6542   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6543   emit_int16(0x5E, (0xC0 | encode));
 6544 }
 6545 
 6546 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6547   assert(VM_Version::supports_avx(), "");
 6548   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6549   attributes.set_rex_vex_w_reverted();
 6550   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6551   emit_int16(0x5E, (0xC0 | encode));
 6552 }
 6553 
 6554 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6555   assert(VM_Version::supports_avx(), "");
 6556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6557   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6558   emit_int16(0x5E, (0xC0 | encode));
 6559 }
 6560 
 6561 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6562   assert(VM_Version::supports_avx(), "");
 6563   InstructionMark im(this);
 6564   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6565   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6566   attributes.set_rex_vex_w_reverted();
 6567   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6568   emit_int8(0x5E);
 6569   emit_operand(dst, src);
 6570 }
 6571 
 6572 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6573   assert(VM_Version::supports_avx(), "");
 6574   InstructionMark im(this);
 6575   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6576   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6577   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6578   emit_int8(0x5E);
 6579   emit_operand(dst, src);
 6580 }
 6581 
 6582 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6583   assert(VM_Version::supports_avx(), "");
 6584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6585   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6586   emit_int24(0x09, (0xC0 | encode), (rmode));
 6587 }
 6588 
 6589 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6590   assert(VM_Version::supports_avx(), "");
 6591   InstructionMark im(this);
 6592   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6593   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6594   emit_int8(0x09);
 6595   emit_operand(dst, src);
 6596   emit_int8((rmode));
 6597 }
 6598 
 6599 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6600   assert(VM_Version::supports_evex(), "requires EVEX support");
 6601   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6602   attributes.set_is_evex_instruction();
 6603   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6604   emit_int24(0x09, (0xC0 | encode), (rmode));
 6605 }
 6606 
 6607 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6608   assert(VM_Version::supports_evex(), "requires EVEX support");
 6609   assert(dst != xnoreg, "sanity");
 6610   InstructionMark im(this);
 6611   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6612   attributes.set_is_evex_instruction();
 6613   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6614   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6615   emit_int8(0x09);
 6616   emit_operand(dst, src);
 6617   emit_int8((rmode));
 6618 }
 6619 
 6620 
 6621 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6622   assert(VM_Version::supports_avx(), "");
 6623   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6624   attributes.set_rex_vex_w_reverted();
 6625   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6626   emit_int16(0x51, (0xC0 | encode));
 6627 }
 6628 
 6629 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6630   assert(VM_Version::supports_avx(), "");
 6631   InstructionMark im(this);
 6632   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6633   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6634   attributes.set_rex_vex_w_reverted();
 6635   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6636   emit_int8(0x51);
 6637   emit_operand(dst, src);
 6638 }
 6639 
 6640 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6641   assert(VM_Version::supports_avx(), "");
 6642   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6643   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6644   emit_int16(0x51, (0xC0 | encode));
 6645 }
 6646 
 6647 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6648   assert(VM_Version::supports_avx(), "");
 6649   InstructionMark im(this);
 6650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6651   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6652   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6653   emit_int8(0x51);
 6654   emit_operand(dst, src);
 6655 }
 6656 
 6657 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6658   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6659   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6660   attributes.set_rex_vex_w_reverted();
 6661   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6662   emit_int16(0x54, (0xC0 | encode));
 6663 }
 6664 
 6665 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6666   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6667   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6668   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6669   emit_int16(0x54, (0xC0 | encode));
 6670 }
 6671 
 6672 void Assembler::andps(XMMRegister dst, Address src) {
 6673   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6674   InstructionMark im(this);
 6675   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6676   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6677   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6678   emit_int8(0x54);
 6679   emit_operand(dst, src);
 6680 }
 6681 
 6682 void Assembler::andpd(XMMRegister dst, Address src) {
 6683   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6684   InstructionMark im(this);
 6685   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6686   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6687   attributes.set_rex_vex_w_reverted();
 6688   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6689   emit_int8(0x54);
 6690   emit_operand(dst, src);
 6691 }
 6692 
 6693 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6694   assert(VM_Version::supports_avx(), "");
 6695   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6696   attributes.set_rex_vex_w_reverted();
 6697   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6698   emit_int16(0x54, (0xC0 | encode));
 6699 }
 6700 
 6701 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6702   assert(VM_Version::supports_avx(), "");
 6703   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6705   emit_int16(0x54, (0xC0 | encode));
 6706 }
 6707 
 6708 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6709   assert(VM_Version::supports_avx(), "");
 6710   InstructionMark im(this);
 6711   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6712   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6713   attributes.set_rex_vex_w_reverted();
 6714   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6715   emit_int8(0x54);
 6716   emit_operand(dst, src);
 6717 }
 6718 
 6719 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6720   assert(VM_Version::supports_avx(), "");
 6721   InstructionMark im(this);
 6722   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6723   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6724   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6725   emit_int8(0x54);
 6726   emit_operand(dst, src);
 6727 }
 6728 
 6729 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6731   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6732   attributes.set_rex_vex_w_reverted();
 6733   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6734   emit_int8(0x15);
 6735   emit_int8((0xC0 | encode));
 6736 }
 6737 
 6738 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6740   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6741   attributes.set_rex_vex_w_reverted();
 6742   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6743   emit_int16(0x14, (0xC0 | encode));
 6744 }
 6745 
 6746 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6748   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6749   attributes.set_rex_vex_w_reverted();
 6750   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6751   emit_int16(0x57, (0xC0 | encode));
 6752 }
 6753 
 6754 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6755   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6756   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6757   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6758   emit_int16(0x57, (0xC0 | encode));
 6759 }
 6760 
 6761 void Assembler::xorpd(XMMRegister dst, Address src) {
 6762   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6763   InstructionMark im(this);
 6764   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6765   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6766   attributes.set_rex_vex_w_reverted();
 6767   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6768   emit_int8(0x57);
 6769   emit_operand(dst, src);
 6770 }
 6771 
 6772 void Assembler::xorps(XMMRegister dst, Address src) {
 6773   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6774   InstructionMark im(this);
 6775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6776   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6777   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6778   emit_int8(0x57);
 6779   emit_operand(dst, src);
 6780 }
 6781 
 6782 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6783   assert(VM_Version::supports_avx(), "");
 6784   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6785   attributes.set_rex_vex_w_reverted();
 6786   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6787   emit_int16(0x57, (0xC0 | encode));
 6788 }
 6789 
 6790 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6791   assert(VM_Version::supports_avx(), "");
 6792   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6793   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6794   emit_int16(0x57, (0xC0 | encode));
 6795 }
 6796 
 6797 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6798   assert(VM_Version::supports_avx(), "");
 6799   InstructionMark im(this);
 6800   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6801   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6802   attributes.set_rex_vex_w_reverted();
 6803   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6804   emit_int8(0x57);
 6805   emit_operand(dst, src);
 6806 }
 6807 
 6808 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6809   assert(VM_Version::supports_avx(), "");
 6810   InstructionMark im(this);
 6811   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6812   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6813   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6814   emit_int8(0x57);
 6815   emit_operand(dst, src);
 6816 }
 6817 
 6818 // Integer vector arithmetic
 6819 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6820   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6821          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6822   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6824   emit_int16(0x01, (0xC0 | encode));
 6825 }
 6826 
 6827 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6828   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6829          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6830   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6831   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6832   emit_int16(0x02, (0xC0 | encode));
 6833 }
 6834 
 6835 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6836   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6837   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6838   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6839   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6840 }
 6841 
 6842 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6843   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6844   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6845   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6846   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6847 }
 6848 
 6849 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6851   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6852   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6853   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6854 }
 6855 
 6856 void Assembler::paddd(XMMRegister dst, Address src) {
 6857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6858   InstructionMark im(this);
 6859   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6860   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6861   emit_int8((unsigned char)0xFE);
 6862   emit_operand(dst, src);
 6863 }
 6864 
 6865 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6866   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6867   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6868   attributes.set_rex_vex_w_reverted();
 6869   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6870   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6871 }
 6872 
 6873 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6874   assert(VM_Version::supports_sse3(), "");
 6875   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6876   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6877   emit_int16(0x01, (0xC0 | encode));
 6878 }
 6879 
 6880 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6881   assert(VM_Version::supports_sse3(), "");
 6882   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6883   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6884   emit_int16(0x02, (0xC0 | encode));
 6885 }
 6886 
 6887 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6888   assert(UseAVX > 0, "requires some form of AVX");
 6889   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6890   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6891   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6892 }
 6893 
 6894 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6895   assert(UseAVX > 0, "requires some form of AVX");
 6896   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6897   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6898   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6899 }
 6900 
 6901 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6902   assert(UseAVX > 0, "requires some form of AVX");
 6903   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6904   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6905   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6906 }
 6907 
 6908 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6909   assert(UseAVX > 0, "requires some form of AVX");
 6910   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6911   attributes.set_rex_vex_w_reverted();
 6912   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6913   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6914 }
 6915 
 6916 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6917   assert(UseAVX > 0, "requires some form of AVX");
 6918   InstructionMark im(this);
 6919   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6920   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6921   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6922   emit_int8((unsigned char)0xFC);
 6923   emit_operand(dst, src);
 6924 }
 6925 
 6926 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6927   assert(UseAVX > 0, "requires some form of AVX");
 6928   InstructionMark im(this);
 6929   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6930   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6931   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6932   emit_int8((unsigned char)0xFD);
 6933   emit_operand(dst, src);
 6934 }
 6935 
 6936 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6937   assert(UseAVX > 0, "requires some form of AVX");
 6938   InstructionMark im(this);
 6939   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6940   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6941   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6942   emit_int8((unsigned char)0xFE);
 6943   emit_operand(dst, src);
 6944 }
 6945 
 6946 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6947   assert(UseAVX > 0, "requires some form of AVX");
 6948   InstructionMark im(this);
 6949   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6950   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6951   attributes.set_rex_vex_w_reverted();
 6952   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6953   emit_int8((unsigned char)0xD4);
 6954   emit_operand(dst, src);
 6955 }
 6956 
 6957 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6958   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6960   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6961   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6962 }
 6963 
 6964 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6965   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6966   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6967   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6968   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6969 }
 6970 
 6971 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6973   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6974   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6975 }
 6976 
 6977 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6978   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6979   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6980   attributes.set_rex_vex_w_reverted();
 6981   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6982   emit_int8((unsigned char)0xFB);
 6983   emit_int8((0xC0 | encode));
 6984 }
 6985 
 6986 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6987   assert(UseAVX > 0, "requires some form of AVX");
 6988   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6989   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6990   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6991 }
 6992 
 6993 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6994   assert(UseAVX > 0, "requires some form of AVX");
 6995   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6996   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6997   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6998 }
 6999 
 7000 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7001   assert(UseAVX > 0, "requires some form of AVX");
 7002   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7003   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7004   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7005 }
 7006 
 7007 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7008   assert(UseAVX > 0, "requires some form of AVX");
 7009   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7010   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7011   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7012 }
 7013 
 7014 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7015   assert(UseAVX > 0, "requires some form of AVX");
 7016   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7017   attributes.set_rex_vex_w_reverted();
 7018   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7019   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7020 }
 7021 
 7022 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7023   assert(UseAVX > 0, "requires some form of AVX");
 7024   InstructionMark im(this);
 7025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7026   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7027   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7028   emit_int8((unsigned char)0xF8);
 7029   emit_operand(dst, src);
 7030 }
 7031 
 7032 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7033   assert(UseAVX > 0, "requires some form of AVX");
 7034   InstructionMark im(this);
 7035   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7036   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7037   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7038   emit_int8((unsigned char)0xF9);
 7039   emit_operand(dst, src);
 7040 }
 7041 
 7042 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7043   assert(UseAVX > 0, "requires some form of AVX");
 7044   InstructionMark im(this);
 7045   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7046   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7047   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7048   emit_int8((unsigned char)0xFA);
 7049   emit_operand(dst, src);
 7050 }
 7051 
 7052 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7053   assert(UseAVX > 0, "requires some form of AVX");
 7054   InstructionMark im(this);
 7055   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7056   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7057   attributes.set_rex_vex_w_reverted();
 7058   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7059   emit_int8((unsigned char)0xFB);
 7060   emit_operand(dst, src);
 7061 }
 7062 
 7063 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7065   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7066   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7067   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7068 }
 7069 
 7070 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7071   assert(VM_Version::supports_sse4_1(), "");
 7072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7073   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7074   emit_int16(0x40, (0xC0 | encode));
 7075 }
 7076 
 7077 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7078   assert(VM_Version::supports_sse2(), "");
 7079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7080   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7081   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7082 }
 7083 
 7084 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7085   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7086          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7087          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7088   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7089   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7090   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7091 }
 7092 
 7093 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7094   assert(UseAVX > 0, "requires some form of AVX");
 7095   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7096   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7097   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7098 }
 7099 
 7100 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7101   assert(UseAVX > 0, "requires some form of AVX");
 7102   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7103   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7104   emit_int16(0x40, (0xC0 | encode));
 7105 }
 7106 
 7107 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7108   assert(UseAVX > 2, "requires some form of EVEX");
 7109   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7110   attributes.set_is_evex_instruction();
 7111   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7112   emit_int16(0x40, (0xC0 | encode));
 7113 }
 7114 
 7115 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7116   assert(UseAVX > 0, "requires some form of AVX");
 7117   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7118   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7119   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7120 }
 7121 
 7122 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7123   assert(UseAVX > 0, "requires some form of AVX");
 7124   InstructionMark im(this);
 7125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7126   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7127   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7128   emit_int8((unsigned char)0xD5);
 7129   emit_operand(dst, src);
 7130 }
 7131 
 7132 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7133   assert(UseAVX > 0, "requires some form of AVX");
 7134   InstructionMark im(this);
 7135   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7136   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7137   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7138   emit_int8(0x40);
 7139   emit_operand(dst, src);
 7140 }
 7141 
 7142 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7143   assert(UseAVX > 2, "requires some form of EVEX");
 7144   InstructionMark im(this);
 7145   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7146   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7147   attributes.set_is_evex_instruction();
 7148   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7149   emit_int8(0x40);
 7150   emit_operand(dst, src);
 7151 }
 7152 
 7153 // Min, max
 7154 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7155   assert(VM_Version::supports_sse4_1(), "");
 7156   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7157   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7158   emit_int16(0x38, (0xC0 | encode));
 7159 }
 7160 
 7161 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7162   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7163         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7164   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7165   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7166   emit_int16(0x38, (0xC0 | encode));
 7167 }
 7168 
 7169 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7170   assert(VM_Version::supports_sse2(), "");
 7171   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7172   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7173   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7174 }
 7175 
 7176 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7177   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7178         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7179   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7180   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7181   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7182 }
 7183 
 7184 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7185   assert(VM_Version::supports_sse4_1(), "");
 7186   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7187   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7188   emit_int16(0x39, (0xC0 | encode));
 7189 }
 7190 
 7191 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7192   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7193         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7194   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7195   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7196   emit_int16(0x39, (0xC0 | encode));
 7197 }
 7198 
 7199 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7200   assert(UseAVX > 2, "requires AVX512F");
 7201   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7202   attributes.set_is_evex_instruction();
 7203   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7204   emit_int16(0x39, (0xC0 | encode));
 7205 }
 7206 
 7207 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7208   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7209   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7210   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7211   emit_int16(0x5D, (0xC0 | encode));
 7212 }
 7213 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7214   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7216   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7217   emit_int16(0x5D, (0xC0 | encode));
 7218 }
 7219 
 7220 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7221   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7222   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7223   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7224   emit_int16(0x5D, (0xC0 | encode));
 7225 }
 7226 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7227   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7228   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7229   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7230   emit_int16(0x5D, (0xC0 | encode));
 7231 }
 7232 
 7233 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7234   assert(VM_Version::supports_sse4_1(), "");
 7235   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7236   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7237   emit_int16(0x3C, (0xC0 | encode));
 7238 }
 7239 
 7240 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7241   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7242         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7243   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7244   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7245   emit_int16(0x3C, (0xC0 | encode));
 7246 }
 7247 
 7248 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7249   assert(VM_Version::supports_sse2(), "");
 7250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7251   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7252   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7253 }
 7254 
 7255 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7256   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7257         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7258   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7260   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7261 }
 7262 
 7263 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7264   assert(VM_Version::supports_sse4_1(), "");
 7265   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7266   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7267   emit_int16(0x3D, (0xC0 | encode));
 7268 }
 7269 
 7270 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7271   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7272         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7273   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7274   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7275   emit_int16(0x3D, (0xC0 | encode));
 7276 }
 7277 
 7278 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7279   assert(UseAVX > 2, "requires AVX512F");
 7280   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7281   attributes.set_is_evex_instruction();
 7282   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7283   emit_int16(0x3D, (0xC0 | encode));
 7284 }
 7285 
 7286 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7287   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7289   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7290   emit_int16(0x5F, (0xC0 | encode));
 7291 }
 7292 
 7293 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7294   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7295   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7296   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7297   emit_int16(0x5F, (0xC0 | encode));
 7298 }
 7299 
 7300 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7301   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7303   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7304   emit_int16(0x5F, (0xC0 | encode));
 7305 }
 7306 
 7307 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7308   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7309   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7311   emit_int16(0x5F, (0xC0 | encode));
 7312 }
 7313 
 7314 // Shift packed integers left by specified number of bits.
 7315 void Assembler::psllw(XMMRegister dst, int shift) {
 7316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7318   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7319   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7320   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7321 }
 7322 
 7323 void Assembler::pslld(XMMRegister dst, int shift) {
 7324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7325   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7326   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7327   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7328   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7329 }
 7330 
 7331 void Assembler::psllq(XMMRegister dst, int shift) {
 7332   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7333   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7334   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7335   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7336   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7337 }
 7338 
 7339 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7342   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7343   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7344 }
 7345 
 7346 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7347   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7348   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7349   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7350   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7351 }
 7352 
 7353 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7355   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7356   attributes.set_rex_vex_w_reverted();
 7357   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7358   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7359 }
 7360 
 7361 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7362   assert(UseAVX > 0, "requires some form of AVX");
 7363   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7364   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7365   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7366   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7367 }
 7368 
 7369 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7370   assert(UseAVX > 0, "requires some form of AVX");
 7371   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7372   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7373   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7374   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7375   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7376 }
 7377 
 7378 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7379   assert(UseAVX > 0, "requires some form of AVX");
 7380   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7381   attributes.set_rex_vex_w_reverted();
 7382   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7383   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7384   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7385 }
 7386 
 7387 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7388   assert(UseAVX > 0, "requires some form of AVX");
 7389   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7390   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7391   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7392 }
 7393 
 7394 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7395   assert(UseAVX > 0, "requires some form of AVX");
 7396   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7397   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7398   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7399 }
 7400 
 7401 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7402   assert(UseAVX > 0, "requires some form of AVX");
 7403   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7404   attributes.set_rex_vex_w_reverted();
 7405   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7406   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7407 }
 7408 
 7409 // Shift packed integers logically right by specified number of bits.
 7410 void Assembler::psrlw(XMMRegister dst, int shift) {
 7411   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7413   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7414   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7415   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7416 }
 7417 
 7418 void Assembler::psrld(XMMRegister dst, int shift) {
 7419   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7420   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7421   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7422   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7423   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7424 }
 7425 
 7426 void Assembler::psrlq(XMMRegister dst, int shift) {
 7427   // Do not confuse it with psrldq SSE2 instruction which
 7428   // shifts 128 bit value in xmm register by number of bytes.
 7429   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7430   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7431   attributes.set_rex_vex_w_reverted();
 7432   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7433   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7434   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7435 }
 7436 
 7437 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7440   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7441   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7442 }
 7443 
 7444 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7445   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7446   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7447   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7448   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7449 }
 7450 
 7451 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7452   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7453   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7454   attributes.set_rex_vex_w_reverted();
 7455   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7456   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7457 }
 7458 
 7459 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7460   assert(UseAVX > 0, "requires some form of AVX");
 7461   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7462   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7463   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7464   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7465 }
 7466 
 7467 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7468   assert(UseAVX > 0, "requires some form of AVX");
 7469   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7470   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7471   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7472   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7473 }
 7474 
 7475 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7476   assert(UseAVX > 0, "requires some form of AVX");
 7477   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7478   attributes.set_rex_vex_w_reverted();
 7479   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7480   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7481   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7482 }
 7483 
 7484 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7485   assert(UseAVX > 0, "requires some form of AVX");
 7486   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7487   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7488   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7489 }
 7490 
 7491 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7492   assert(UseAVX > 0, "requires some form of AVX");
 7493   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7494   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7495   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7496 }
 7497 
 7498 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7499   assert(UseAVX > 0, "requires some form of AVX");
 7500   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7501   attributes.set_rex_vex_w_reverted();
 7502   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7503   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7504 }
 7505 
 7506 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7507   assert(VM_Version::supports_avx512bw(), "");
 7508   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7509   attributes.set_is_evex_instruction();
 7510   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7511   emit_int16(0x10, (0xC0 | encode));
 7512 }
 7513 
 7514 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7515   assert(VM_Version::supports_avx512bw(), "");
 7516   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7517   attributes.set_is_evex_instruction();
 7518   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7519   emit_int16(0x12, (0xC0 | encode));
 7520 }
 7521 
 7522 // Shift packed integers arithmetically right by specified number of bits.
 7523 void Assembler::psraw(XMMRegister dst, int shift) {
 7524   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7525   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7526   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7527   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7528   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7529 }
 7530 
 7531 void Assembler::psrad(XMMRegister dst, int shift) {
 7532   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7534   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7535   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7536   emit_int8(0x72);
 7537   emit_int8((0xC0 | encode));
 7538   emit_int8(shift & 0xFF);
 7539 }
 7540 
 7541 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7542   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7543   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7544   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7545   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7546 }
 7547 
 7548 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7549   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7550   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7551   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7552   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7553 }
 7554 
 7555 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7556   assert(UseAVX > 0, "requires some form of AVX");
 7557   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7558   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7559   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7560   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7561 }
 7562 
 7563 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7564   assert(UseAVX > 0, "requires some form of AVX");
 7565   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7566   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7567   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7568   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7569 }
 7570 
 7571 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7572   assert(UseAVX > 0, "requires some form of AVX");
 7573   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7574   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7575   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7576 }
 7577 
 7578 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7579   assert(UseAVX > 0, "requires some form of AVX");
 7580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7581   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7582   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7583 }
 7584 
 7585 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7586   assert(UseAVX > 2, "requires AVX512");
 7587   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7588   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7589   attributes.set_is_evex_instruction();
 7590   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7591   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7592 }
 7593 
 7594 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7595   assert(UseAVX > 2, "requires AVX512");
 7596   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7597   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7598   attributes.set_is_evex_instruction();
 7599   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7600   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7601 }
 7602 
 7603 // logical operations packed integers
 7604 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7605   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7606   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7607   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7608   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7609 }
 7610 
 7611 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7612   assert(UseAVX > 0, "requires some form of AVX");
 7613   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7614   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7615   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7616 }
 7617 
 7618 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7619   assert(UseAVX > 0, "requires some form of AVX");
 7620   InstructionMark im(this);
 7621   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7622   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7623   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7624   emit_int8((unsigned char)0xDB);
 7625   emit_operand(dst, src);
 7626 }
 7627 
 7628 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7629   assert(VM_Version::supports_evex(), "");
 7630   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7631   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7632   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7633 }
 7634 
 7635 //Variable Shift packed integers logically left.
 7636 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7637   assert(UseAVX > 1, "requires AVX2");
 7638   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7639   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7640   emit_int16(0x47, (0xC0 | encode));
 7641 }
 7642 
 7643 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7644   assert(UseAVX > 1, "requires AVX2");
 7645   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7646   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7647   emit_int16(0x47, (0xC0 | encode));
 7648 }
 7649 
 7650 //Variable Shift packed integers logically right.
 7651 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7652   assert(UseAVX > 1, "requires AVX2");
 7653   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7654   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7655   emit_int16(0x45, (0xC0 | encode));
 7656 }
 7657 
 7658 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7659   assert(UseAVX > 1, "requires AVX2");
 7660   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7661   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7662   emit_int16(0x45, (0xC0 | encode));
 7663 }
 7664 
 7665 //Variable right Shift arithmetic packed integers .
 7666 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7667   assert(UseAVX > 1, "requires AVX2");
 7668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7669   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7670   emit_int16(0x46, (0xC0 | encode));
 7671 }
 7672 
 7673 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7674   assert(VM_Version::supports_avx512bw(), "");
 7675   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7676   attributes.set_is_evex_instruction();
 7677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7678   emit_int16(0x11, (0xC0 | encode));
 7679 }
 7680 
 7681 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7682   assert(UseAVX > 2, "requires AVX512");
 7683   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7684   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7685   attributes.set_is_evex_instruction();
 7686   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7687   emit_int16(0x46, (0xC0 | encode));
 7688 }
 7689 
 7690 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7691   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7692   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7693   attributes.set_is_evex_instruction();
 7694   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7695   emit_int16(0x71, (0xC0 | encode));
 7696 }
 7697 
 7698 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7699   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7700   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7701   attributes.set_is_evex_instruction();
 7702   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7703   emit_int16(0x73, (0xC0 | encode));
 7704 }
 7705 
 7706 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7708   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7709   attributes.set_rex_vex_w_reverted();
 7710   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7711   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7712 }
 7713 
 7714 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7715   assert(UseAVX > 0, "requires some form of AVX");
 7716   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7717   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7718   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7719 }
 7720 
 7721 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7723   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7724   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7725   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7726 }
 7727 
 7728 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7729   assert(UseAVX > 0, "requires some form of AVX");
 7730   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7731   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7732   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7733 }
 7734 
 7735 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7736   assert(UseAVX > 0, "requires some form of AVX");
 7737   InstructionMark im(this);
 7738   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7739   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7740   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7741   emit_int8((unsigned char)0xEB);
 7742   emit_operand(dst, src);
 7743 }
 7744 
 7745 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7746   assert(VM_Version::supports_evex(), "");
 7747   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7748   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7749   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7750 }
 7751 
 7752 
 7753 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7754   assert(VM_Version::supports_evex(), "");
 7755   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7756   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7757   attributes.set_is_evex_instruction();
 7758   attributes.set_embedded_opmask_register_specifier(mask);
 7759   if (merge) {
 7760     attributes.reset_is_clear_context();
 7761   }
 7762   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7763   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7764 }
 7765 
 7766 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7767   assert(VM_Version::supports_evex(), "");
 7768   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7769   InstructionMark im(this);
 7770   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7771   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7772   attributes.set_is_evex_instruction();
 7773   attributes.set_embedded_opmask_register_specifier(mask);
 7774   if (merge) {
 7775     attributes.reset_is_clear_context();
 7776   }
 7777   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7778   emit_int8((unsigned char)0xEB);
 7779   emit_operand(dst, src);
 7780 }
 7781 
 7782 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7784   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7785   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7786   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7787 }
 7788 
 7789 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7790   assert(UseAVX > 0, "requires some form of AVX");
 7791   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7792   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7793   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7794 }
 7795 
 7796 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7797   assert(UseAVX > 0, "requires some form of AVX");
 7798   InstructionMark im(this);
 7799   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7800   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7801   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7802   emit_int8((unsigned char)0xEF);
 7803   emit_operand(dst, src);
 7804 }
 7805 
 7806 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7807   assert(UseAVX > 2, "requires some form of EVEX");
 7808   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7809   attributes.set_rex_vex_w_reverted();
 7810   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7811   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7812 }
 7813 
 7814 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7815   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7816   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7817   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7818   attributes.set_is_evex_instruction();
 7819   attributes.set_embedded_opmask_register_specifier(mask);
 7820   if (merge) {
 7821     attributes.reset_is_clear_context();
 7822   }
 7823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7824   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7825 }
 7826 
 7827 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7828   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7829   InstructionMark im(this);
 7830   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7832   attributes.set_is_evex_instruction();
 7833   attributes.set_embedded_opmask_register_specifier(mask);
 7834   if (merge) {
 7835     attributes.reset_is_clear_context();
 7836   }
 7837   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7838   emit_int8((unsigned char)0xEF);
 7839   emit_operand(dst, src);
 7840 }
 7841 
 7842 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7843   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7844   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7845   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7846   attributes.set_is_evex_instruction();
 7847   attributes.set_embedded_opmask_register_specifier(mask);
 7848   if (merge) {
 7849     attributes.reset_is_clear_context();
 7850   }
 7851   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7852   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7853 }
 7854 
 7855 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7856   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7857   InstructionMark im(this);
 7858   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7859   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7860   attributes.set_is_evex_instruction();
 7861   attributes.set_embedded_opmask_register_specifier(mask);
 7862   if (merge) {
 7863     attributes.reset_is_clear_context();
 7864   }
 7865   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7866   emit_int8((unsigned char)0xEF);
 7867   emit_operand(dst, src);
 7868 }
 7869 
 7870 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7871   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7872   InstructionMark im(this);
 7873   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7874   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7875   attributes.set_is_evex_instruction();
 7876   attributes.set_embedded_opmask_register_specifier(mask);
 7877   if (merge) {
 7878     attributes.reset_is_clear_context();
 7879   }
 7880   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7881   emit_int8((unsigned char)0xDB);
 7882   emit_operand(dst, src);
 7883 }
 7884 
 7885 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7886   assert(VM_Version::supports_evex(), "");
 7887   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7888   attributes.set_is_evex_instruction();
 7889   attributes.set_embedded_opmask_register_specifier(mask);
 7890   if (merge) {
 7891     attributes.reset_is_clear_context();
 7892   }
 7893   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7894   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7895 }
 7896 
 7897 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7898   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7899   InstructionMark im(this);
 7900   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7901   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7902   attributes.set_is_evex_instruction();
 7903   attributes.set_embedded_opmask_register_specifier(mask);
 7904   if (merge) {
 7905     attributes.reset_is_clear_context();
 7906   }
 7907   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7908   emit_int8((unsigned char)0xDB);
 7909   emit_operand(dst, src);
 7910 }
 7911 
 7912 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7913   assert(VM_Version::supports_evex(), "");
 7914   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7915   attributes.set_is_evex_instruction();
 7916   attributes.set_embedded_opmask_register_specifier(mask);
 7917   if (merge) {
 7918     attributes.reset_is_clear_context();
 7919   }
 7920   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7921   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7922 }
 7923 
 7924 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7925   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7926   InstructionMark im(this);
 7927   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7928   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7929   attributes.set_is_evex_instruction();
 7930   attributes.set_embedded_opmask_register_specifier(mask);
 7931   if (merge) {
 7932     attributes.reset_is_clear_context();
 7933   }
 7934   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7935   emit_int8((unsigned char)0xEB);
 7936   emit_operand(dst, src);
 7937 }
 7938 
 7939 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7940   assert(VM_Version::supports_evex(), "requires EVEX support");
 7941   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7942   attributes.set_is_evex_instruction();
 7943   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7944   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7945 }
 7946 
 7947 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7948   assert(VM_Version::supports_evex(), "requires EVEX support");
 7949   assert(dst != xnoreg, "sanity");
 7950   InstructionMark im(this);
 7951   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7952   attributes.set_is_evex_instruction();
 7953   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7954   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7955   emit_int8((unsigned char)0xEF);
 7956   emit_operand(dst, src);
 7957 }
 7958 
 7959 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7960   assert(VM_Version::supports_evex(), "requires EVEX support");
 7961   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7962   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7963   attributes.set_is_evex_instruction();
 7964   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7965   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7966 }
 7967 
 7968 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7969   assert(VM_Version::supports_evex(), "requires EVEX support");
 7970   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7971   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7972   attributes.set_is_evex_instruction();
 7973   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7974   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7975 }
 7976 
 7977 // Register is a class, but it would be assigned numerical value.
 7978 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7979 PRAGMA_DIAG_PUSH
 7980 PRAGMA_NONNULL_IGNORED
 7981 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7982   assert(VM_Version::supports_evex(), "requires EVEX support");
 7983   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7984   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7985   attributes.set_is_evex_instruction();
 7986   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7987   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7988 }
 7989 
 7990 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7991   assert(VM_Version::supports_evex(), "requires EVEX support");
 7992   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7993   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7994   attributes.set_is_evex_instruction();
 7995   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7996   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7997 }
 7998 PRAGMA_DIAG_POP
 7999 
 8000 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8001   assert(VM_Version::supports_evex(), "requires EVEX support");
 8002   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8003   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8004   attributes.set_is_evex_instruction();
 8005   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8006   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8007 }
 8008 
 8009 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8010   assert(VM_Version::supports_evex(), "requires EVEX support");
 8011   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8013   attributes.set_is_evex_instruction();
 8014   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8015   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8016 }
 8017 
 8018 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8019   assert(VM_Version::supports_evex(), "requires EVEX support");
 8020   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8021   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8022   attributes.set_is_evex_instruction();
 8023   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8024   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8025 }
 8026 
 8027 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8028   assert(VM_Version::supports_evex(), "requires EVEX support");
 8029   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8030   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8031   attributes.set_is_evex_instruction();
 8032   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8033   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8034 }
 8035 
 8036 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8037   assert(VM_Version::supports_evex(), "requires EVEX support");
 8038   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8040   attributes.set_is_evex_instruction();
 8041   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8042   emit_int8(0x25);
 8043   emit_int8((unsigned char)(0xC0 | encode));
 8044   emit_int8(imm8);
 8045 }
 8046 
 8047 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8048   assert(VM_Version::supports_evex(), "requires EVEX support");
 8049   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8050   assert(dst != xnoreg, "sanity");
 8051   InstructionMark im(this);
 8052   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8053   attributes.set_is_evex_instruction();
 8054   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8055   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8056   emit_int8(0x25);
 8057   emit_operand(dst, src3);
 8058   emit_int8(imm8);
 8059 }
 8060 
 8061 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8062   assert(VM_Version::supports_evex(), "requires EVEX support");
 8063   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8064   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8065   attributes.set_is_evex_instruction();
 8066   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8067   emit_int8(0x25);
 8068   emit_int8((unsigned char)(0xC0 | encode));
 8069   emit_int8(imm8);
 8070 }
 8071 
 8072 // vinserti forms
 8073 
 8074 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8075   assert(VM_Version::supports_avx2(), "");
 8076   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8077   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8078   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8079   // last byte:
 8080   // 0x00 - insert into lower 128 bits
 8081   // 0x01 - insert into upper 128 bits
 8082   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8083 }
 8084 
 8085 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8086   assert(VM_Version::supports_avx2(), "");
 8087   assert(dst != xnoreg, "sanity");
 8088   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8089   InstructionMark im(this);
 8090   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8091   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8092   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8093   emit_int8(0x38);
 8094   emit_operand(dst, src);
 8095   // 0x00 - insert into lower 128 bits
 8096   // 0x01 - insert into upper 128 bits
 8097   emit_int8(imm8 & 0x01);
 8098 }
 8099 
 8100 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8101   assert(VM_Version::supports_evex(), "");
 8102   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8103   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8104   attributes.set_is_evex_instruction();
 8105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8106   // imm8:
 8107   // 0x00 - insert into q0 128 bits (0..127)
 8108   // 0x01 - insert into q1 128 bits (128..255)
 8109   // 0x02 - insert into q2 128 bits (256..383)
 8110   // 0x03 - insert into q3 128 bits (384..511)
 8111   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8112 }
 8113 
 8114 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8115   assert(VM_Version::supports_avx(), "");
 8116   assert(dst != xnoreg, "sanity");
 8117   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8118   InstructionMark im(this);
 8119   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8120   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8121   attributes.set_is_evex_instruction();
 8122   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8123   emit_int8(0x18);
 8124   emit_operand(dst, src);
 8125   // 0x00 - insert into q0 128 bits (0..127)
 8126   // 0x01 - insert into q1 128 bits (128..255)
 8127   // 0x02 - insert into q2 128 bits (256..383)
 8128   // 0x03 - insert into q3 128 bits (384..511)
 8129   emit_int8(imm8 & 0x03);
 8130 }
 8131 
 8132 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8133   assert(VM_Version::supports_evex(), "");
 8134   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8135   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8136   attributes.set_is_evex_instruction();
 8137   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8138   //imm8:
 8139   // 0x00 - insert into lower 256 bits
 8140   // 0x01 - insert into upper 256 bits
 8141   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8142 }
 8143 
 8144 
 8145 // vinsertf forms
 8146 
 8147 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8148   assert(VM_Version::supports_avx(), "");
 8149   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8150   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8151   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8152   // imm8:
 8153   // 0x00 - insert into lower 128 bits
 8154   // 0x01 - insert into upper 128 bits
 8155   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8156 }
 8157 
 8158 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8159   assert(VM_Version::supports_avx(), "");
 8160   assert(dst != xnoreg, "sanity");
 8161   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8162   InstructionMark im(this);
 8163   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8164   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8165   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8166   emit_int8(0x18);
 8167   emit_operand(dst, src);
 8168   // 0x00 - insert into lower 128 bits
 8169   // 0x01 - insert into upper 128 bits
 8170   emit_int8(imm8 & 0x01);
 8171 }
 8172 
 8173 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8174   assert(VM_Version::supports_avx2(), "");
 8175   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8176   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8177   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8178   // imm8:
 8179   // 0x00 - insert into q0 128 bits (0..127)
 8180   // 0x01 - insert into q1 128 bits (128..255)
 8181   // 0x02 - insert into q0 128 bits (256..383)
 8182   // 0x03 - insert into q1 128 bits (384..512)
 8183   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8184 }
 8185 
 8186 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8187   assert(VM_Version::supports_avx(), "");
 8188   assert(dst != xnoreg, "sanity");
 8189   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8190   InstructionMark im(this);
 8191   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8192   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8193   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8194   emit_int8(0x18);
 8195   emit_operand(dst, src);
 8196   // 0x00 - insert into q0 128 bits (0..127)
 8197   // 0x01 - insert into q1 128 bits (128..255)
 8198   // 0x02 - insert into q0 128 bits (256..383)
 8199   // 0x03 - insert into q1 128 bits (384..512)
 8200   emit_int8(imm8 & 0x03);
 8201 }
 8202 
 8203 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8204   assert(VM_Version::supports_evex(), "");
 8205   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8206   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8207   attributes.set_is_evex_instruction();
 8208   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8209   // imm8:
 8210   // 0x00 - insert into lower 256 bits
 8211   // 0x01 - insert into upper 256 bits
 8212   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8213 }
 8214 
 8215 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8216   assert(VM_Version::supports_evex(), "");
 8217   assert(dst != xnoreg, "sanity");
 8218   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8219   InstructionMark im(this);
 8220   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8221   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8222   attributes.set_is_evex_instruction();
 8223   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8224   emit_int8(0x1A);
 8225   emit_operand(dst, src);
 8226   // 0x00 - insert into lower 256 bits
 8227   // 0x01 - insert into upper 256 bits
 8228   emit_int8(imm8 & 0x01);
 8229 }
 8230 
 8231 
 8232 // vextracti forms
 8233 
 8234 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8235   assert(VM_Version::supports_avx2(), "");
 8236   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8237   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8238   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8239   // imm8:
 8240   // 0x00 - extract from lower 128 bits
 8241   // 0x01 - extract from upper 128 bits
 8242   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8243 }
 8244 
 8245 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8246   assert(VM_Version::supports_avx2(), "");
 8247   assert(src != xnoreg, "sanity");
 8248   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8249   InstructionMark im(this);
 8250   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8251   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8252   attributes.reset_is_clear_context();
 8253   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8254   emit_int8(0x39);
 8255   emit_operand(src, dst);
 8256   // 0x00 - extract from lower 128 bits
 8257   // 0x01 - extract from upper 128 bits
 8258   emit_int8(imm8 & 0x01);
 8259 }
 8260 
 8261 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8262   assert(VM_Version::supports_evex(), "");
 8263   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8264   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8265   attributes.set_is_evex_instruction();
 8266   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8267   // imm8:
 8268   // 0x00 - extract from bits 127:0
 8269   // 0x01 - extract from bits 255:128
 8270   // 0x02 - extract from bits 383:256
 8271   // 0x03 - extract from bits 511:384
 8272   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8273 }
 8274 
 8275 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8276   assert(VM_Version::supports_evex(), "");
 8277   assert(src != xnoreg, "sanity");
 8278   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8279   InstructionMark im(this);
 8280   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8281   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8282   attributes.reset_is_clear_context();
 8283   attributes.set_is_evex_instruction();
 8284   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8285   emit_int8(0x39);
 8286   emit_operand(src, dst);
 8287   // 0x00 - extract from bits 127:0
 8288   // 0x01 - extract from bits 255:128
 8289   // 0x02 - extract from bits 383:256
 8290   // 0x03 - extract from bits 511:384
 8291   emit_int8(imm8 & 0x03);
 8292 }
 8293 
 8294 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8295   assert(VM_Version::supports_avx512dq(), "");
 8296   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8297   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8298   attributes.set_is_evex_instruction();
 8299   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8300   // imm8:
 8301   // 0x00 - extract from bits 127:0
 8302   // 0x01 - extract from bits 255:128
 8303   // 0x02 - extract from bits 383:256
 8304   // 0x03 - extract from bits 511:384
 8305   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8306 }
 8307 
 8308 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8309   assert(VM_Version::supports_evex(), "");
 8310   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8311   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8312   attributes.set_is_evex_instruction();
 8313   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8314   // imm8:
 8315   // 0x00 - extract from lower 256 bits
 8316   // 0x01 - extract from upper 256 bits
 8317   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8318 }
 8319 
 8320 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8321   assert(VM_Version::supports_evex(), "");
 8322   assert(src != xnoreg, "sanity");
 8323   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8324   InstructionMark im(this);
 8325   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8326   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8327   attributes.reset_is_clear_context();
 8328   attributes.set_is_evex_instruction();
 8329   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8330   emit_int8(0x38);
 8331   emit_operand(src, dst);
 8332   // 0x00 - extract from lower 256 bits
 8333   // 0x01 - extract from upper 256 bits
 8334   emit_int8(imm8 & 0x01);
 8335 }
 8336 // vextractf forms
 8337 
 8338 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8339   assert(VM_Version::supports_avx(), "");
 8340   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8341   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8342   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8343   // imm8:
 8344   // 0x00 - extract from lower 128 bits
 8345   // 0x01 - extract from upper 128 bits
 8346   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8347 }
 8348 
 8349 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8350   assert(VM_Version::supports_avx(), "");
 8351   assert(src != xnoreg, "sanity");
 8352   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8353   InstructionMark im(this);
 8354   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8355   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8356   attributes.reset_is_clear_context();
 8357   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8358   emit_int8(0x19);
 8359   emit_operand(src, dst);
 8360   // 0x00 - extract from lower 128 bits
 8361   // 0x01 - extract from upper 128 bits
 8362   emit_int8(imm8 & 0x01);
 8363 }
 8364 
 8365 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8366   assert(VM_Version::supports_evex(), "");
 8367   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8368   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8369   attributes.set_is_evex_instruction();
 8370   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8371   // imm8:
 8372   // 0x00 - extract from bits 127:0
 8373   // 0x01 - extract from bits 255:128
 8374   // 0x02 - extract from bits 383:256
 8375   // 0x03 - extract from bits 511:384
 8376   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8377 }
 8378 
 8379 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8380   assert(VM_Version::supports_evex(), "");
 8381   assert(src != xnoreg, "sanity");
 8382   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8383   InstructionMark im(this);
 8384   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8385   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8386   attributes.reset_is_clear_context();
 8387   attributes.set_is_evex_instruction();
 8388   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8389   emit_int8(0x19);
 8390   emit_operand(src, dst);
 8391   // 0x00 - extract from bits 127:0
 8392   // 0x01 - extract from bits 255:128
 8393   // 0x02 - extract from bits 383:256
 8394   // 0x03 - extract from bits 511:384
 8395   emit_int8(imm8 & 0x03);
 8396 }
 8397 
 8398 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8399   assert(VM_Version::supports_avx512dq(), "");
 8400   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8401   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8402   attributes.set_is_evex_instruction();
 8403   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8404   // imm8:
 8405   // 0x00 - extract from bits 127:0
 8406   // 0x01 - extract from bits 255:128
 8407   // 0x02 - extract from bits 383:256
 8408   // 0x03 - extract from bits 511:384
 8409   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8410 }
 8411 
 8412 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8413   assert(VM_Version::supports_evex(), "");
 8414   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8415   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8416   attributes.set_is_evex_instruction();
 8417   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8418   // imm8:
 8419   // 0x00 - extract from lower 256 bits
 8420   // 0x01 - extract from upper 256 bits
 8421   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8422 }
 8423 
 8424 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8425   assert(VM_Version::supports_evex(), "");
 8426   assert(src != xnoreg, "sanity");
 8427   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8428   InstructionMark im(this);
 8429   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8430   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8431   attributes.reset_is_clear_context();
 8432   attributes.set_is_evex_instruction();
 8433   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8434   emit_int8(0x1B);
 8435   emit_operand(src, dst);
 8436   // 0x00 - extract from lower 256 bits
 8437   // 0x01 - extract from upper 256 bits
 8438   emit_int8(imm8 & 0x01);
 8439 }
 8440 
 8441 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8442 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8443   assert(VM_Version::supports_avx2(), "");
 8444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8445   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8446   emit_int16(0x78, (0xC0 | encode));
 8447 }
 8448 
 8449 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8450   assert(VM_Version::supports_avx2(), "");
 8451   assert(dst != xnoreg, "sanity");
 8452   InstructionMark im(this);
 8453   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8454   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8455   // swap src<->dst for encoding
 8456   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8457   emit_int8(0x78);
 8458   emit_operand(dst, src);
 8459 }
 8460 
 8461 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8462 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8463   assert(VM_Version::supports_avx2(), "");
 8464   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8465   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8466   emit_int16(0x79, (0xC0 | encode));
 8467 }
 8468 
 8469 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8470   assert(VM_Version::supports_avx2(), "");
 8471   assert(dst != xnoreg, "sanity");
 8472   InstructionMark im(this);
 8473   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8474   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8475   // swap src<->dst for encoding
 8476   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8477   emit_int8(0x79);
 8478   emit_operand(dst, src);
 8479 }
 8480 
 8481 // xmm/mem sourced byte/word/dword/qword replicate
 8482 
 8483 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8484   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8486   attributes.set_is_evex_instruction();
 8487   attributes.set_embedded_opmask_register_specifier(mask);
 8488   if (merge) {
 8489     attributes.reset_is_clear_context();
 8490   }
 8491   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8492   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8493 }
 8494 
 8495 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8496   InstructionMark im(this);
 8497   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8498   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8499   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8500   attributes.set_is_evex_instruction();
 8501   attributes.set_embedded_opmask_register_specifier(mask);
 8502   if (merge) {
 8503     attributes.reset_is_clear_context();
 8504   }
 8505   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8506   emit_int8((unsigned char)0xFC);
 8507   emit_operand(dst, src);
 8508 }
 8509 
 8510 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8511   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8512   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8513   attributes.set_is_evex_instruction();
 8514   attributes.set_embedded_opmask_register_specifier(mask);
 8515   if (merge) {
 8516     attributes.reset_is_clear_context();
 8517   }
 8518   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8519   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8520 }
 8521 
 8522 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8523   InstructionMark im(this);
 8524   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8525   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8526   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8527   attributes.set_is_evex_instruction();
 8528   attributes.set_embedded_opmask_register_specifier(mask);
 8529   if (merge) {
 8530     attributes.reset_is_clear_context();
 8531   }
 8532   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8533   emit_int8((unsigned char)0xFD);
 8534   emit_operand(dst, src);
 8535 }
 8536 
 8537 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8538   assert(VM_Version::supports_evex(), "");
 8539   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8540   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8541   attributes.set_is_evex_instruction();
 8542   attributes.set_embedded_opmask_register_specifier(mask);
 8543   if (merge) {
 8544     attributes.reset_is_clear_context();
 8545   }
 8546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8547   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8548 }
 8549 
 8550 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8551   InstructionMark im(this);
 8552   assert(VM_Version::supports_evex(), "");
 8553   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8554   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8555   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8556   attributes.set_is_evex_instruction();
 8557   attributes.set_embedded_opmask_register_specifier(mask);
 8558   if (merge) {
 8559     attributes.reset_is_clear_context();
 8560   }
 8561   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8562   emit_int8((unsigned char)0xFE);
 8563   emit_operand(dst, src);
 8564 }
 8565 
 8566 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8567   assert(VM_Version::supports_evex(), "");
 8568   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8569   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8570   attributes.set_is_evex_instruction();
 8571   attributes.set_embedded_opmask_register_specifier(mask);
 8572   if (merge) {
 8573     attributes.reset_is_clear_context();
 8574   }
 8575   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8576   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8577 }
 8578 
 8579 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8580   InstructionMark im(this);
 8581   assert(VM_Version::supports_evex(), "");
 8582   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8583   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8584   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8585   attributes.set_is_evex_instruction();
 8586   attributes.set_embedded_opmask_register_specifier(mask);
 8587   if (merge) {
 8588     attributes.reset_is_clear_context();
 8589   }
 8590   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8591   emit_int8((unsigned char)0xD4);
 8592   emit_operand(dst, src);
 8593 }
 8594 
 8595 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8596   assert(VM_Version::supports_evex(), "");
 8597   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8598   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8599   attributes.set_is_evex_instruction();
 8600   attributes.set_embedded_opmask_register_specifier(mask);
 8601   if (merge) {
 8602     attributes.reset_is_clear_context();
 8603   }
 8604   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8605   emit_int16(0x58, (0xC0 | encode));
 8606 }
 8607 
 8608 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8609   InstructionMark im(this);
 8610   assert(VM_Version::supports_evex(), "");
 8611   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8612   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8613   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8614   attributes.set_is_evex_instruction();
 8615   attributes.set_embedded_opmask_register_specifier(mask);
 8616   if (merge) {
 8617     attributes.reset_is_clear_context();
 8618   }
 8619   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8620   emit_int8(0x58);
 8621   emit_operand(dst, src);
 8622 }
 8623 
 8624 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8625   assert(VM_Version::supports_evex(), "");
 8626   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8627   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8628   attributes.set_is_evex_instruction();
 8629   attributes.set_embedded_opmask_register_specifier(mask);
 8630   if (merge) {
 8631     attributes.reset_is_clear_context();
 8632   }
 8633   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8634   emit_int16(0x58, (0xC0 | encode));
 8635 }
 8636 
 8637 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8638   InstructionMark im(this);
 8639   assert(VM_Version::supports_evex(), "");
 8640   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8641   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8642   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8643   attributes.set_is_evex_instruction();
 8644   attributes.set_embedded_opmask_register_specifier(mask);
 8645   if (merge) {
 8646     attributes.reset_is_clear_context();
 8647   }
 8648   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8649   emit_int8(0x58);
 8650   emit_operand(dst, src);
 8651 }
 8652 
 8653 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8654   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8655   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8656   attributes.set_is_evex_instruction();
 8657   attributes.set_embedded_opmask_register_specifier(mask);
 8658   if (merge) {
 8659     attributes.reset_is_clear_context();
 8660   }
 8661   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8662   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8663 }
 8664 
 8665 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8666   InstructionMark im(this);
 8667   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8668   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8669   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8670   attributes.set_is_evex_instruction();
 8671   attributes.set_embedded_opmask_register_specifier(mask);
 8672   if (merge) {
 8673     attributes.reset_is_clear_context();
 8674   }
 8675   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8676   emit_int8((unsigned char)0xF8);
 8677   emit_operand(dst, src);
 8678 }
 8679 
 8680 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8681   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8682   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8683   attributes.set_is_evex_instruction();
 8684   attributes.set_embedded_opmask_register_specifier(mask);
 8685   if (merge) {
 8686     attributes.reset_is_clear_context();
 8687   }
 8688   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8689   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8690 }
 8691 
 8692 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8693   InstructionMark im(this);
 8694   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8695   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8696   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8697   attributes.set_is_evex_instruction();
 8698   attributes.set_embedded_opmask_register_specifier(mask);
 8699   if (merge) {
 8700     attributes.reset_is_clear_context();
 8701   }
 8702   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8703   emit_int8((unsigned char)0xF9);
 8704   emit_operand(dst, src);
 8705 }
 8706 
 8707 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8708   assert(VM_Version::supports_evex(), "");
 8709   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8710   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8711   attributes.set_is_evex_instruction();
 8712   attributes.set_embedded_opmask_register_specifier(mask);
 8713   if (merge) {
 8714     attributes.reset_is_clear_context();
 8715   }
 8716   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8717   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8718 }
 8719 
 8720 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8721   InstructionMark im(this);
 8722   assert(VM_Version::supports_evex(), "");
 8723   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8724   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8725   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8726   attributes.set_is_evex_instruction();
 8727   attributes.set_embedded_opmask_register_specifier(mask);
 8728   if (merge) {
 8729     attributes.reset_is_clear_context();
 8730   }
 8731   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8732   emit_int8((unsigned char)0xFA);
 8733   emit_operand(dst, src);
 8734 }
 8735 
 8736 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8737   assert(VM_Version::supports_evex(), "");
 8738   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8739   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8740   attributes.set_is_evex_instruction();
 8741   attributes.set_embedded_opmask_register_specifier(mask);
 8742   if (merge) {
 8743     attributes.reset_is_clear_context();
 8744   }
 8745   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8746   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 8747 }
 8748 
 8749 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8750   InstructionMark im(this);
 8751   assert(VM_Version::supports_evex(), "");
 8752   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8753   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8754   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8755   attributes.set_is_evex_instruction();
 8756   attributes.set_embedded_opmask_register_specifier(mask);
 8757   if (merge) {
 8758     attributes.reset_is_clear_context();
 8759   }
 8760   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8761   emit_int8((unsigned char)0xFB);
 8762   emit_operand(dst, src);
 8763 }
 8764 
 8765 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8766   assert(VM_Version::supports_evex(), "");
 8767   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8768   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8769   attributes.set_is_evex_instruction();
 8770   attributes.set_embedded_opmask_register_specifier(mask);
 8771   if (merge) {
 8772     attributes.reset_is_clear_context();
 8773   }
 8774   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8775   emit_int16(0x5C, (0xC0 | encode));
 8776 }
 8777 
 8778 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8779   InstructionMark im(this);
 8780   assert(VM_Version::supports_evex(), "");
 8781   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8782   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8783   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8784   attributes.set_is_evex_instruction();
 8785   attributes.set_embedded_opmask_register_specifier(mask);
 8786   if (merge) {
 8787     attributes.reset_is_clear_context();
 8788   }
 8789   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8790   emit_int8(0x5C);
 8791   emit_operand(dst, src);
 8792 }
 8793 
 8794 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8795   assert(VM_Version::supports_evex(), "");
 8796   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8797   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8798   attributes.set_is_evex_instruction();
 8799   attributes.set_embedded_opmask_register_specifier(mask);
 8800   if (merge) {
 8801     attributes.reset_is_clear_context();
 8802   }
 8803   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8804   emit_int16(0x5C, (0xC0 | encode));
 8805 }
 8806 
 8807 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8808   InstructionMark im(this);
 8809   assert(VM_Version::supports_evex(), "");
 8810   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8811   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8812   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8813   attributes.set_is_evex_instruction();
 8814   attributes.set_embedded_opmask_register_specifier(mask);
 8815   if (merge) {
 8816     attributes.reset_is_clear_context();
 8817   }
 8818   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8819   emit_int8(0x5C);
 8820   emit_operand(dst, src);
 8821 }
 8822 
 8823 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8824   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8825   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8826   attributes.set_is_evex_instruction();
 8827   attributes.set_embedded_opmask_register_specifier(mask);
 8828   if (merge) {
 8829     attributes.reset_is_clear_context();
 8830   }
 8831   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8832   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 8833 }
 8834 
 8835 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8836   InstructionMark im(this);
 8837   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8838   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8839   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8840   attributes.set_is_evex_instruction();
 8841   attributes.set_embedded_opmask_register_specifier(mask);
 8842   if (merge) {
 8843     attributes.reset_is_clear_context();
 8844   }
 8845   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8846   emit_int8((unsigned char)0xD5);
 8847   emit_operand(dst, src);
 8848 }
 8849 
 8850 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8851   assert(VM_Version::supports_evex(), "");
 8852   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8853   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8854   attributes.set_is_evex_instruction();
 8855   attributes.set_embedded_opmask_register_specifier(mask);
 8856   if (merge) {
 8857     attributes.reset_is_clear_context();
 8858   }
 8859   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8860   emit_int16(0x40, (0xC0 | encode));
 8861 }
 8862 
 8863 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8864   InstructionMark im(this);
 8865   assert(VM_Version::supports_evex(), "");
 8866   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8867   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8868   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8869   attributes.set_is_evex_instruction();
 8870   attributes.set_embedded_opmask_register_specifier(mask);
 8871   if (merge) {
 8872     attributes.reset_is_clear_context();
 8873   }
 8874   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8875   emit_int8(0x40);
 8876   emit_operand(dst, src);
 8877 }
 8878 
 8879 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8880   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8881   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8882   attributes.set_is_evex_instruction();
 8883   attributes.set_embedded_opmask_register_specifier(mask);
 8884   if (merge) {
 8885     attributes.reset_is_clear_context();
 8886   }
 8887   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8888   emit_int16(0x40, (0xC0 | encode));
 8889 }
 8890 
 8891 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8892   InstructionMark im(this);
 8893   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8894   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8895   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8896   attributes.set_is_evex_instruction();
 8897   attributes.set_embedded_opmask_register_specifier(mask);
 8898   if (merge) {
 8899     attributes.reset_is_clear_context();
 8900   }
 8901   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8902   emit_int8(0x40);
 8903   emit_operand(dst, src);
 8904 }
 8905 
 8906 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8907   assert(VM_Version::supports_evex(), "");
 8908   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8909   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8910   attributes.set_is_evex_instruction();
 8911   attributes.set_embedded_opmask_register_specifier(mask);
 8912   if (merge) {
 8913     attributes.reset_is_clear_context();
 8914   }
 8915   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8916   emit_int16(0x59, (0xC0 | encode));
 8917 }
 8918 
 8919 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8920   InstructionMark im(this);
 8921   assert(VM_Version::supports_evex(), "");
 8922   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8923   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8924   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8925   attributes.set_is_evex_instruction();
 8926   attributes.set_embedded_opmask_register_specifier(mask);
 8927   if (merge) {
 8928     attributes.reset_is_clear_context();
 8929   }
 8930   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8931   emit_int8(0x59);
 8932   emit_operand(dst, src);
 8933 }
 8934 
 8935 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8936   assert(VM_Version::supports_evex(), "");
 8937   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8938   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8939   attributes.set_is_evex_instruction();
 8940   attributes.set_embedded_opmask_register_specifier(mask);
 8941   if (merge) {
 8942     attributes.reset_is_clear_context();
 8943   }
 8944   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8945   emit_int16(0x59, (0xC0 | encode));
 8946 }
 8947 
 8948 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8949   InstructionMark im(this);
 8950   assert(VM_Version::supports_evex(), "");
 8951   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8952   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8953   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8954   attributes.set_is_evex_instruction();
 8955   attributes.set_embedded_opmask_register_specifier(mask);
 8956   if (merge) {
 8957     attributes.reset_is_clear_context();
 8958   }
 8959   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8960   emit_int8(0x59);
 8961   emit_operand(dst, src);
 8962 }
 8963 
 8964 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8965   assert(VM_Version::supports_evex(), "");
 8966   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8967   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8968   attributes.set_is_evex_instruction();
 8969   attributes.set_embedded_opmask_register_specifier(mask);
 8970   if (merge) {
 8971     attributes.reset_is_clear_context();
 8972   }
 8973   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8974   emit_int16(0x51, (0xC0 | encode));
 8975 }
 8976 
 8977 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8978   InstructionMark im(this);
 8979   assert(VM_Version::supports_evex(), "");
 8980   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8981   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8982   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8983   attributes.set_is_evex_instruction();
 8984   attributes.set_embedded_opmask_register_specifier(mask);
 8985   if (merge) {
 8986     attributes.reset_is_clear_context();
 8987   }
 8988   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8989   emit_int8(0x51);
 8990   emit_operand(dst, src);
 8991 }
 8992 
 8993 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8994   assert(VM_Version::supports_evex(), "");
 8995   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8996   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8997   attributes.set_is_evex_instruction();
 8998   attributes.set_embedded_opmask_register_specifier(mask);
 8999   if (merge) {
 9000     attributes.reset_is_clear_context();
 9001   }
 9002   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9003   emit_int16(0x51, (0xC0 | encode));
 9004 }
 9005 
 9006 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9007   InstructionMark im(this);
 9008   assert(VM_Version::supports_evex(), "");
 9009   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9010   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9011   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9012   attributes.set_is_evex_instruction();
 9013   attributes.set_embedded_opmask_register_specifier(mask);
 9014   if (merge) {
 9015     attributes.reset_is_clear_context();
 9016   }
 9017   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9018   emit_int8(0x51);
 9019   emit_operand(dst, src);
 9020 }
 9021 
 9022 
 9023 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9024   assert(VM_Version::supports_evex(), "");
 9025   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9026   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9027   attributes.set_is_evex_instruction();
 9028   attributes.set_embedded_opmask_register_specifier(mask);
 9029   if (merge) {
 9030     attributes.reset_is_clear_context();
 9031   }
 9032   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9033   emit_int16(0x5E, (0xC0 | encode));
 9034 }
 9035 
 9036 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9037   InstructionMark im(this);
 9038   assert(VM_Version::supports_evex(), "");
 9039   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9040   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9041   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9042   attributes.set_is_evex_instruction();
 9043   attributes.set_embedded_opmask_register_specifier(mask);
 9044   if (merge) {
 9045     attributes.reset_is_clear_context();
 9046   }
 9047   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9048   emit_int8(0x5E);
 9049   emit_operand(dst, src);
 9050 }
 9051 
 9052 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9053   assert(VM_Version::supports_evex(), "");
 9054   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9055   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9056   attributes.set_is_evex_instruction();
 9057   attributes.set_embedded_opmask_register_specifier(mask);
 9058   if (merge) {
 9059     attributes.reset_is_clear_context();
 9060   }
 9061   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9062   emit_int16(0x5E, (0xC0 | encode));
 9063 }
 9064 
 9065 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9066   InstructionMark im(this);
 9067   assert(VM_Version::supports_evex(), "");
 9068   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9069   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9070   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9071   attributes.set_is_evex_instruction();
 9072   attributes.set_embedded_opmask_register_specifier(mask);
 9073   if (merge) {
 9074     attributes.reset_is_clear_context();
 9075   }
 9076   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9077   emit_int8(0x5E);
 9078   emit_operand(dst, src);
 9079 }
 9080 
 9081 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9082   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9083   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9084   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9085   attributes.set_is_evex_instruction();
 9086   attributes.set_embedded_opmask_register_specifier(mask);
 9087   if (merge) {
 9088     attributes.reset_is_clear_context();
 9089   }
 9090   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9091   emit_int16(0x1C, (0xC0 | encode));
 9092 }
 9093 
 9094 
 9095 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9096   InstructionMark im(this);
 9097   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9098   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9099   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9100   attributes.set_is_evex_instruction();
 9101   attributes.set_embedded_opmask_register_specifier(mask);
 9102   if (merge) {
 9103     attributes.reset_is_clear_context();
 9104   }
 9105   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9106   emit_int8(0x1C);
 9107   emit_operand(dst, src);
 9108 }
 9109 
 9110 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9111   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9112   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9113   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9114   attributes.set_is_evex_instruction();
 9115   attributes.set_embedded_opmask_register_specifier(mask);
 9116   if (merge) {
 9117     attributes.reset_is_clear_context();
 9118   }
 9119   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9120   emit_int16(0x1D, (0xC0 | encode));
 9121 }
 9122 
 9123 
 9124 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9125   InstructionMark im(this);
 9126   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9127   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9128   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9129   attributes.set_is_evex_instruction();
 9130   attributes.set_embedded_opmask_register_specifier(mask);
 9131   if (merge) {
 9132     attributes.reset_is_clear_context();
 9133   }
 9134   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9135   emit_int8(0x1D);
 9136   emit_operand(dst, src);
 9137 }
 9138 
 9139 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9140   assert(VM_Version::supports_evex(), "");
 9141   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9142   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9143   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9144   attributes.set_is_evex_instruction();
 9145   attributes.set_embedded_opmask_register_specifier(mask);
 9146   if (merge) {
 9147     attributes.reset_is_clear_context();
 9148   }
 9149   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9150   emit_int16(0x1E, (0xC0 | encode));
 9151 }
 9152 
 9153 
 9154 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9155   InstructionMark im(this);
 9156   assert(VM_Version::supports_evex(), "");
 9157   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9158   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9159   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9160   attributes.set_is_evex_instruction();
 9161   attributes.set_embedded_opmask_register_specifier(mask);
 9162   if (merge) {
 9163     attributes.reset_is_clear_context();
 9164   }
 9165   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9166   emit_int8(0x1E);
 9167   emit_operand(dst, src);
 9168 }
 9169 
 9170 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9171   assert(VM_Version::supports_evex(), "");
 9172   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9173   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9174   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9175   attributes.set_is_evex_instruction();
 9176   attributes.set_embedded_opmask_register_specifier(mask);
 9177   if (merge) {
 9178     attributes.reset_is_clear_context();
 9179   }
 9180   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9181   emit_int16(0x1F, (0xC0 | encode));
 9182 }
 9183 
 9184 
 9185 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9186   InstructionMark im(this);
 9187   assert(VM_Version::supports_evex(), "");
 9188   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9189   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9190   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9191   attributes.set_is_evex_instruction();
 9192   attributes.set_embedded_opmask_register_specifier(mask);
 9193   if (merge) {
 9194     attributes.reset_is_clear_context();
 9195   }
 9196   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9197   emit_int8(0x1F);
 9198   emit_operand(dst, src);
 9199 }
 9200 
 9201 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9202   assert(VM_Version::supports_evex(), "");
 9203   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9204   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9205   attributes.set_is_evex_instruction();
 9206   attributes.set_embedded_opmask_register_specifier(mask);
 9207   if (merge) {
 9208     attributes.reset_is_clear_context();
 9209   }
 9210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9211   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9212 }
 9213 
 9214 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9215   InstructionMark im(this);
 9216   assert(VM_Version::supports_evex(), "");
 9217   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9218   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9219   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9220   attributes.set_is_evex_instruction();
 9221   attributes.set_embedded_opmask_register_specifier(mask);
 9222   if (merge) {
 9223     attributes.reset_is_clear_context();
 9224   }
 9225   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9226   emit_int8((unsigned char)0xA8);
 9227   emit_operand(dst, src);
 9228 }
 9229 
 9230 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9231   assert(VM_Version::supports_evex(), "");
 9232   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9233   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9234   attributes.set_is_evex_instruction();
 9235   attributes.set_embedded_opmask_register_specifier(mask);
 9236   if (merge) {
 9237     attributes.reset_is_clear_context();
 9238   }
 9239   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9240   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9241 }
 9242 
 9243 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9244   InstructionMark im(this);
 9245   assert(VM_Version::supports_evex(), "");
 9246   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9247   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9248   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9249   attributes.set_is_evex_instruction();
 9250   attributes.set_embedded_opmask_register_specifier(mask);
 9251   if (merge) {
 9252     attributes.reset_is_clear_context();
 9253   }
 9254   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9255   emit_int8((unsigned char)0xA8);
 9256   emit_operand(dst, src);
 9257 }
 9258 
 9259 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9260   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9261   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9262   attributes.set_is_evex_instruction();
 9263   attributes.set_embedded_opmask_register_specifier(mask);
 9264   if (merge) {
 9265     attributes.reset_is_clear_context();
 9266   }
 9267   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9268   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9269 }
 9270 
 9271 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9272   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9273   InstructionMark im(this);
 9274   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9275   attributes.set_is_evex_instruction();
 9276   attributes.set_embedded_opmask_register_specifier(mask);
 9277   if (merge) {
 9278     attributes.reset_is_clear_context();
 9279   }
 9280   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9281   emit_int8((unsigned char)0x8D);
 9282   emit_operand(dst, src);
 9283 }
 9284 
 9285 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9286   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9287   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9288   attributes.set_is_evex_instruction();
 9289   attributes.set_embedded_opmask_register_specifier(mask);
 9290   if (merge) {
 9291     attributes.reset_is_clear_context();
 9292   }
 9293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9294   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9295 }
 9296 
 9297 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9298   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9299   InstructionMark im(this);
 9300   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9301   attributes.set_is_evex_instruction();
 9302   attributes.set_embedded_opmask_register_specifier(mask);
 9303   if (merge) {
 9304     attributes.reset_is_clear_context();
 9305   }
 9306   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9307   emit_int8((unsigned char)0x8D);
 9308   emit_operand(dst, src);
 9309 }
 9310 
 9311 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9312   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9313   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9314   attributes.set_is_evex_instruction();
 9315   attributes.set_embedded_opmask_register_specifier(mask);
 9316   if (merge) {
 9317     attributes.reset_is_clear_context();
 9318   }
 9319   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9320   emit_int16(0x36, (0xC0 | encode));
 9321 }
 9322 
 9323 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9324   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9325   InstructionMark im(this);
 9326   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9327   attributes.set_is_evex_instruction();
 9328   attributes.set_embedded_opmask_register_specifier(mask);
 9329   if (merge) {
 9330     attributes.reset_is_clear_context();
 9331   }
 9332   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9333   emit_int8(0x36);
 9334   emit_operand(dst, src);
 9335 }
 9336 
 9337 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9338   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9339   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9340   attributes.set_is_evex_instruction();
 9341   attributes.set_embedded_opmask_register_specifier(mask);
 9342   if (merge) {
 9343     attributes.reset_is_clear_context();
 9344   }
 9345   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9346   emit_int16(0x36, (0xC0 | encode));
 9347 }
 9348 
 9349 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9350   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9351   InstructionMark im(this);
 9352   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9353   attributes.set_is_evex_instruction();
 9354   attributes.set_embedded_opmask_register_specifier(mask);
 9355   if (merge) {
 9356     attributes.reset_is_clear_context();
 9357   }
 9358   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9359   emit_int8(0x36);
 9360   emit_operand(dst, src);
 9361 }
 9362 
 9363 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9364   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9365   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9366   attributes.set_is_evex_instruction();
 9367   attributes.set_embedded_opmask_register_specifier(mask);
 9368   if (merge) {
 9369     attributes.reset_is_clear_context();
 9370   }
 9371   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9372   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9373 }
 9374 
 9375 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9376   assert(VM_Version::supports_evex(), "");
 9377   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9379   attributes.set_is_evex_instruction();
 9380   attributes.set_embedded_opmask_register_specifier(mask);
 9381   if (merge) {
 9382     attributes.reset_is_clear_context();
 9383   }
 9384   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9385   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9386 }
 9387 
 9388 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9389   assert(VM_Version::supports_evex(), "");
 9390   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9392   attributes.set_is_evex_instruction();
 9393   attributes.set_embedded_opmask_register_specifier(mask);
 9394   if (merge) {
 9395     attributes.reset_is_clear_context();
 9396   }
 9397   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9398   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9399 }
 9400 
 9401 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9402   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9403   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9404   attributes.set_is_evex_instruction();
 9405   attributes.set_embedded_opmask_register_specifier(mask);
 9406   if (merge) {
 9407     attributes.reset_is_clear_context();
 9408   }
 9409   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9410   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9411 }
 9412 
 9413 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9414   assert(VM_Version::supports_evex(), "");
 9415   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9416   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9417   attributes.set_is_evex_instruction();
 9418   attributes.set_embedded_opmask_register_specifier(mask);
 9419   if (merge) {
 9420     attributes.reset_is_clear_context();
 9421   }
 9422   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9423   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9424 }
 9425 
 9426 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9427   assert(VM_Version::supports_evex(), "");
 9428   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9429   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9430   attributes.set_is_evex_instruction();
 9431   attributes.set_embedded_opmask_register_specifier(mask);
 9432   if (merge) {
 9433     attributes.reset_is_clear_context();
 9434   }
 9435   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9436   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9437 }
 9438 
 9439 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9440   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9442   attributes.set_is_evex_instruction();
 9443   attributes.set_embedded_opmask_register_specifier(mask);
 9444   if (merge) {
 9445     attributes.reset_is_clear_context();
 9446   }
 9447   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9448   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9449 }
 9450 
 9451 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9452   assert(VM_Version::supports_evex(), "");
 9453   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9454   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9455   attributes.set_is_evex_instruction();
 9456   attributes.set_embedded_opmask_register_specifier(mask);
 9457   if (merge) {
 9458     attributes.reset_is_clear_context();
 9459   }
 9460   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9461   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9462 }
 9463 
 9464 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9465   assert(VM_Version::supports_evex(), "");
 9466   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9467   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9468   attributes.set_is_evex_instruction();
 9469   attributes.set_embedded_opmask_register_specifier(mask);
 9470   if (merge) {
 9471     attributes.reset_is_clear_context();
 9472   }
 9473   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9474   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9475 }
 9476 
 9477 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9478   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9479   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9480   attributes.set_is_evex_instruction();
 9481   attributes.set_embedded_opmask_register_specifier(mask);
 9482   if (merge) {
 9483     attributes.reset_is_clear_context();
 9484   }
 9485   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9486   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9487 }
 9488 
 9489 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9490   assert(VM_Version::supports_evex(), "");
 9491   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9493   attributes.set_is_evex_instruction();
 9494   attributes.set_embedded_opmask_register_specifier(mask);
 9495   if (merge) {
 9496     attributes.reset_is_clear_context();
 9497   }
 9498   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9499   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9500 }
 9501 
 9502 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9503   assert(VM_Version::supports_evex(), "");
 9504   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9505   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9506   attributes.set_is_evex_instruction();
 9507   attributes.set_embedded_opmask_register_specifier(mask);
 9508   if (merge) {
 9509     attributes.reset_is_clear_context();
 9510   }
 9511   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9512   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9513 }
 9514 
 9515 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9516   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9517   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9518   attributes.set_is_evex_instruction();
 9519   attributes.set_embedded_opmask_register_specifier(mask);
 9520   if (merge) {
 9521     attributes.reset_is_clear_context();
 9522   }
 9523   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9524   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9525 }
 9526 
 9527 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9528   assert(VM_Version::supports_evex(), "");
 9529   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9530   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9531   attributes.set_is_evex_instruction();
 9532   attributes.set_embedded_opmask_register_specifier(mask);
 9533   if (merge) {
 9534     attributes.reset_is_clear_context();
 9535   }
 9536   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9537   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9538 }
 9539 
 9540 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9541   assert(VM_Version::supports_evex(), "");
 9542   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9543   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9544   attributes.set_is_evex_instruction();
 9545   attributes.set_embedded_opmask_register_specifier(mask);
 9546   if (merge) {
 9547     attributes.reset_is_clear_context();
 9548   }
 9549   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9550   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9551 }
 9552 
 9553 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9554   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9555   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9556   attributes.set_is_evex_instruction();
 9557   attributes.set_embedded_opmask_register_specifier(mask);
 9558   if (merge) {
 9559     attributes.reset_is_clear_context();
 9560   }
 9561   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9562   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9563 }
 9564 
 9565 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9566   assert(VM_Version::supports_evex(), "");
 9567   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9568   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9569   attributes.set_is_evex_instruction();
 9570   attributes.set_embedded_opmask_register_specifier(mask);
 9571   if (merge) {
 9572     attributes.reset_is_clear_context();
 9573   }
 9574   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9575   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9576 }
 9577 
 9578 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9579   assert(VM_Version::supports_evex(), "");
 9580   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9581   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9582   attributes.set_is_evex_instruction();
 9583   attributes.set_embedded_opmask_register_specifier(mask);
 9584   if (merge) {
 9585     attributes.reset_is_clear_context();
 9586   }
 9587   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9588   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9589 }
 9590 
 9591 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9592   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9593   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9594   attributes.set_is_evex_instruction();
 9595   attributes.set_embedded_opmask_register_specifier(mask);
 9596   if (merge) {
 9597     attributes.reset_is_clear_context();
 9598   }
 9599   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9600   emit_int16(0x12, (0xC0 | encode));
 9601 }
 9602 
 9603 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9604   assert(VM_Version::supports_evex(), "");
 9605   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9606   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9607   attributes.set_is_evex_instruction();
 9608   attributes.set_embedded_opmask_register_specifier(mask);
 9609   if (merge) {
 9610     attributes.reset_is_clear_context();
 9611   }
 9612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9613   emit_int16(0x47, (0xC0 | encode));
 9614 }
 9615 
 9616 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9617   assert(VM_Version::supports_evex(), "");
 9618   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9620   attributes.set_is_evex_instruction();
 9621   attributes.set_embedded_opmask_register_specifier(mask);
 9622   if (merge) {
 9623     attributes.reset_is_clear_context();
 9624   }
 9625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9626   emit_int16(0x47, (0xC0 | encode));
 9627 }
 9628 
 9629 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9630   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9631   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9632   attributes.set_is_evex_instruction();
 9633   attributes.set_embedded_opmask_register_specifier(mask);
 9634   if (merge) {
 9635     attributes.reset_is_clear_context();
 9636   }
 9637   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9638   emit_int16(0x10, (0xC0 | encode));
 9639 }
 9640 
 9641 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9642   assert(VM_Version::supports_evex(), "");
 9643   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9645   attributes.set_is_evex_instruction();
 9646   attributes.set_embedded_opmask_register_specifier(mask);
 9647   if (merge) {
 9648     attributes.reset_is_clear_context();
 9649   }
 9650   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9651   emit_int16(0x45, (0xC0 | encode));
 9652 }
 9653 
 9654 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9655   assert(VM_Version::supports_evex(), "");
 9656   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9657   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9658   attributes.set_is_evex_instruction();
 9659   attributes.set_embedded_opmask_register_specifier(mask);
 9660   if (merge) {
 9661     attributes.reset_is_clear_context();
 9662   }
 9663   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9664   emit_int16(0x45, (0xC0 | encode));
 9665 }
 9666 
 9667 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9668   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9669   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9670   attributes.set_is_evex_instruction();
 9671   attributes.set_embedded_opmask_register_specifier(mask);
 9672   if (merge) {
 9673     attributes.reset_is_clear_context();
 9674   }
 9675   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9676   emit_int16(0x11, (0xC0 | encode));
 9677 }
 9678 
 9679 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9680   assert(VM_Version::supports_evex(), "");
 9681   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9682   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9683   attributes.set_is_evex_instruction();
 9684   attributes.set_embedded_opmask_register_specifier(mask);
 9685   if (merge) {
 9686     attributes.reset_is_clear_context();
 9687   }
 9688   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9689   emit_int16(0x46, (0xC0 | encode));
 9690 }
 9691 
 9692 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9693   assert(VM_Version::supports_evex(), "");
 9694   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9695   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9696   attributes.set_is_evex_instruction();
 9697   attributes.set_embedded_opmask_register_specifier(mask);
 9698   if (merge) {
 9699     attributes.reset_is_clear_context();
 9700   }
 9701   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9702   emit_int16(0x46, (0xC0 | encode));
 9703 }
 9704 
 9705 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9706   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9707   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9708   attributes.set_is_evex_instruction();
 9709   attributes.set_embedded_opmask_register_specifier(mask);
 9710   if (merge) {
 9711     attributes.reset_is_clear_context();
 9712   }
 9713   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9714   emit_int16(0x38, (0xC0 | encode));
 9715 }
 9716 
 9717 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9718   assert(VM_Version::supports_avx512bw(), "");
 9719   InstructionMark im(this);
 9720   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9721   attributes.set_is_evex_instruction();
 9722   attributes.set_embedded_opmask_register_specifier(mask);
 9723   if (merge) {
 9724     attributes.reset_is_clear_context();
 9725   }
 9726   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9727   emit_int8(0x38);
 9728   emit_operand(dst, src);
 9729 }
 9730 
 9731 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9732   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9733   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9734   attributes.set_is_evex_instruction();
 9735   attributes.set_embedded_opmask_register_specifier(mask);
 9736   if (merge) {
 9737     attributes.reset_is_clear_context();
 9738   }
 9739   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9740   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9741 }
 9742 
 9743 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9744   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9745   InstructionMark im(this);
 9746   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9747   attributes.set_is_evex_instruction();
 9748   attributes.set_embedded_opmask_register_specifier(mask);
 9749   if (merge) {
 9750     attributes.reset_is_clear_context();
 9751   }
 9752   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9753   emit_int8((unsigned char)0xEA);
 9754   emit_operand(dst, src);
 9755 }
 9756 
 9757 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9758   assert(VM_Version::supports_evex(), "");
 9759   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9760   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9761   attributes.set_is_evex_instruction();
 9762   attributes.set_embedded_opmask_register_specifier(mask);
 9763   if (merge) {
 9764     attributes.reset_is_clear_context();
 9765   }
 9766   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9767   emit_int16(0x39, (0xC0 | encode));
 9768 }
 9769 
 9770 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9771   assert(VM_Version::supports_evex(), "");
 9772   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9773   InstructionMark im(this);
 9774   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9775   attributes.set_is_evex_instruction();
 9776   attributes.set_embedded_opmask_register_specifier(mask);
 9777   if (merge) {
 9778     attributes.reset_is_clear_context();
 9779   }
 9780   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9781   emit_int8(0x39);
 9782   emit_operand(dst, src);
 9783 }
 9784 
 9785 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9786   assert(VM_Version::supports_evex(), "");
 9787   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9788   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9789   attributes.set_is_evex_instruction();
 9790   attributes.set_embedded_opmask_register_specifier(mask);
 9791   if (merge) {
 9792     attributes.reset_is_clear_context();
 9793   }
 9794   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9795   emit_int16(0x39, (0xC0 | encode));
 9796 }
 9797 
 9798 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9799   assert(VM_Version::supports_evex(), "");
 9800   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9801   InstructionMark im(this);
 9802   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9803   attributes.set_is_evex_instruction();
 9804   attributes.set_embedded_opmask_register_specifier(mask);
 9805   if (merge) {
 9806     attributes.reset_is_clear_context();
 9807   }
 9808   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9809   emit_int8(0x39);
 9810   emit_operand(dst, src);
 9811 }
 9812 
 9813 
 9814 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9815   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9816   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9817   attributes.set_is_evex_instruction();
 9818   attributes.set_embedded_opmask_register_specifier(mask);
 9819   if (merge) {
 9820     attributes.reset_is_clear_context();
 9821   }
 9822   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9823   emit_int16(0x3C, (0xC0 | encode));
 9824 }
 9825 
 9826 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9827   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9828   InstructionMark im(this);
 9829   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9830   attributes.set_is_evex_instruction();
 9831   attributes.set_embedded_opmask_register_specifier(mask);
 9832   if (merge) {
 9833     attributes.reset_is_clear_context();
 9834   }
 9835   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9836   emit_int8(0x3C);
 9837   emit_operand(dst, src);
 9838 }
 9839 
 9840 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9841   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9842   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9843   attributes.set_is_evex_instruction();
 9844   attributes.set_embedded_opmask_register_specifier(mask);
 9845   if (merge) {
 9846     attributes.reset_is_clear_context();
 9847   }
 9848   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9849   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 9850 }
 9851 
 9852 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9853   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9854   InstructionMark im(this);
 9855   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9856   attributes.set_is_evex_instruction();
 9857   attributes.set_embedded_opmask_register_specifier(mask);
 9858   if (merge) {
 9859     attributes.reset_is_clear_context();
 9860   }
 9861   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9862   emit_int8((unsigned char)0xEE);
 9863   emit_operand(dst, src);
 9864 }
 9865 
 9866 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9867   assert(VM_Version::supports_evex(), "");
 9868   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9869   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9870   attributes.set_is_evex_instruction();
 9871   attributes.set_embedded_opmask_register_specifier(mask);
 9872   if (merge) {
 9873     attributes.reset_is_clear_context();
 9874   }
 9875   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9876   emit_int16(0x3D, (0xC0 | encode));
 9877 }
 9878 
 9879 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9880   assert(VM_Version::supports_evex(), "");
 9881   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9882   InstructionMark im(this);
 9883   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9884   attributes.set_is_evex_instruction();
 9885   attributes.set_embedded_opmask_register_specifier(mask);
 9886   if (merge) {
 9887     attributes.reset_is_clear_context();
 9888   }
 9889   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9890   emit_int8(0x3D);
 9891   emit_operand(dst, src);
 9892 }
 9893 
 9894 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9895   assert(VM_Version::supports_evex(), "");
 9896   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9897   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9898   attributes.set_is_evex_instruction();
 9899   attributes.set_embedded_opmask_register_specifier(mask);
 9900   if (merge) {
 9901     attributes.reset_is_clear_context();
 9902   }
 9903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9904   emit_int16(0x3D, (0xC0 | encode));
 9905 }
 9906 
 9907 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9908   assert(VM_Version::supports_evex(), "");
 9909   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9910   InstructionMark im(this);
 9911   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9912   attributes.set_is_evex_instruction();
 9913   attributes.set_embedded_opmask_register_specifier(mask);
 9914   if (merge) {
 9915     attributes.reset_is_clear_context();
 9916   }
 9917   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9918   emit_int8(0x3D);
 9919   emit_operand(dst, src);
 9920 }
 9921 
 9922 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 9923 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
 9924   assert(UseAVX >= 2, "");
 9925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9926   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9927   emit_int16(0x58, (0xC0 | encode));
 9928 }
 9929 
 9930 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
 9931   assert(VM_Version::supports_avx2(), "");
 9932   assert(dst != xnoreg, "sanity");
 9933   InstructionMark im(this);
 9934   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9935   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9936   // swap src<->dst for encoding
 9937   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9938   emit_int8(0x58);
 9939   emit_operand(dst, src);
 9940 }
 9941 
 9942 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 9943 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
 9944   assert(VM_Version::supports_avx2(), "");
 9945   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9946   attributes.set_rex_vex_w_reverted();
 9947   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9948   emit_int16(0x59, (0xC0 | encode));
 9949 }
 9950 
 9951 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
 9952   assert(VM_Version::supports_avx2(), "");
 9953   assert(dst != xnoreg, "sanity");
 9954   InstructionMark im(this);
 9955   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9956   attributes.set_rex_vex_w_reverted();
 9957   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 9958   // swap src<->dst for encoding
 9959   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9960   emit_int8(0x59);
 9961   emit_operand(dst, src);
 9962 }
 9963 
 9964 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
 9965   assert(vector_len != Assembler::AVX_128bit, "");
 9966   assert(VM_Version::supports_avx512dq(), "");
 9967   assert(dst != xnoreg, "sanity");
 9968   InstructionMark im(this);
 9969   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9970   attributes.set_rex_vex_w_reverted();
 9971   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 9972   // swap src<->dst for encoding
 9973   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9974   emit_int8(0x5A);
 9975   emit_operand(dst, src);
 9976 }
 9977 
 9978 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
 9979   assert(vector_len != Assembler::AVX_128bit, "");
 9980   assert(VM_Version::supports_avx512dq(), "");
 9981   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9982   attributes.set_rex_vex_w_reverted();
 9983   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9984   emit_int16(0x5A, (0xC0 | encode));
 9985 }
 9986 
 9987 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
 9988   assert(vector_len != Assembler::AVX_128bit, "");
 9989   assert(VM_Version::supports_avx512dq(), "");
 9990   assert(dst != xnoreg, "sanity");
 9991   InstructionMark im(this);
 9992   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9993   attributes.set_rex_vex_w_reverted();
 9994   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 9995   // swap src<->dst for encoding
 9996   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9997   emit_int8(0x5A);
 9998   emit_operand(dst, src);
 9999 }
10000 
10001 // scalar single/double precision replicate
10002 
10003 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10004 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10005   assert(VM_Version::supports_avx2(), "");
10006   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10007   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10008   emit_int16(0x18, (0xC0 | encode));
10009 }
10010 
10011 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10012   assert(VM_Version::supports_avx(), "");
10013   assert(dst != xnoreg, "sanity");
10014   InstructionMark im(this);
10015   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10016   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10017   // swap src<->dst for encoding
10018   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10019   emit_int8(0x18);
10020   emit_operand(dst, src);
10021 }
10022 
10023 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10024 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10025   assert(VM_Version::supports_avx2(), "");
10026   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10027   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10028   attributes.set_rex_vex_w_reverted();
10029   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10030   emit_int16(0x19, (0xC0 | encode));
10031 }
10032 
10033 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10034   assert(VM_Version::supports_avx(), "");
10035   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10036   assert(dst != xnoreg, "sanity");
10037   InstructionMark im(this);
10038   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10039   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10040   attributes.set_rex_vex_w_reverted();
10041   // swap src<->dst for encoding
10042   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10043   emit_int8(0x19);
10044   emit_operand(dst, src);
10045 }
10046 
10047 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10048   assert(VM_Version::supports_avx(), "");
10049   assert(vector_len == AVX_256bit, "");
10050   assert(dst != xnoreg, "sanity");
10051   InstructionMark im(this);
10052   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10053   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10054   // swap src<->dst for encoding
10055   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10056   emit_int8(0x1A);
10057   emit_operand(dst, src);
10058 }
10059 
10060 // gpr source broadcast forms
10061 
10062 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10063 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10064   assert(VM_Version::supports_avx512bw(), "");
10065   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10066   attributes.set_is_evex_instruction();
10067   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10068   emit_int16(0x7A, (0xC0 | encode));
10069 }
10070 
10071 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10072 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10073   assert(VM_Version::supports_avx512bw(), "");
10074   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10075   attributes.set_is_evex_instruction();
10076   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10077   emit_int16(0x7B, (0xC0 | encode));
10078 }
10079 
10080 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10081 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10082   assert(VM_Version::supports_evex(), "");
10083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10084   attributes.set_is_evex_instruction();
10085   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10086   emit_int16(0x7C, (0xC0 | encode));
10087 }
10088 
10089 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10090 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10091   assert(VM_Version::supports_evex(), "");
10092   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10093   attributes.set_is_evex_instruction();
10094   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10095   emit_int16(0x7C, (0xC0 | encode));
10096 }
10097 
10098 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10099   assert(VM_Version::supports_avx2(), "");
10100   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10101   assert(dst != xnoreg, "sanity");
10102   assert(src.isxmmindex(),"expected to be xmm index");
10103   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10104   InstructionMark im(this);
10105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10106   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10107   emit_int8((unsigned char)0x90);
10108   emit_operand(dst, src);
10109 }
10110 
10111 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10112   assert(VM_Version::supports_avx2(), "");
10113   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10114   assert(dst != xnoreg, "sanity");
10115   assert(src.isxmmindex(),"expected to be xmm index");
10116   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10117   InstructionMark im(this);
10118   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10119   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10120   emit_int8((unsigned char)0x90);
10121   emit_operand(dst, src);
10122 }
10123 
10124 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10125   assert(VM_Version::supports_avx2(), "");
10126   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10127   assert(dst != xnoreg, "sanity");
10128   assert(src.isxmmindex(),"expected to be xmm index");
10129   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10130   InstructionMark im(this);
10131   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10132   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10133   emit_int8((unsigned char)0x92);
10134   emit_operand(dst, src);
10135 }
10136 
10137 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10138   assert(VM_Version::supports_avx2(), "");
10139   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10140   assert(dst != xnoreg, "sanity");
10141   assert(src.isxmmindex(),"expected to be xmm index");
10142   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10143   InstructionMark im(this);
10144   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10145   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10146   emit_int8((unsigned char)0x92);
10147   emit_operand(dst, src);
10148 }
10149 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10150   assert(VM_Version::supports_evex(), "");
10151   assert(dst != xnoreg, "sanity");
10152   assert(src.isxmmindex(),"expected to be xmm index");
10153   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10154   assert(mask != k0, "instruction will #UD if mask is in k0");
10155   InstructionMark im(this);
10156   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10157   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10158   attributes.reset_is_clear_context();
10159   attributes.set_embedded_opmask_register_specifier(mask);
10160   attributes.set_is_evex_instruction();
10161   // swap src<->dst for encoding
10162   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10163   emit_int8((unsigned char)0x90);
10164   emit_operand(dst, src);
10165 }
10166 
10167 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10168   assert(VM_Version::supports_evex(), "");
10169   assert(dst != xnoreg, "sanity");
10170   assert(src.isxmmindex(),"expected to be xmm index");
10171   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10172   assert(mask != k0, "instruction will #UD if mask is in k0");
10173   InstructionMark im(this);
10174   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10175   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10176   attributes.reset_is_clear_context();
10177   attributes.set_embedded_opmask_register_specifier(mask);
10178   attributes.set_is_evex_instruction();
10179   // swap src<->dst for encoding
10180   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10181   emit_int8((unsigned char)0x90);
10182   emit_operand(dst, src);
10183 }
10184 
10185 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10186   assert(VM_Version::supports_evex(), "");
10187   assert(dst != xnoreg, "sanity");
10188   assert(src.isxmmindex(),"expected to be xmm index");
10189   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10190   assert(mask != k0, "instruction will #UD if mask is in k0");
10191   InstructionMark im(this);
10192   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10193   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10194   attributes.reset_is_clear_context();
10195   attributes.set_embedded_opmask_register_specifier(mask);
10196   attributes.set_is_evex_instruction();
10197   // swap src<->dst for encoding
10198   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10199   emit_int8((unsigned char)0x92);
10200   emit_operand(dst, src);
10201 }
10202 
10203 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10204   assert(VM_Version::supports_evex(), "");
10205   assert(dst != xnoreg, "sanity");
10206   assert(src.isxmmindex(),"expected to be xmm index");
10207   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10208   assert(mask != k0, "instruction will #UD if mask is in k0");
10209   InstructionMark im(this);
10210   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10211   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10212   attributes.reset_is_clear_context();
10213   attributes.set_embedded_opmask_register_specifier(mask);
10214   attributes.set_is_evex_instruction();
10215   // swap src<->dst for encoding
10216   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10217   emit_int8((unsigned char)0x92);
10218   emit_operand(dst, src);
10219 }
10220 
10221 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10222   assert(VM_Version::supports_evex(), "");
10223   assert(mask != k0, "instruction will #UD if mask is in k0");
10224   InstructionMark im(this);
10225   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10226   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10227   attributes.reset_is_clear_context();
10228   attributes.set_embedded_opmask_register_specifier(mask);
10229   attributes.set_is_evex_instruction();
10230   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10231   emit_int8((unsigned char)0xA0);
10232   emit_operand(src, dst);
10233 }
10234 
10235 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10236   assert(VM_Version::supports_evex(), "");
10237   assert(mask != k0, "instruction will #UD if mask is in k0");
10238   InstructionMark im(this);
10239   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10240   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10241   attributes.reset_is_clear_context();
10242   attributes.set_embedded_opmask_register_specifier(mask);
10243   attributes.set_is_evex_instruction();
10244   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10245   emit_int8((unsigned char)0xA0);
10246   emit_operand(src, dst);
10247 }
10248 
10249 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10250   assert(VM_Version::supports_evex(), "");
10251   assert(mask != k0, "instruction will #UD if mask is in k0");
10252   InstructionMark im(this);
10253   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10254   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10255   attributes.reset_is_clear_context();
10256   attributes.set_embedded_opmask_register_specifier(mask);
10257   attributes.set_is_evex_instruction();
10258   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10259   emit_int8((unsigned char)0xA2);
10260   emit_operand(src, dst);
10261 }
10262 
10263 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10264   assert(VM_Version::supports_evex(), "");
10265   assert(mask != k0, "instruction will #UD if mask is in k0");
10266   InstructionMark im(this);
10267   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10268   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10269   attributes.reset_is_clear_context();
10270   attributes.set_embedded_opmask_register_specifier(mask);
10271   attributes.set_is_evex_instruction();
10272   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10273   emit_int8((unsigned char)0xA2);
10274   emit_operand(src, dst);
10275 }
10276 // Carry-Less Multiplication Quadword
10277 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10278   assert(VM_Version::supports_clmul(), "");
10279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10281   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10282 }
10283 
10284 // Carry-Less Multiplication Quadword
10285 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10286   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10287   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10288   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10289   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10290 }
10291 
10292 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10293   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10295   attributes.set_is_evex_instruction();
10296   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10297   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10298 }
10299 
10300 void Assembler::vzeroupper_uncached() {
10301   if (VM_Version::supports_vzeroupper()) {
10302     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10303     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10304     emit_int8(0x77);
10305   }
10306 }
10307 
10308 void Assembler::fld_x(Address adr) {
10309   InstructionMark im(this);
10310   emit_int8((unsigned char)0xDB);
10311   emit_operand32(rbp, adr);
10312 }
10313 
10314 void Assembler::fstp_x(Address adr) {
10315   InstructionMark im(this);
10316   emit_int8((unsigned char)0xDB);
10317   emit_operand32(rdi, adr);
10318 }
10319 
10320 void Assembler::emit_operand32(Register reg, Address adr) {
10321   assert(reg->encoding() < 8, "no extended registers");
10322   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10323   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
10324                adr._rspec);
10325 }
10326 
10327 #ifndef _LP64
10328 // 32bit only pieces of the assembler
10329 
10330 void Assembler::emms() {
10331   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10332   emit_int16(0x0F, 0x77);
10333 }
10334 
10335 void Assembler::vzeroupper() {
10336   vzeroupper_uncached();
10337 }
10338 
10339 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10340   // NO PREFIX AS NEVER 64BIT
10341   InstructionMark im(this);
10342   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10343   emit_data(imm32, rspec, 0);
10344 }
10345 
10346 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10347   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10348   InstructionMark im(this);
10349   emit_int8((unsigned char)0x81);
10350   emit_operand(rdi, src1);
10351   emit_data(imm32, rspec, 0);
10352 }
10353 
10354 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10355 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10356 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10357 void Assembler::cmpxchg8(Address adr) {
10358   InstructionMark im(this);
10359   emit_int16(0x0F, (unsigned char)0xC7);
10360   emit_operand(rcx, adr);
10361 }
10362 
10363 void Assembler::decl(Register dst) {
10364   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10365  emit_int8(0x48 | dst->encoding());
10366 }
10367 
10368 // 64bit doesn't use the x87
10369 
10370 void Assembler::emit_farith(int b1, int b2, int i) {
10371   assert(isByte(b1) && isByte(b2), "wrong opcode");
10372   assert(0 <= i &&  i < 8, "illegal stack offset");
10373   emit_int16(b1, b2 + i);
10374 }
10375 
10376 void Assembler::fabs() {
10377   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10378 }
10379 
10380 void Assembler::fadd(int i) {
10381   emit_farith(0xD8, 0xC0, i);
10382 }
10383 
10384 void Assembler::fadd_d(Address src) {
10385   InstructionMark im(this);
10386   emit_int8((unsigned char)0xDC);
10387   emit_operand32(rax, src);
10388 }
10389 
10390 void Assembler::fadd_s(Address src) {
10391   InstructionMark im(this);
10392   emit_int8((unsigned char)0xD8);
10393   emit_operand32(rax, src);
10394 }
10395 
10396 void Assembler::fadda(int i) {
10397   emit_farith(0xDC, 0xC0, i);
10398 }
10399 
10400 void Assembler::faddp(int i) {
10401   emit_farith(0xDE, 0xC0, i);
10402 }
10403 
10404 void Assembler::fchs() {
10405   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10406 }
10407 
10408 void Assembler::fcom(int i) {
10409   emit_farith(0xD8, 0xD0, i);
10410 }
10411 
10412 void Assembler::fcomp(int i) {
10413   emit_farith(0xD8, 0xD8, i);
10414 }
10415 
10416 void Assembler::fcomp_d(Address src) {
10417   InstructionMark im(this);
10418   emit_int8((unsigned char)0xDC);
10419   emit_operand32(rbx, src);
10420 }
10421 
10422 void Assembler::fcomp_s(Address src) {
10423   InstructionMark im(this);
10424   emit_int8((unsigned char)0xD8);
10425   emit_operand32(rbx, src);
10426 }
10427 
10428 void Assembler::fcompp() {
10429   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10430 }
10431 
10432 void Assembler::fcos() {
10433   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10434 }
10435 
10436 void Assembler::fdecstp() {
10437   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10438 }
10439 
10440 void Assembler::fdiv(int i) {
10441   emit_farith(0xD8, 0xF0, i);
10442 }
10443 
10444 void Assembler::fdiv_d(Address src) {
10445   InstructionMark im(this);
10446   emit_int8((unsigned char)0xDC);
10447   emit_operand32(rsi, src);
10448 }
10449 
10450 void Assembler::fdiv_s(Address src) {
10451   InstructionMark im(this);
10452   emit_int8((unsigned char)0xD8);
10453   emit_operand32(rsi, src);
10454 }
10455 
10456 void Assembler::fdiva(int i) {
10457   emit_farith(0xDC, 0xF8, i);
10458 }
10459 
10460 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10461 //       is erroneous for some of the floating-point instructions below.
10462 
10463 void Assembler::fdivp(int i) {
10464   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10465 }
10466 
10467 void Assembler::fdivr(int i) {
10468   emit_farith(0xD8, 0xF8, i);
10469 }
10470 
10471 void Assembler::fdivr_d(Address src) {
10472   InstructionMark im(this);
10473   emit_int8((unsigned char)0xDC);
10474   emit_operand32(rdi, src);
10475 }
10476 
10477 void Assembler::fdivr_s(Address src) {
10478   InstructionMark im(this);
10479   emit_int8((unsigned char)0xD8);
10480   emit_operand32(rdi, src);
10481 }
10482 
10483 void Assembler::fdivra(int i) {
10484   emit_farith(0xDC, 0xF0, i);
10485 }
10486 
10487 void Assembler::fdivrp(int i) {
10488   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10489 }
10490 
10491 void Assembler::ffree(int i) {
10492   emit_farith(0xDD, 0xC0, i);
10493 }
10494 
10495 void Assembler::fild_d(Address adr) {
10496   InstructionMark im(this);
10497   emit_int8((unsigned char)0xDF);
10498   emit_operand32(rbp, adr);
10499 }
10500 
10501 void Assembler::fild_s(Address adr) {
10502   InstructionMark im(this);
10503   emit_int8((unsigned char)0xDB);
10504   emit_operand32(rax, adr);
10505 }
10506 
10507 void Assembler::fincstp() {
10508   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10509 }
10510 
10511 void Assembler::finit() {
10512   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10513 }
10514 
10515 void Assembler::fist_s(Address adr) {
10516   InstructionMark im(this);
10517   emit_int8((unsigned char)0xDB);
10518   emit_operand32(rdx, adr);
10519 }
10520 
10521 void Assembler::fistp_d(Address adr) {
10522   InstructionMark im(this);
10523   emit_int8((unsigned char)0xDF);
10524   emit_operand32(rdi, adr);
10525 }
10526 
10527 void Assembler::fistp_s(Address adr) {
10528   InstructionMark im(this);
10529   emit_int8((unsigned char)0xDB);
10530   emit_operand32(rbx, adr);
10531 }
10532 
10533 void Assembler::fld1() {
10534   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10535 }
10536 
10537 void Assembler::fld_d(Address adr) {
10538   InstructionMark im(this);
10539   emit_int8((unsigned char)0xDD);
10540   emit_operand32(rax, adr);
10541 }
10542 
10543 void Assembler::fld_s(Address adr) {
10544   InstructionMark im(this);
10545   emit_int8((unsigned char)0xD9);
10546   emit_operand32(rax, adr);
10547 }
10548 
10549 
10550 void Assembler::fld_s(int index) {
10551   emit_farith(0xD9, 0xC0, index);
10552 }
10553 
10554 void Assembler::fldcw(Address src) {
10555   InstructionMark im(this);
10556   emit_int8((unsigned char)0xD9);
10557   emit_operand32(rbp, src);
10558 }
10559 
10560 void Assembler::fldenv(Address src) {
10561   InstructionMark im(this);
10562   emit_int8((unsigned char)0xD9);
10563   emit_operand32(rsp, src);
10564 }
10565 
10566 void Assembler::fldlg2() {
10567   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10568 }
10569 
10570 void Assembler::fldln2() {
10571   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10572 }
10573 
10574 void Assembler::fldz() {
10575   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10576 }
10577 
10578 void Assembler::flog() {
10579   fldln2();
10580   fxch();
10581   fyl2x();
10582 }
10583 
10584 void Assembler::flog10() {
10585   fldlg2();
10586   fxch();
10587   fyl2x();
10588 }
10589 
10590 void Assembler::fmul(int i) {
10591   emit_farith(0xD8, 0xC8, i);
10592 }
10593 
10594 void Assembler::fmul_d(Address src) {
10595   InstructionMark im(this);
10596   emit_int8((unsigned char)0xDC);
10597   emit_operand32(rcx, src);
10598 }
10599 
10600 void Assembler::fmul_s(Address src) {
10601   InstructionMark im(this);
10602   emit_int8((unsigned char)0xD8);
10603   emit_operand32(rcx, src);
10604 }
10605 
10606 void Assembler::fmula(int i) {
10607   emit_farith(0xDC, 0xC8, i);
10608 }
10609 
10610 void Assembler::fmulp(int i) {
10611   emit_farith(0xDE, 0xC8, i);
10612 }
10613 
10614 void Assembler::fnsave(Address dst) {
10615   InstructionMark im(this);
10616   emit_int8((unsigned char)0xDD);
10617   emit_operand32(rsi, dst);
10618 }
10619 
10620 void Assembler::fnstcw(Address src) {
10621   InstructionMark im(this);
10622   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10623   emit_operand32(rdi, src);
10624 }
10625 
10626 void Assembler::fnstsw_ax() {
10627   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10628 }
10629 
10630 void Assembler::fprem() {
10631   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10632 }
10633 
10634 void Assembler::fprem1() {
10635   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10636 }
10637 
10638 void Assembler::frstor(Address src) {
10639   InstructionMark im(this);
10640   emit_int8((unsigned char)0xDD);
10641   emit_operand32(rsp, src);
10642 }
10643 
10644 void Assembler::fsin() {
10645   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10646 }
10647 
10648 void Assembler::fsqrt() {
10649   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
10650 }
10651 
10652 void Assembler::fst_d(Address adr) {
10653   InstructionMark im(this);
10654   emit_int8((unsigned char)0xDD);
10655   emit_operand32(rdx, adr);
10656 }
10657 
10658 void Assembler::fst_s(Address adr) {
10659   InstructionMark im(this);
10660   emit_int8((unsigned char)0xD9);
10661   emit_operand32(rdx, adr);
10662 }
10663 
10664 void Assembler::fstp_d(Address adr) {
10665   InstructionMark im(this);
10666   emit_int8((unsigned char)0xDD);
10667   emit_operand32(rbx, adr);
10668 }
10669 
10670 void Assembler::fstp_d(int index) {
10671   emit_farith(0xDD, 0xD8, index);
10672 }
10673 
10674 void Assembler::fstp_s(Address adr) {
10675   InstructionMark im(this);
10676   emit_int8((unsigned char)0xD9);
10677   emit_operand32(rbx, adr);
10678 }
10679 
10680 void Assembler::fsub(int i) {
10681   emit_farith(0xD8, 0xE0, i);
10682 }
10683 
10684 void Assembler::fsub_d(Address src) {
10685   InstructionMark im(this);
10686   emit_int8((unsigned char)0xDC);
10687   emit_operand32(rsp, src);
10688 }
10689 
10690 void Assembler::fsub_s(Address src) {
10691   InstructionMark im(this);
10692   emit_int8((unsigned char)0xD8);
10693   emit_operand32(rsp, src);
10694 }
10695 
10696 void Assembler::fsuba(int i) {
10697   emit_farith(0xDC, 0xE8, i);
10698 }
10699 
10700 void Assembler::fsubp(int i) {
10701   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
10702 }
10703 
10704 void Assembler::fsubr(int i) {
10705   emit_farith(0xD8, 0xE8, i);
10706 }
10707 
10708 void Assembler::fsubr_d(Address src) {
10709   InstructionMark im(this);
10710   emit_int8((unsigned char)0xDC);
10711   emit_operand32(rbp, src);
10712 }
10713 
10714 void Assembler::fsubr_s(Address src) {
10715   InstructionMark im(this);
10716   emit_int8((unsigned char)0xD8);
10717   emit_operand32(rbp, src);
10718 }
10719 
10720 void Assembler::fsubra(int i) {
10721   emit_farith(0xDC, 0xE0, i);
10722 }
10723 
10724 void Assembler::fsubrp(int i) {
10725   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
10726 }
10727 
10728 void Assembler::ftan() {
10729   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
10730 }
10731 
10732 void Assembler::ftst() {
10733   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
10734 }
10735 
10736 void Assembler::fucomi(int i) {
10737   // make sure the instruction is supported (introduced for P6, together with cmov)
10738   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10739   emit_farith(0xDB, 0xE8, i);
10740 }
10741 
10742 void Assembler::fucomip(int i) {
10743   // make sure the instruction is supported (introduced for P6, together with cmov)
10744   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10745   emit_farith(0xDF, 0xE8, i);
10746 }
10747 
10748 void Assembler::fwait() {
10749   emit_int8((unsigned char)0x9B);
10750 }
10751 
10752 void Assembler::fxch(int i) {
10753   emit_farith(0xD9, 0xC8, i);
10754 }
10755 
10756 void Assembler::fyl2x() {
10757   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
10758 }
10759 
10760 void Assembler::frndint() {
10761   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
10762 }
10763 
10764 void Assembler::f2xm1() {
10765   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
10766 }
10767 
10768 void Assembler::fldl2e() {
10769   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
10770 }
10771 #endif // !_LP64
10772 
10773 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
10774 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
10775 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
10776 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
10777 
10778 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
10779 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10780   if (pre > 0) {
10781     emit_int8(simd_pre[pre]);
10782   }
10783   if (rex_w) {
10784     prefixq(adr, xreg);
10785   } else {
10786     prefix(adr, xreg);
10787   }
10788   if (opc > 0) {
10789     emit_int8(0x0F);
10790     int opc2 = simd_opc[opc];
10791     if (opc2 > 0) {
10792       emit_int8(opc2);
10793     }
10794   }
10795 }
10796 
10797 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10798   if (pre > 0) {
10799     emit_int8(simd_pre[pre]);
10800   }
10801   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
10802   if (opc > 0) {
10803     emit_int8(0x0F);
10804     int opc2 = simd_opc[opc];
10805     if (opc2 > 0) {
10806       emit_int8(opc2);
10807     }
10808   }
10809   return encode;
10810 }
10811 
10812 
10813 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
10814   int vector_len = _attributes->get_vector_len();
10815   bool vex_w = _attributes->is_rex_vex_w();
10816   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
10817     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
10818     byte1 = (~byte1) & 0xE0;
10819     byte1 |= opc;
10820 
10821     int byte2 = ((~nds_enc) & 0xf) << 3;
10822     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
10823 
10824     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
10825   } else {
10826     int byte1 = vex_r ? VEX_R : 0;
10827     byte1 = (~byte1) & 0x80;
10828     byte1 |= ((~nds_enc) & 0xf) << 3;
10829     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
10830     emit_int16((unsigned char)VEX_2bytes, byte1);
10831   }
10832 }
10833 
10834 // This is a 4 byte encoding
10835 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
10836   // EVEX 0x62 prefix
10837   // byte1 = EVEX_4bytes;
10838 
10839   bool vex_w = _attributes->is_rex_vex_w();
10840   int evex_encoding = (vex_w ? VEX_W : 0);
10841   // EVEX.b is not currently used for broadcast of single element or data rounding modes
10842   _attributes->set_evex_encoding(evex_encoding);
10843 
10844   // P0: byte 2, initialized to RXBR`00mm
10845   // instead of not'd
10846   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
10847   byte2 = (~byte2) & 0xF0;
10848   // confine opc opcode extensions in mm bits to lower two bits
10849   // of form {0F, 0F_38, 0F_3A}
10850   byte2 |= opc;
10851 
10852   // P1: byte 3 as Wvvvv1pp
10853   int byte3 = ((~nds_enc) & 0xf) << 3;
10854   // p[10] is always 1
10855   byte3 |= EVEX_F;
10856   byte3 |= (vex_w & 1) << 7;
10857   // confine pre opcode extensions in pp bits to lower two bits
10858   // of form {66, F3, F2}
10859   byte3 |= pre;
10860 
10861   // P2: byte 4 as zL'Lbv'aaa
10862   // kregs are implemented in the low 3 bits as aaa
10863   int byte4 = (_attributes->is_no_reg_mask()) ?
10864               0 :
10865               _attributes->get_embedded_opmask_register_specifier();
10866   // EVEX.v` for extending EVEX.vvvv or VIDX
10867   byte4 |= (evex_v ? 0: EVEX_V);
10868   // third EXEC.b for broadcast actions
10869   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
10870   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
10871   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
10872   // last is EVEX.z for zero/merge actions
10873   if (_attributes->is_no_reg_mask() == false &&
10874       _attributes->get_embedded_opmask_register_specifier() != 0) {
10875     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
10876   }
10877 
10878   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
10879 }
10880 
10881 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
10882   bool vex_r = (xreg_enc & 8) == 8;
10883   bool vex_b = adr.base_needs_rex();
10884   bool vex_x;
10885   if (adr.isxmmindex()) {
10886     vex_x = adr.xmmindex_needs_rex();
10887   } else {
10888     vex_x = adr.index_needs_rex();
10889   }
10890   set_attributes(attributes);
10891   attributes->set_current_assembler(this);
10892 
10893   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
10894   // is allowed in legacy mode and has resources which will fit in it.
10895   // Pure EVEX instructions will have is_evex_instruction set in their definition.
10896   if (!attributes->is_legacy_mode()) {
10897     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
10898       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
10899           attributes->set_is_legacy_mode();
10900       }
10901     }
10902   }
10903 
10904   if (UseAVX > 2) {
10905     assert(((!attributes->uses_vl()) ||
10906             (attributes->get_vector_len() == AVX_512bit) ||
10907             (!_legacy_mode_vl) ||
10908             (attributes->is_legacy_mode())),"XMM register should be 0-15");
10909     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
10910   }
10911 
10912   clear_managed();
10913   if (UseAVX > 2 && !attributes->is_legacy_mode())
10914   {
10915     bool evex_r = (xreg_enc >= 16);
10916     bool evex_v;
10917     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
10918     if (adr.isxmmindex())  {
10919       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
10920     } else {
10921       evex_v = (nds_enc >= 16);
10922     }
10923     attributes->set_is_evex_instruction();
10924     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
10925   } else {
10926     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
10927       attributes->set_rex_vex_w(false);
10928     }
10929     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
10930   }
10931 }
10932 
10933 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
10934   bool vex_r = (dst_enc & 8) == 8;
10935   bool vex_b = (src_enc & 8) == 8;
10936   bool vex_x = false;
10937   set_attributes(attributes);
10938   attributes->set_current_assembler(this);
10939 
10940   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
10941   // is allowed in legacy mode and has resources which will fit in it.
10942   // Pure EVEX instructions will have is_evex_instruction set in their definition.
10943   if (!attributes->is_legacy_mode()) {
10944     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
10945       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
10946           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
10947           attributes->set_is_legacy_mode();
10948       }
10949     }
10950   }
10951 
10952   if (UseAVX > 2) {
10953     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
10954     // Instruction with uses_vl true are vector instructions
10955     // All the vector instructions with AVX_512bit length can have legacy_mode as false
10956     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
10957     // Rest all should have legacy_mode set as true
10958     assert(((!attributes->uses_vl()) ||
10959             (attributes->get_vector_len() == AVX_512bit) ||
10960             (!_legacy_mode_vl) ||
10961             (attributes->is_legacy_mode())),"XMM register should be 0-15");
10962     // Instruction with legacy_mode true should have dst, nds and src < 15
10963     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
10964   }
10965 
10966   clear_managed();
10967   if (UseAVX > 2 && !attributes->is_legacy_mode())
10968   {
10969     bool evex_r = (dst_enc >= 16);
10970     bool evex_v = (nds_enc >= 16);
10971     // can use vex_x as bank extender on rm encoding
10972     vex_x = (src_enc >= 16);
10973     attributes->set_is_evex_instruction();
10974     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
10975   } else {
10976     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
10977       attributes->set_rex_vex_w(false);
10978     }
10979     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
10980   }
10981 
10982   // return modrm byte components for operands
10983   return (((dst_enc & 7) << 3) | (src_enc & 7));
10984 }
10985 
10986 
10987 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
10988                             VexOpcode opc, InstructionAttr *attributes) {
10989   if (UseAVX > 0) {
10990     int xreg_enc = xreg->encoding();
10991     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
10992     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
10993   } else {
10994     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
10995     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
10996   }
10997 }
10998 
10999 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11000                                       VexOpcode opc, InstructionAttr *attributes) {
11001   int dst_enc = dst->encoding();
11002   int src_enc = src->encoding();
11003   if (UseAVX > 0) {
11004     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11005     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11006   } else {
11007     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11008     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11009   }
11010 }
11011 
11012 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11013   assert(VM_Version::supports_avx(), "");
11014   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11015   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11016   emit_int16(0x5F, (0xC0 | encode));
11017 }
11018 
11019 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11020   assert(VM_Version::supports_avx(), "");
11021   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11022   attributes.set_rex_vex_w_reverted();
11023   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11024   emit_int16(0x5F, (0xC0 | encode));
11025 }
11026 
11027 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11028   assert(VM_Version::supports_avx(), "");
11029   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11030   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11031   emit_int16(0x5D, (0xC0 | encode));
11032 }
11033 
11034 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11035   assert(VM_Version::supports_avx(), "");
11036   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11037   attributes.set_rex_vex_w_reverted();
11038   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11039   emit_int16(0x5D, (0xC0 | encode));
11040 }
11041 
11042 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11043   assert(VM_Version::supports_avx(), "");
11044   assert(vector_len <= AVX_256bit, "");
11045   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11046   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11047   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11048 }
11049 
11050 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11051   assert(VM_Version::supports_avx(), "");
11052   assert(vector_len <= AVX_256bit, "");
11053   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11054   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11055   int src2_enc = src2->encoding();
11056   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11057 }
11058 
11059 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11060   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11061   assert(vector_len <= AVX_256bit, "");
11062   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11063   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11064   int src2_enc = src2->encoding();
11065   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11066 }
11067 
11068 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11069   assert(VM_Version::supports_avx2(), "");
11070   assert(vector_len <= AVX_256bit, "");
11071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11072   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11073   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11074 }
11075 
11076 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11077   assert(VM_Version::supports_avx(), "");
11078   assert(vector_len <= AVX_256bit, "");
11079   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11080   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11081   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11082 }
11083 
11084 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11085                         ComparisonPredicateFP comparison, int vector_len) {
11086   assert(VM_Version::supports_evex(), "");
11087   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11088   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11089   attributes.set_is_evex_instruction();
11090   attributes.set_embedded_opmask_register_specifier(mask);
11091   attributes.reset_is_clear_context();
11092   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11093   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11094 }
11095 
11096 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11097                         ComparisonPredicateFP comparison, int vector_len) {
11098   assert(VM_Version::supports_evex(), "");
11099   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11100   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11101   attributes.set_is_evex_instruction();
11102   attributes.set_embedded_opmask_register_specifier(mask);
11103   attributes.reset_is_clear_context();
11104   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11105   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11106 }
11107 
11108 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11109   assert(VM_Version::supports_sse4_1(), "");
11110   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11112   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11113   emit_int16(0x14, (0xC0 | encode));
11114 }
11115 
11116 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11117   assert(VM_Version::supports_sse4_1(), "");
11118   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11119   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11120   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11121   emit_int16(0x15, (0xC0 | encode));
11122 }
11123 
11124 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11125   assert(VM_Version::supports_sse4_1(), "");
11126   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11127   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11128   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11129   emit_int16(0x10, (0xC0 | encode));
11130 }
11131 
11132 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11133   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11134   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11135   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11136   int src2_enc = src2->encoding();
11137   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11138 }
11139 
11140 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11142   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11143   emit_int24(0x0C, (0xC0 | encode), imm8);
11144 }
11145 
11146 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11147   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11148   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11149   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11150   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11151   emit_int16(0x64, (0xC0 | encode));
11152 }
11153 
11154 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11155   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11156   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11157   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11158   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11159   emit_int16(0x65, (0xC0 | encode));
11160 }
11161 
11162 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11163   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11164   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11165   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11166   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11167   emit_int16(0x66, (0xC0 | encode));
11168 }
11169 
11170 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11171   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11172   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11173   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11174   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11175   emit_int16(0x37, (0xC0 | encode));
11176 }
11177 
11178 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11179                         int comparison, bool is_signed, int vector_len) {
11180   assert(VM_Version::supports_evex(), "");
11181   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11182   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11184   attributes.set_is_evex_instruction();
11185   attributes.set_embedded_opmask_register_specifier(mask);
11186   attributes.reset_is_clear_context();
11187   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11188   int opcode = is_signed ? 0x1F : 0x1E;
11189   emit_int24(opcode, (0xC0 | encode), comparison);
11190 }
11191 
11192 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11193                         int comparison, bool is_signed, int vector_len) {
11194   assert(VM_Version::supports_evex(), "");
11195   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11196   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11197   InstructionMark im(this);
11198   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11199   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11200   attributes.set_is_evex_instruction();
11201   attributes.set_embedded_opmask_register_specifier(mask);
11202   attributes.reset_is_clear_context();
11203   int dst_enc = kdst->encoding();
11204   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11205   int opcode = is_signed ? 0x1F : 0x1E;
11206   emit_int8((unsigned char)opcode);
11207   emit_operand(as_Register(dst_enc), src);
11208   emit_int8((unsigned char)comparison);
11209 }
11210 
11211 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11212                         int comparison, bool is_signed, int vector_len) {
11213   assert(VM_Version::supports_evex(), "");
11214   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11215   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11216   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11217   attributes.set_is_evex_instruction();
11218   attributes.set_embedded_opmask_register_specifier(mask);
11219   attributes.reset_is_clear_context();
11220   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11221   int opcode = is_signed ? 0x1F : 0x1E;
11222   emit_int24(opcode, (0xC0 | encode), comparison);
11223 }
11224 
11225 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11226                         int comparison, bool is_signed, int vector_len) {
11227   assert(VM_Version::supports_evex(), "");
11228   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11229   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11230   InstructionMark im(this);
11231   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11232   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11233   attributes.set_is_evex_instruction();
11234   attributes.set_embedded_opmask_register_specifier(mask);
11235   attributes.reset_is_clear_context();
11236   int dst_enc = kdst->encoding();
11237   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11238   int opcode = is_signed ? 0x1F : 0x1E;
11239   emit_int8((unsigned char)opcode);
11240   emit_operand(as_Register(dst_enc), src);
11241   emit_int8((unsigned char)comparison);
11242 }
11243 
11244 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11245                         int comparison, bool is_signed, int vector_len) {
11246   assert(VM_Version::supports_evex(), "");
11247   assert(VM_Version::supports_avx512bw(), "");
11248   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11249   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11250   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11251   attributes.set_is_evex_instruction();
11252   attributes.set_embedded_opmask_register_specifier(mask);
11253   attributes.reset_is_clear_context();
11254   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11255   int opcode = is_signed ? 0x3F : 0x3E;
11256   emit_int24(opcode, (0xC0 | encode), comparison);
11257 }
11258 
11259 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11260                         int comparison, bool is_signed, int vector_len) {
11261   assert(VM_Version::supports_evex(), "");
11262   assert(VM_Version::supports_avx512bw(), "");
11263   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11264   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11265   InstructionMark im(this);
11266   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11267   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11268   attributes.set_is_evex_instruction();
11269   attributes.set_embedded_opmask_register_specifier(mask);
11270   attributes.reset_is_clear_context();
11271   int dst_enc = kdst->encoding();
11272   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11273   int opcode = is_signed ? 0x3F : 0x3E;
11274   emit_int8((unsigned char)opcode);
11275   emit_operand(as_Register(dst_enc), src);
11276   emit_int8((unsigned char)comparison);
11277 }
11278 
11279 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11280                         int comparison, bool is_signed, int vector_len) {
11281   assert(VM_Version::supports_evex(), "");
11282   assert(VM_Version::supports_avx512bw(), "");
11283   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11284   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11285   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11286   attributes.set_is_evex_instruction();
11287   attributes.set_embedded_opmask_register_specifier(mask);
11288   attributes.reset_is_clear_context();
11289   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11290   int opcode = is_signed ? 0x3F : 0x3E;
11291   emit_int24(opcode, (0xC0 | encode), comparison);
11292 }
11293 
11294 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11295                         int comparison, bool is_signed, int vector_len) {
11296   assert(VM_Version::supports_evex(), "");
11297   assert(VM_Version::supports_avx512bw(), "");
11298   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11299   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11300   InstructionMark im(this);
11301   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11302   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11303   attributes.set_is_evex_instruction();
11304   attributes.set_embedded_opmask_register_specifier(mask);
11305   attributes.reset_is_clear_context();
11306   int dst_enc = kdst->encoding();
11307   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11308   int opcode = is_signed ? 0x3F : 0x3E;
11309   emit_int8((unsigned char)opcode);
11310   emit_operand(as_Register(dst_enc), src);
11311   emit_int8((unsigned char)comparison);
11312 }
11313 
11314 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11315   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11316   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11317   attributes.set_is_evex_instruction();
11318   attributes.set_embedded_opmask_register_specifier(mask);
11319   if (merge) {
11320     attributes.reset_is_clear_context();
11321   }
11322   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11323   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11324 }
11325 
11326 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11327   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11328   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11329   attributes.set_is_evex_instruction();
11330   attributes.set_embedded_opmask_register_specifier(mask);
11331   if (merge) {
11332     attributes.reset_is_clear_context();
11333   }
11334   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11335   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11336 }
11337 
11338 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11339   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11340   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11341   attributes.set_is_evex_instruction();
11342   attributes.set_embedded_opmask_register_specifier(mask);
11343   if (merge) {
11344     attributes.reset_is_clear_context();
11345   }
11346   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11347   emit_int16(0x14, (0xC0 | encode));
11348 }
11349 
11350 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11351   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11352   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11353   attributes.set_is_evex_instruction();
11354   attributes.set_embedded_opmask_register_specifier(mask);
11355   if (merge) {
11356     attributes.reset_is_clear_context();
11357   }
11358   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11359   emit_int16(0x14, (0xC0 | encode));
11360 }
11361 
11362 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11363   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11364   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11365   attributes.set_is_evex_instruction();
11366   attributes.set_embedded_opmask_register_specifier(mask);
11367   if (merge) {
11368     attributes.reset_is_clear_context();
11369   }
11370   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11371   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11372 }
11373 
11374 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11375   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11376   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11377   attributes.set_is_evex_instruction();
11378   attributes.set_embedded_opmask_register_specifier(mask);
11379   if (merge) {
11380     attributes.reset_is_clear_context();
11381   }
11382   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11383   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11384 }
11385 
11386 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11387   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11389   attributes.set_is_evex_instruction();
11390   attributes.set_embedded_opmask_register_specifier(mask);
11391   if (merge) {
11392     attributes.reset_is_clear_context();
11393   }
11394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11395   emit_int16(0x15, (0xC0 | encode));
11396 }
11397 
11398 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11399   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11400   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11401   attributes.set_is_evex_instruction();
11402   attributes.set_embedded_opmask_register_specifier(mask);
11403   if (merge) {
11404     attributes.reset_is_clear_context();
11405   }
11406   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11407   emit_int16(0x15, (0xC0 | encode));
11408 }
11409 
11410 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11411   assert(VM_Version::supports_avx(), "");
11412   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11413   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11414   int mask_enc = mask->encoding();
11415   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11416 }
11417 
11418 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11419   assert(VM_Version::supports_evex(), "");
11420   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11421   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11422   attributes.set_is_evex_instruction();
11423   attributes.set_embedded_opmask_register_specifier(mask);
11424   if (merge) {
11425     attributes.reset_is_clear_context();
11426   }
11427   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11428   emit_int16(0x65, (0xC0 | encode));
11429 }
11430 
11431 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11432   assert(VM_Version::supports_evex(), "");
11433   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11435   attributes.set_is_evex_instruction();
11436   attributes.set_embedded_opmask_register_specifier(mask);
11437   if (merge) {
11438     attributes.reset_is_clear_context();
11439   }
11440   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11441   emit_int16(0x65, (0xC0 | encode));
11442 }
11443 
11444 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11445   assert(VM_Version::supports_evex(), "");
11446   assert(VM_Version::supports_avx512bw(), "");
11447   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11448   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11449   attributes.set_is_evex_instruction();
11450   attributes.set_embedded_opmask_register_specifier(mask);
11451   if (merge) {
11452     attributes.reset_is_clear_context();
11453   }
11454   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11455   emit_int16(0x66, (0xC0 | encode));
11456 }
11457 
11458 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11459   assert(VM_Version::supports_evex(), "");
11460   assert(VM_Version::supports_avx512bw(), "");
11461   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11462   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11463   attributes.set_is_evex_instruction();
11464   attributes.set_embedded_opmask_register_specifier(mask);
11465   if (merge) {
11466     attributes.reset_is_clear_context();
11467   }
11468   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11469   emit_int16(0x66, (0xC0 | encode));
11470 }
11471 
11472 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11473   assert(VM_Version::supports_evex(), "");
11474   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11475   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11476   attributes.set_is_evex_instruction();
11477   attributes.set_embedded_opmask_register_specifier(mask);
11478   if (merge) {
11479     attributes.reset_is_clear_context();
11480   }
11481   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11482   emit_int16(0x64, (0xC0 | encode));
11483 }
11484 
11485 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11486   assert(VM_Version::supports_evex(), "");
11487   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11488   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11489   attributes.set_is_evex_instruction();
11490   attributes.set_embedded_opmask_register_specifier(mask);
11491   if (merge) {
11492     attributes.reset_is_clear_context();
11493   }
11494   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11495   emit_int16(0x64, (0xC0 | encode));
11496 }
11497 
11498 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11499   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11500   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11501   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11502   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11503 }
11504 
11505 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11506   assert(VM_Version::supports_bmi2(), "");
11507   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11508   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11509   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11510 }
11511 
11512 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11513   assert(VM_Version::supports_bmi2(), "");
11514   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11515   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11516   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11517 }
11518 
11519 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11520   assert(VM_Version::supports_bmi2(), "");
11521   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11522   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11523   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11524 }
11525 
11526 void Assembler::shrxq(Register dst, Register src1, Register src2) {
11527   assert(VM_Version::supports_bmi2(), "");
11528   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11529   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11530   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11531 }
11532 
11533 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
11534   assert(VM_Version::supports_avx512vldq(), "");
11535   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11536   attributes.set_is_evex_instruction();
11537   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11538   emit_int16(0x39, (0xC0 | encode));
11539 }
11540 
11541 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
11542   assert(VM_Version::supports_avx512vldq(), "");
11543   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11544   attributes.set_is_evex_instruction();
11545   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11546   emit_int16(0x39, (0xC0 | encode));
11547 }
11548 
11549 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
11550   assert(VM_Version::supports_avx512vlbw(), "");
11551   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11552   attributes.set_is_evex_instruction();
11553   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11554   emit_int16(0x29, (0xC0 | encode));
11555 }
11556 
11557 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
11558   assert(VM_Version::supports_avx512vlbw(), "");
11559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11560   attributes.set_is_evex_instruction();
11561   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11562   emit_int16(0x29, (0xC0 | encode));
11563 }
11564 
11565 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
11566   assert(VM_Version::supports_avx512vldq(), "");
11567   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11568   attributes.set_is_evex_instruction();
11569   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11570   emit_int16(0x38, (0xC0 | encode));
11571 }
11572 
11573 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
11574   assert(VM_Version::supports_avx512vldq(), "");
11575   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11576   attributes.set_is_evex_instruction();
11577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11578   emit_int16(0x38, (0xC0 | encode));
11579 }
11580 
11581 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
11582   assert(VM_Version::supports_avx512vlbw(), "");
11583   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11584   attributes.set_is_evex_instruction();
11585   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11586   emit_int16(0x28, (0xC0 | encode));
11587 }
11588 
11589 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
11590   assert(VM_Version::supports_avx512vlbw(), "");
11591   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11592   attributes.set_is_evex_instruction();
11593   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11594   emit_int16(0x28, (0xC0 | encode));
11595 }
11596 #ifndef _LP64
11597 
11598 void Assembler::incl(Register dst) {
11599   // Don't use it directly. Use MacroAssembler::incrementl() instead.
11600   emit_int8(0x40 | dst->encoding());
11601 }
11602 
11603 void Assembler::lea(Register dst, Address src) {
11604   leal(dst, src);
11605 }
11606 
11607 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
11608   InstructionMark im(this);
11609   emit_int8((unsigned char)0xC7);
11610   emit_operand(rax, dst);
11611   emit_data((int)imm32, rspec, 0);
11612 }
11613 
11614 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
11615   InstructionMark im(this);
11616   int encode = prefix_and_encode(dst->encoding());
11617   emit_int8((0xB8 | encode));
11618   emit_data((int)imm32, rspec, 0);
11619 }
11620 
11621 void Assembler::popa() { // 32bit
11622   emit_int8(0x61);
11623 }
11624 
11625 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
11626   InstructionMark im(this);
11627   emit_int8(0x68);
11628   emit_data(imm32, rspec, 0);
11629 }
11630 
11631 void Assembler::pusha() { // 32bit
11632   emit_int8(0x60);
11633 }
11634 
11635 void Assembler::set_byte_if_not_zero(Register dst) {
11636   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
11637 }
11638 
11639 #else // LP64
11640 
11641 void Assembler::set_byte_if_not_zero(Register dst) {
11642   int enc = prefix_and_encode(dst->encoding(), true);
11643   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
11644 }
11645 
11646 // 64bit only pieces of the assembler
11647 // This should only be used by 64bit instructions that can use rip-relative
11648 // it cannot be used by instructions that want an immediate value.
11649 
11650 bool Assembler::reachable(AddressLiteral adr) {
11651   int64_t disp;
11652   relocInfo::relocType relocType = adr.reloc();
11653 
11654   // None will force a 64bit literal to the code stream. Likely a placeholder
11655   // for something that will be patched later and we need to certain it will
11656   // always be reachable.
11657   if (relocType == relocInfo::none) {
11658     return false;
11659   }
11660   if (relocType == relocInfo::internal_word_type) {
11661     // This should be rip relative and easily reachable.
11662     return true;
11663   }
11664   if (relocType == relocInfo::virtual_call_type ||
11665       relocType == relocInfo::opt_virtual_call_type ||
11666       relocType == relocInfo::static_call_type ||
11667       relocType == relocInfo::static_stub_type ) {
11668     // This should be rip relative within the code cache and easily
11669     // reachable until we get huge code caches. (At which point
11670     // ic code is going to have issues).
11671     return true;
11672   }
11673   if (relocType != relocInfo::external_word_type &&
11674       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
11675       relocType != relocInfo::poll_type &&         // relocs to identify them
11676       relocType != relocInfo::runtime_call_type ) {
11677     return false;
11678   }
11679 
11680   // Stress the correction code
11681   if (ForceUnreachable) {
11682     // Must be runtimecall reloc, see if it is in the codecache
11683     // Flipping stuff in the codecache to be unreachable causes issues
11684     // with things like inline caches where the additional instructions
11685     // are not handled.
11686     if (CodeCache::find_blob(adr._target) == NULL) {
11687       return false;
11688     }
11689   }
11690   // For external_word_type/runtime_call_type if it is reachable from where we
11691   // are now (possibly a temp buffer) and where we might end up
11692   // anywhere in the codeCache then we are always reachable.
11693   // This would have to change if we ever save/restore shared code
11694   // to be more pessimistic.
11695   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
11696   if (!is_simm32(disp)) return false;
11697   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
11698   if (!is_simm32(disp)) return false;
11699 
11700   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
11701 
11702   // Because rip relative is a disp + address_of_next_instruction and we
11703   // don't know the value of address_of_next_instruction we apply a fudge factor
11704   // to make sure we will be ok no matter the size of the instruction we get placed into.
11705   // We don't have to fudge the checks above here because they are already worst case.
11706 
11707   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
11708   // + 4 because better safe than sorry.
11709   const int fudge = 12 + 4;
11710   if (disp < 0) {
11711     disp -= fudge;
11712   } else {
11713     disp += fudge;
11714   }
11715   return is_simm32(disp);
11716 }
11717 
11718 void Assembler::emit_data64(jlong data,
11719                             relocInfo::relocType rtype,
11720                             int format) {
11721   if (rtype == relocInfo::none) {
11722     emit_int64(data);
11723   } else {
11724     emit_data64(data, Relocation::spec_simple(rtype), format);
11725   }
11726 }
11727 
11728 void Assembler::emit_data64(jlong data,
11729                             RelocationHolder const& rspec,
11730                             int format) {
11731   assert(imm_operand == 0, "default format must be immediate in this file");
11732   assert(imm_operand == format, "must be immediate");
11733   assert(inst_mark() != NULL, "must be inside InstructionMark");
11734   // Do not use AbstractAssembler::relocate, which is not intended for
11735   // embedded words.  Instead, relocate to the enclosing instruction.
11736   code_section()->relocate(inst_mark(), rspec, format);
11737 #ifdef ASSERT
11738   check_relocation(rspec, format);
11739 #endif
11740   emit_int64(data);
11741 }
11742 
11743 void Assembler::prefix(Register reg) {
11744   if (reg->encoding() >= 8) {
11745     prefix(REX_B);
11746   }
11747 }
11748 
11749 void Assembler::prefix(Register dst, Register src, Prefix p) {
11750   if (src->encoding() >= 8) {
11751     p = (Prefix)(p | REX_B);
11752   }
11753   if (dst->encoding() >= 8) {
11754     p = (Prefix)(p | REX_R);
11755   }
11756   if (p != Prefix_EMPTY) {
11757     // do not generate an empty prefix
11758     prefix(p);
11759   }
11760 }
11761 
11762 void Assembler::prefix(Register dst, Address adr, Prefix p) {
11763   if (adr.base_needs_rex()) {
11764     if (adr.index_needs_rex()) {
11765       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
11766     } else {
11767       prefix(REX_B);
11768     }
11769   } else {
11770     if (adr.index_needs_rex()) {
11771       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
11772     }
11773   }
11774   if (dst->encoding() >= 8) {
11775     p = (Prefix)(p | REX_R);
11776   }
11777   if (p != Prefix_EMPTY) {
11778     // do not generate an empty prefix
11779     prefix(p);
11780   }
11781 }
11782 
11783 void Assembler::prefix(Address adr) {
11784   if (adr.base_needs_rex()) {
11785     if (adr.index_needs_rex()) {
11786       prefix(REX_XB);
11787     } else {
11788       prefix(REX_B);
11789     }
11790   } else {
11791     if (adr.index_needs_rex()) {
11792       prefix(REX_X);
11793     }
11794   }
11795 }
11796 
11797 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
11798   if (reg->encoding() < 8) {
11799     if (adr.base_needs_rex()) {
11800       if (adr.index_needs_rex()) {
11801         prefix(REX_XB);
11802       } else {
11803         prefix(REX_B);
11804       }
11805     } else {
11806       if (adr.index_needs_rex()) {
11807         prefix(REX_X);
11808       } else if (byteinst && reg->encoding() >= 4) {
11809         prefix(REX);
11810       }
11811     }
11812   } else {
11813     if (adr.base_needs_rex()) {
11814       if (adr.index_needs_rex()) {
11815         prefix(REX_RXB);
11816       } else {
11817         prefix(REX_RB);
11818       }
11819     } else {
11820       if (adr.index_needs_rex()) {
11821         prefix(REX_RX);
11822       } else {
11823         prefix(REX_R);
11824       }
11825     }
11826   }
11827 }
11828 
11829 void Assembler::prefix(Address adr, XMMRegister reg) {
11830   if (reg->encoding() < 8) {
11831     if (adr.base_needs_rex()) {
11832       if (adr.index_needs_rex()) {
11833         prefix(REX_XB);
11834       } else {
11835         prefix(REX_B);
11836       }
11837     } else {
11838       if (adr.index_needs_rex()) {
11839         prefix(REX_X);
11840       }
11841     }
11842   } else {
11843     if (adr.base_needs_rex()) {
11844       if (adr.index_needs_rex()) {
11845         prefix(REX_RXB);
11846       } else {
11847         prefix(REX_RB);
11848       }
11849     } else {
11850       if (adr.index_needs_rex()) {
11851         prefix(REX_RX);
11852       } else {
11853         prefix(REX_R);
11854       }
11855     }
11856   }
11857 }
11858 
11859 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
11860   if (reg_enc >= 8) {
11861     prefix(REX_B);
11862     reg_enc -= 8;
11863   } else if (byteinst && reg_enc >= 4) {
11864     prefix(REX);
11865   }
11866   return reg_enc;
11867 }
11868 
11869 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
11870   if (dst_enc < 8) {
11871     if (src_enc >= 8) {
11872       prefix(REX_B);
11873       src_enc -= 8;
11874     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
11875       prefix(REX);
11876     }
11877   } else {
11878     if (src_enc < 8) {
11879       prefix(REX_R);
11880     } else {
11881       prefix(REX_RB);
11882       src_enc -= 8;
11883     }
11884     dst_enc -= 8;
11885   }
11886   return dst_enc << 3 | src_enc;
11887 }
11888 
11889 int8_t Assembler::get_prefixq(Address adr) {
11890   int8_t prfx = get_prefixq(adr, rax);
11891   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
11892   return prfx;
11893 }
11894 
11895 int8_t Assembler::get_prefixq(Address adr, Register src) {
11896   int8_t prfx = (int8_t)(REX_W +
11897                          ((int)adr.base_needs_rex()) +
11898                          ((int)adr.index_needs_rex() << 1) +
11899                          ((int)(src->encoding() >= 8) << 2));
11900 #ifdef ASSERT
11901   if (src->encoding() < 8) {
11902     if (adr.base_needs_rex()) {
11903       if (adr.index_needs_rex()) {
11904         assert(prfx == REX_WXB, "must be");
11905       } else {
11906         assert(prfx == REX_WB, "must be");
11907       }
11908     } else {
11909       if (adr.index_needs_rex()) {
11910         assert(prfx == REX_WX, "must be");
11911       } else {
11912         assert(prfx == REX_W, "must be");
11913       }
11914     }
11915   } else {
11916     if (adr.base_needs_rex()) {
11917       if (adr.index_needs_rex()) {
11918         assert(prfx == REX_WRXB, "must be");
11919       } else {
11920         assert(prfx == REX_WRB, "must be");
11921       }
11922     } else {
11923       if (adr.index_needs_rex()) {
11924         assert(prfx == REX_WRX, "must be");
11925       } else {
11926         assert(prfx == REX_WR, "must be");
11927       }
11928     }
11929   }
11930 #endif
11931   return prfx;
11932 }
11933 
11934 void Assembler::prefixq(Address adr) {
11935   emit_int8(get_prefixq(adr));
11936 }
11937 
11938 void Assembler::prefixq(Address adr, Register src) {
11939   emit_int8(get_prefixq(adr, src));
11940 }
11941 
11942 void Assembler::prefixq(Address adr, XMMRegister src) {
11943   if (src->encoding() < 8) {
11944     if (adr.base_needs_rex()) {
11945       if (adr.index_needs_rex()) {
11946         prefix(REX_WXB);
11947       } else {
11948         prefix(REX_WB);
11949       }
11950     } else {
11951       if (adr.index_needs_rex()) {
11952         prefix(REX_WX);
11953       } else {
11954         prefix(REX_W);
11955       }
11956     }
11957   } else {
11958     if (adr.base_needs_rex()) {
11959       if (adr.index_needs_rex()) {
11960         prefix(REX_WRXB);
11961       } else {
11962         prefix(REX_WRB);
11963       }
11964     } else {
11965       if (adr.index_needs_rex()) {
11966         prefix(REX_WRX);
11967       } else {
11968         prefix(REX_WR);
11969       }
11970     }
11971   }
11972 }
11973 
11974 int Assembler::prefixq_and_encode(int reg_enc) {
11975   if (reg_enc < 8) {
11976     prefix(REX_W);
11977   } else {
11978     prefix(REX_WB);
11979     reg_enc -= 8;
11980   }
11981   return reg_enc;
11982 }
11983 
11984 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
11985   if (dst_enc < 8) {
11986     if (src_enc < 8) {
11987       prefix(REX_W);
11988     } else {
11989       prefix(REX_WB);
11990       src_enc -= 8;
11991     }
11992   } else {
11993     if (src_enc < 8) {
11994       prefix(REX_WR);
11995     } else {
11996       prefix(REX_WRB);
11997       src_enc -= 8;
11998     }
11999     dst_enc -= 8;
12000   }
12001   return dst_enc << 3 | src_enc;
12002 }
12003 
12004 void Assembler::adcq(Register dst, int32_t imm32) {
12005   (void) prefixq_and_encode(dst->encoding());
12006   emit_arith(0x81, 0xD0, dst, imm32);
12007 }
12008 
12009 void Assembler::adcq(Register dst, Address src) {
12010   InstructionMark im(this);
12011   emit_int16(get_prefixq(src, dst), 0x13);
12012   emit_operand(dst, src);
12013 }
12014 
12015 void Assembler::adcq(Register dst, Register src) {
12016   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12017   emit_arith(0x13, 0xC0, dst, src);
12018 }
12019 
12020 void Assembler::addq(Address dst, int32_t imm32) {
12021   InstructionMark im(this);
12022   prefixq(dst);
12023   emit_arith_operand(0x81, rax, dst, imm32);
12024 }
12025 
12026 void Assembler::addq(Address dst, Register src) {
12027   InstructionMark im(this);
12028   emit_int16(get_prefixq(dst, src), 0x01);
12029   emit_operand(src, dst);
12030 }
12031 
12032 void Assembler::addq(Register dst, int32_t imm32) {
12033   (void) prefixq_and_encode(dst->encoding());
12034   emit_arith(0x81, 0xC0, dst, imm32);
12035 }
12036 
12037 void Assembler::addq(Register dst, Address src) {
12038   InstructionMark im(this);
12039   emit_int16(get_prefixq(src, dst), 0x03);
12040   emit_operand(dst, src);
12041 }
12042 
12043 void Assembler::addq(Register dst, Register src) {
12044   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12045   emit_arith(0x03, 0xC0, dst, src);
12046 }
12047 
12048 void Assembler::adcxq(Register dst, Register src) {
12049   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12050   emit_int8(0x66);
12051   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12052   emit_int32(0x0F,
12053              0x38,
12054              (unsigned char)0xF6,
12055              (0xC0 | encode));
12056 }
12057 
12058 void Assembler::adoxq(Register dst, Register src) {
12059   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12060   emit_int8((unsigned char)0xF3);
12061   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12062   emit_int32(0x0F,
12063              0x38,
12064              (unsigned char)0xF6,
12065              (0xC0 | encode));
12066 }
12067 
12068 void Assembler::andq(Address dst, int32_t imm32) {
12069   InstructionMark im(this);
12070   prefixq(dst);
12071   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12072 }
12073 
12074 void Assembler::andq(Register dst, int32_t imm32) {
12075   (void) prefixq_and_encode(dst->encoding());
12076   emit_arith(0x81, 0xE0, dst, imm32);
12077 }
12078 
12079 void Assembler::andq(Register dst, Address src) {
12080   InstructionMark im(this);
12081   emit_int16(get_prefixq(src, dst), 0x23);
12082   emit_operand(dst, src);
12083 }
12084 
12085 void Assembler::andq(Register dst, Register src) {
12086   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12087   emit_arith(0x23, 0xC0, dst, src);
12088 }
12089 
12090 void Assembler::andq(Address dst, Register src) {
12091   InstructionMark im(this);
12092   emit_int16(get_prefixq(dst, src), 0x21);
12093   emit_operand(src, dst);
12094 }
12095 
12096 void Assembler::andnq(Register dst, Register src1, Register src2) {
12097   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12098   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12099   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12100   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12101 }
12102 
12103 void Assembler::andnq(Register dst, Register src1, Address src2) {
12104   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12105   InstructionMark im(this);
12106   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12107   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12108   emit_int8((unsigned char)0xF2);
12109   emit_operand(dst, src2);
12110 }
12111 
12112 void Assembler::bsfq(Register dst, Register src) {
12113   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12114   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12115 }
12116 
12117 void Assembler::bsrq(Register dst, Register src) {
12118   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12119   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12120 }
12121 
12122 void Assembler::bswapq(Register reg) {
12123   int encode = prefixq_and_encode(reg->encoding());
12124   emit_int16(0x0F, (0xC8 | encode));
12125 }
12126 
12127 void Assembler::blsiq(Register dst, Register src) {
12128   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12129   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12130   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12131   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12132 }
12133 
12134 void Assembler::blsiq(Register dst, Address src) {
12135   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12136   InstructionMark im(this);
12137   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12138   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12139   emit_int8((unsigned char)0xF3);
12140   emit_operand(rbx, src);
12141 }
12142 
12143 void Assembler::blsmskq(Register dst, Register src) {
12144   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12145   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12146   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12147   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12148 }
12149 
12150 void Assembler::blsmskq(Register dst, Address src) {
12151   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12152   InstructionMark im(this);
12153   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12154   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12155   emit_int8((unsigned char)0xF3);
12156   emit_operand(rdx, src);
12157 }
12158 
12159 void Assembler::blsrq(Register dst, Register src) {
12160   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12161   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12162   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12163   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12164 }
12165 
12166 void Assembler::blsrq(Register dst, Address src) {
12167   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12168   InstructionMark im(this);
12169   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12170   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12171   emit_int8((unsigned char)0xF3);
12172   emit_operand(rcx, src);
12173 }
12174 
12175 void Assembler::cdqq() {
12176   emit_int16(REX_W, (unsigned char)0x99);
12177 }
12178 
12179 void Assembler::clflush(Address adr) {
12180   assert(VM_Version::supports_clflush(), "should do");
12181   prefix(adr);
12182   emit_int16(0x0F, (unsigned char)0xAE);
12183   emit_operand(rdi, adr);
12184 }
12185 
12186 void Assembler::clflushopt(Address adr) {
12187   assert(VM_Version::supports_clflushopt(), "should do!");
12188   // adr should be base reg only with no index or offset
12189   assert(adr.index() == noreg, "index should be noreg");
12190   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12191   assert(adr.disp() == 0, "displacement should be 0");
12192   // instruction prefix is 0x66
12193   emit_int8(0x66);
12194   prefix(adr);
12195   // opcode family is 0x0F 0xAE
12196   emit_int16(0x0F, (unsigned char)0xAE);
12197   // extended opcode byte is 7 == rdi
12198   emit_operand(rdi, adr);
12199 }
12200 
12201 void Assembler::clwb(Address adr) {
12202   assert(VM_Version::supports_clwb(), "should do!");
12203   // adr should be base reg only with no index or offset
12204   assert(adr.index() == noreg, "index should be noreg");
12205   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12206   assert(adr.disp() == 0, "displacement should be 0");
12207   // instruction prefix is 0x66
12208   emit_int8(0x66);
12209   prefix(adr);
12210   // opcode family is 0x0f 0xAE
12211   emit_int16(0x0F, (unsigned char)0xAE);
12212   // extended opcode byte is 6 == rsi
12213   emit_operand(rsi, adr);
12214 }
12215 
12216 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12217   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12218   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12219 }
12220 
12221 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12222   InstructionMark im(this);
12223   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12224   emit_operand(dst, src);
12225 }
12226 
12227 void Assembler::cmpq(Address dst, int32_t imm32) {
12228   InstructionMark im(this);
12229   emit_int16(get_prefixq(dst), (unsigned char)0x81);
12230   emit_operand(rdi, dst, 4);
12231   emit_int32(imm32);
12232 }
12233 
12234 void Assembler::cmpq(Register dst, int32_t imm32) {
12235   (void) prefixq_and_encode(dst->encoding());
12236   emit_arith(0x81, 0xF8, dst, imm32);
12237 }
12238 
12239 void Assembler::cmpq(Address dst, Register src) {
12240   InstructionMark im(this);
12241   emit_int16(get_prefixq(dst, src), 0x39);
12242   emit_operand(src, dst);
12243 }
12244 
12245 void Assembler::cmpq(Register dst, Register src) {
12246   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12247   emit_arith(0x3B, 0xC0, dst, src);
12248 }
12249 
12250 void Assembler::cmpq(Register dst, Address src) {
12251   InstructionMark im(this);
12252   emit_int16(get_prefixq(src, dst), 0x3B);
12253   emit_operand(dst, src);
12254 }
12255 
12256 void Assembler::cmpxchgq(Register reg, Address adr) {
12257   InstructionMark im(this);
12258   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12259   emit_operand(reg, adr);
12260 }
12261 
12262 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12264   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12265   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12266   emit_int16(0x2A, (0xC0 | encode));
12267 }
12268 
12269 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12271   InstructionMark im(this);
12272   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12273   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12274   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12275   emit_int8(0x2A);
12276   emit_operand(dst, src);
12277 }
12278 
12279 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12280   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12281   InstructionMark im(this);
12282   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12283   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12284   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12285   emit_int8(0x2A);
12286   emit_operand(dst, src);
12287 }
12288 
12289 void Assembler::cvttsd2siq(Register dst, Address src) {
12290   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12291   // F2 REX.W 0F 2C /r
12292   // CVTTSD2SI r64, xmm1/m64
12293   InstructionMark im(this);
12294   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12295   emit_operand(dst, src);
12296 }
12297 
12298 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12300   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12301   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12302   emit_int16(0x2C, (0xC0 | encode));
12303 }
12304 
12305 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12306   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12307   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12308   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12309   emit_int16(0x2C, (0xC0 | encode));
12310 }
12311 
12312 void Assembler::decl(Register dst) {
12313   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12314   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12315   int encode = prefix_and_encode(dst->encoding());
12316   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12317 }
12318 
12319 void Assembler::decq(Register dst) {
12320   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12321   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12322   int encode = prefixq_and_encode(dst->encoding());
12323   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12324 }
12325 
12326 void Assembler::decq(Address dst) {
12327   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12328   InstructionMark im(this);
12329   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12330   emit_operand(rcx, dst);
12331 }
12332 
12333 void Assembler::fxrstor(Address src) {
12334   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12335   emit_operand(as_Register(1), src);
12336 }
12337 
12338 void Assembler::xrstor(Address src) {
12339   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12340   emit_operand(as_Register(5), src);
12341 }
12342 
12343 void Assembler::fxsave(Address dst) {
12344   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12345   emit_operand(as_Register(0), dst);
12346 }
12347 
12348 void Assembler::xsave(Address dst) {
12349   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12350   emit_operand(as_Register(4), dst);
12351 }
12352 
12353 void Assembler::idivq(Register src) {
12354   int encode = prefixq_and_encode(src->encoding());
12355   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12356 }
12357 
12358 void Assembler::imulq(Register dst, Register src) {
12359   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12360   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12361 }
12362 
12363 void Assembler::imulq(Register src) {
12364   int encode = prefixq_and_encode(src->encoding());
12365   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12366 }
12367 
12368 void Assembler::imulq(Register dst, Address src, int32_t value) {
12369   InstructionMark im(this);
12370   prefixq(src, dst);
12371   if (is8bit(value)) {
12372     emit_int8((unsigned char)0x6B);
12373     emit_operand(dst, src);
12374     emit_int8(value);
12375   } else {
12376     emit_int8((unsigned char)0x69);
12377     emit_operand(dst, src);
12378     emit_int32(value);
12379   }
12380 }
12381 
12382 void Assembler::imulq(Register dst, Register src, int value) {
12383   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12384   if (is8bit(value)) {
12385     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12386   } else {
12387     emit_int16(0x69, (0xC0 | encode));
12388     emit_int32(value);
12389   }
12390 }
12391 
12392 void Assembler::imulq(Register dst, Address src) {
12393   InstructionMark im(this);
12394   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12395   emit_operand(dst, src);
12396 }
12397 
12398 void Assembler::incl(Register dst) {
12399   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12400   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12401   int encode = prefix_and_encode(dst->encoding());
12402   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12403 }
12404 
12405 void Assembler::incq(Register dst) {
12406   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12407   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12408   int encode = prefixq_and_encode(dst->encoding());
12409   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12410 }
12411 
12412 void Assembler::incq(Address dst) {
12413   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12414   InstructionMark im(this);
12415   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12416   emit_operand(rax, dst);
12417 }
12418 
12419 void Assembler::lea(Register dst, Address src) {
12420   leaq(dst, src);
12421 }
12422 
12423 void Assembler::leaq(Register dst, Address src) {
12424   InstructionMark im(this);
12425   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
12426   emit_operand(dst, src);
12427 }
12428 
12429 void Assembler::mov64(Register dst, int64_t imm64) {
12430   InstructionMark im(this);
12431   int encode = prefixq_and_encode(dst->encoding());
12432   emit_int8(0xB8 | encode);
12433   emit_int64(imm64);
12434 }
12435 
12436 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
12437   InstructionMark im(this);
12438   int encode = prefixq_and_encode(dst->encoding());
12439   emit_int8(0xB8 | encode);
12440   emit_data64(imm64, rtype, format);
12441 }
12442 
12443 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
12444   InstructionMark im(this);
12445   int encode = prefixq_and_encode(dst->encoding());
12446   emit_int8(0xB8 | encode);
12447   emit_data64(imm64, rspec);
12448 }
12449 
12450 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12451   InstructionMark im(this);
12452   int encode = prefix_and_encode(dst->encoding());
12453   emit_int8(0xB8 | encode);
12454   emit_data((int)imm32, rspec, narrow_oop_operand);
12455 }
12456 
12457 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
12458   InstructionMark im(this);
12459   prefix(dst);
12460   emit_int8((unsigned char)0xC7);
12461   emit_operand(rax, dst, 4);
12462   emit_data((int)imm32, rspec, narrow_oop_operand);
12463 }
12464 
12465 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
12466   InstructionMark im(this);
12467   int encode = prefix_and_encode(src1->encoding());
12468   emit_int16((unsigned char)0x81, (0xF8 | encode));
12469   emit_data((int)imm32, rspec, narrow_oop_operand);
12470 }
12471 
12472 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
12473   InstructionMark im(this);
12474   prefix(src1);
12475   emit_int8((unsigned char)0x81);
12476   emit_operand(rax, src1, 4);
12477   emit_data((int)imm32, rspec, narrow_oop_operand);
12478 }
12479 
12480 void Assembler::lzcntq(Register dst, Register src) {
12481   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12482   emit_int8((unsigned char)0xF3);
12483   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12484   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12485 }
12486 
12487 void Assembler::movdq(XMMRegister dst, Register src) {
12488   // table D-1 says MMX/SSE2
12489   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12490   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12491   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12492   emit_int16(0x6E, (0xC0 | encode));
12493 }
12494 
12495 void Assembler::movdq(Register dst, XMMRegister src) {
12496   // table D-1 says MMX/SSE2
12497   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12498   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12499   // swap src/dst to get correct prefix
12500   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12501   emit_int16(0x7E,
12502              (0xC0 | encode));
12503 }
12504 
12505 void Assembler::movq(Register dst, Register src) {
12506   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12507   emit_int16((unsigned char)0x8B,
12508              (0xC0 | encode));
12509 }
12510 
12511 void Assembler::movq(Register dst, Address src) {
12512   InstructionMark im(this);
12513   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
12514   emit_operand(dst, src);
12515 }
12516 
12517 void Assembler::movq(Address dst, Register src) {
12518   InstructionMark im(this);
12519   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
12520   emit_operand(src, dst);
12521 }
12522 
12523 void Assembler::movq(Address dst, int32_t imm32) {
12524   InstructionMark im(this);
12525   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12526   emit_operand(as_Register(0), dst);
12527   emit_int32(imm32);
12528 }
12529 
12530 void Assembler::movq(Register dst, int32_t imm32) {
12531   int encode = prefixq_and_encode(dst->encoding());
12532   emit_int16((unsigned char)0xC7, (0xC0 | encode));
12533   emit_int32(imm32);
12534 }
12535 
12536 void Assembler::movsbq(Register dst, Address src) {
12537   InstructionMark im(this);
12538   emit_int24(get_prefixq(src, dst),
12539              0x0F,
12540              (unsigned char)0xBE);
12541   emit_operand(dst, src);
12542 }
12543 
12544 void Assembler::movsbq(Register dst, Register src) {
12545   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12546   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
12547 }
12548 
12549 void Assembler::movslq(Register dst, int32_t imm32) {
12550   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
12551   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
12552   // as a result we shouldn't use until tested at runtime...
12553   ShouldNotReachHere();
12554   InstructionMark im(this);
12555   int encode = prefixq_and_encode(dst->encoding());
12556   emit_int8(0xC7 | encode);
12557   emit_int32(imm32);
12558 }
12559 
12560 void Assembler::movslq(Address dst, int32_t imm32) {
12561   assert(is_simm32(imm32), "lost bits");
12562   InstructionMark im(this);
12563   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12564   emit_operand(rax, dst, 4);
12565   emit_int32(imm32);
12566 }
12567 
12568 void Assembler::movslq(Register dst, Address src) {
12569   InstructionMark im(this);
12570   emit_int16(get_prefixq(src, dst), 0x63);
12571   emit_operand(dst, src);
12572 }
12573 
12574 void Assembler::movslq(Register dst, Register src) {
12575   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12576   emit_int16(0x63, (0xC0 | encode));
12577 }
12578 
12579 void Assembler::movswq(Register dst, Address src) {
12580   InstructionMark im(this);
12581   emit_int24(get_prefixq(src, dst),
12582              0x0F,
12583              (unsigned char)0xBF);
12584   emit_operand(dst, src);
12585 }
12586 
12587 void Assembler::movswq(Register dst, Register src) {
12588   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12589   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
12590 }
12591 
12592 void Assembler::movzbq(Register dst, Address src) {
12593   InstructionMark im(this);
12594   emit_int24(get_prefixq(src, dst),
12595              0x0F,
12596              (unsigned char)0xB6);
12597   emit_operand(dst, src);
12598 }
12599 
12600 void Assembler::movzbq(Register dst, Register src) {
12601   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12602   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
12603 }
12604 
12605 void Assembler::movzwq(Register dst, Address src) {
12606   InstructionMark im(this);
12607   emit_int24(get_prefixq(src, dst),
12608              0x0F,
12609              (unsigned char)0xB7);
12610   emit_operand(dst, src);
12611 }
12612 
12613 void Assembler::movzwq(Register dst, Register src) {
12614   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12615   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
12616 }
12617 
12618 void Assembler::mulq(Address src) {
12619   InstructionMark im(this);
12620   emit_int16(get_prefixq(src), (unsigned char)0xF7);
12621   emit_operand(rsp, src);
12622 }
12623 
12624 void Assembler::mulq(Register src) {
12625   int encode = prefixq_and_encode(src->encoding());
12626   emit_int16((unsigned char)0xF7, (0xE0 | encode));
12627 }
12628 
12629 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
12630   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12631   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12632   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12633   emit_int16((unsigned char)0xF6, (0xC0 | encode));
12634 }
12635 
12636 void Assembler::negq(Register dst) {
12637   int encode = prefixq_and_encode(dst->encoding());
12638   emit_int16((unsigned char)0xF7, (0xD8 | encode));
12639 }
12640 
12641 void Assembler::negq(Address dst) {
12642   InstructionMark im(this);
12643   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
12644   emit_operand(as_Register(3), dst);
12645 }
12646 
12647 void Assembler::notq(Register dst) {
12648   int encode = prefixq_and_encode(dst->encoding());
12649   emit_int16((unsigned char)0xF7, (0xD0 | encode));
12650 }
12651 
12652 void Assembler::btsq(Address dst, int imm8) {
12653   assert(isByte(imm8), "not a byte");
12654   InstructionMark im(this);
12655   emit_int24(get_prefixq(dst),
12656              0x0F,
12657              (unsigned char)0xBA);
12658   emit_operand(rbp /* 5 */, dst, 1);
12659   emit_int8(imm8);
12660 }
12661 
12662 void Assembler::btrq(Address dst, int imm8) {
12663   assert(isByte(imm8), "not a byte");
12664   InstructionMark im(this);
12665   emit_int24(get_prefixq(dst),
12666              0x0F,
12667              (unsigned char)0xBA);
12668   emit_operand(rsi /* 6 */, dst, 1);
12669   emit_int8(imm8);
12670 }
12671 
12672 void Assembler::orq(Address dst, int32_t imm32) {
12673   InstructionMark im(this);
12674   prefixq(dst);
12675   emit_arith_operand(0x81, as_Register(1), dst, imm32);
12676 }
12677 
12678 void Assembler::orq(Address dst, Register src) {
12679   InstructionMark im(this);
12680   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
12681   emit_operand(src, dst);
12682 }
12683 
12684 void Assembler::orq(Register dst, int32_t imm32) {
12685   (void) prefixq_and_encode(dst->encoding());
12686   emit_arith(0x81, 0xC8, dst, imm32);
12687 }
12688 
12689 void Assembler::orq(Register dst, Address src) {
12690   InstructionMark im(this);
12691   emit_int16(get_prefixq(src, dst), 0x0B);
12692   emit_operand(dst, src);
12693 }
12694 
12695 void Assembler::orq(Register dst, Register src) {
12696   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12697   emit_arith(0x0B, 0xC0, dst, src);
12698 }
12699 
12700 void Assembler::popcntq(Register dst, Address src) {
12701   assert(VM_Version::supports_popcnt(), "must support");
12702   InstructionMark im(this);
12703   emit_int32((unsigned char)0xF3,
12704              get_prefixq(src, dst),
12705              0x0F,
12706              (unsigned char)0xB8);
12707   emit_operand(dst, src);
12708 }
12709 
12710 void Assembler::popcntq(Register dst, Register src) {
12711   assert(VM_Version::supports_popcnt(), "must support");
12712   emit_int8((unsigned char)0xF3);
12713   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12714   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
12715 }
12716 
12717 void Assembler::popq(Address dst) {
12718   InstructionMark im(this);
12719   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
12720   emit_operand(rax, dst);
12721 }
12722 
12723 void Assembler::popq(Register dst) {
12724   emit_int8((unsigned char)0x58 | dst->encoding());
12725 }
12726 
12727 // Precomputable: popa, pusha, vzeroupper
12728 
12729 // The result of these routines are invariant from one invocation to another
12730 // invocation for the duration of a run. Caching the result on bootstrap
12731 // and copying it out on subsequent invocations can thus be beneficial
12732 static bool     precomputed = false;
12733 
12734 static u_char* popa_code  = NULL;
12735 static int     popa_len   = 0;
12736 
12737 static u_char* pusha_code = NULL;
12738 static int     pusha_len  = 0;
12739 
12740 static u_char* vzup_code  = NULL;
12741 static int     vzup_len   = 0;
12742 
12743 void Assembler::precompute_instructions() {
12744   assert(!Universe::is_fully_initialized(), "must still be single threaded");
12745   guarantee(!precomputed, "only once");
12746   precomputed = true;
12747   ResourceMark rm;
12748 
12749   // Make a temporary buffer big enough for the routines we're capturing
12750   int size = 256;
12751   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
12752   CodeBuffer buffer((address)tmp_code, size);
12753   MacroAssembler masm(&buffer);
12754 
12755   address begin_popa  = masm.code_section()->end();
12756   masm.popa_uncached();
12757   address end_popa    = masm.code_section()->end();
12758   masm.pusha_uncached();
12759   address end_pusha   = masm.code_section()->end();
12760   masm.vzeroupper_uncached();
12761   address end_vzup    = masm.code_section()->end();
12762 
12763   // Save the instructions to permanent buffers.
12764   popa_len = (int)(end_popa - begin_popa);
12765   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
12766   memcpy(popa_code, begin_popa, popa_len);
12767 
12768   pusha_len = (int)(end_pusha - end_popa);
12769   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
12770   memcpy(pusha_code, end_popa, pusha_len);
12771 
12772   vzup_len = (int)(end_vzup - end_pusha);
12773   if (vzup_len > 0) {
12774     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
12775     memcpy(vzup_code, end_pusha, vzup_len);
12776   } else {
12777     vzup_code = pusha_code; // dummy
12778   }
12779 
12780   assert(masm.code()->total_oop_size() == 0 &&
12781          masm.code()->total_metadata_size() == 0 &&
12782          masm.code()->total_relocation_size() == 0,
12783          "pre-computed code can't reference oops, metadata or contain relocations");
12784 }
12785 
12786 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
12787   assert(src != NULL, "code to copy must have been pre-computed");
12788   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
12789   address end = code_section->end();
12790   memcpy(end, src, src_len);
12791   code_section->set_end(end + src_len);
12792 }
12793 
12794 void Assembler::popa() { // 64bit
12795   emit_copy(code_section(), popa_code, popa_len);
12796 }
12797 
12798 void Assembler::popa_uncached() { // 64bit
12799   movq(r15, Address(rsp, 0));
12800   movq(r14, Address(rsp, wordSize));
12801   movq(r13, Address(rsp, 2 * wordSize));
12802   movq(r12, Address(rsp, 3 * wordSize));
12803   movq(r11, Address(rsp, 4 * wordSize));
12804   movq(r10, Address(rsp, 5 * wordSize));
12805   movq(r9,  Address(rsp, 6 * wordSize));
12806   movq(r8,  Address(rsp, 7 * wordSize));
12807   movq(rdi, Address(rsp, 8 * wordSize));
12808   movq(rsi, Address(rsp, 9 * wordSize));
12809   movq(rbp, Address(rsp, 10 * wordSize));
12810   // Skip rsp as it is restored automatically to the value
12811   // before the corresponding pusha when popa is done.
12812   movq(rbx, Address(rsp, 12 * wordSize));
12813   movq(rdx, Address(rsp, 13 * wordSize));
12814   movq(rcx, Address(rsp, 14 * wordSize));
12815   movq(rax, Address(rsp, 15 * wordSize));
12816 
12817   addq(rsp, 16 * wordSize);
12818 }
12819 
12820 // Does not actually store the value of rsp on the stack.
12821 // The slot for rsp just contains an arbitrary value.
12822 void Assembler::pusha() { // 64bit
12823   emit_copy(code_section(), pusha_code, pusha_len);
12824 }
12825 
12826 // Does not actually store the value of rsp on the stack.
12827 // The slot for rsp just contains an arbitrary value.
12828 void Assembler::pusha_uncached() { // 64bit
12829   subq(rsp, 16 * wordSize);
12830 
12831   movq(Address(rsp, 15 * wordSize), rax);
12832   movq(Address(rsp, 14 * wordSize), rcx);
12833   movq(Address(rsp, 13 * wordSize), rdx);
12834   movq(Address(rsp, 12 * wordSize), rbx);
12835   // Skip rsp as the value is normally not used. There are a few places where
12836   // the original value of rsp needs to be known but that can be computed
12837   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
12838   movq(Address(rsp, 10 * wordSize), rbp);
12839   movq(Address(rsp, 9 * wordSize), rsi);
12840   movq(Address(rsp, 8 * wordSize), rdi);
12841   movq(Address(rsp, 7 * wordSize), r8);
12842   movq(Address(rsp, 6 * wordSize), r9);
12843   movq(Address(rsp, 5 * wordSize), r10);
12844   movq(Address(rsp, 4 * wordSize), r11);
12845   movq(Address(rsp, 3 * wordSize), r12);
12846   movq(Address(rsp, 2 * wordSize), r13);
12847   movq(Address(rsp, wordSize), r14);
12848   movq(Address(rsp, 0), r15);
12849 }
12850 
12851 void Assembler::vzeroupper() {
12852   emit_copy(code_section(), vzup_code, vzup_len);
12853 }
12854 
12855 void Assembler::pushq(Address src) {
12856   InstructionMark im(this);
12857   emit_int16(get_prefixq(src), (unsigned char)0xFF);
12858   emit_operand(rsi, src);
12859 }
12860 
12861 void Assembler::rclq(Register dst, int imm8) {
12862   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12863   int encode = prefixq_and_encode(dst->encoding());
12864   if (imm8 == 1) {
12865     emit_int16((unsigned char)0xD1, (0xD0 | encode));
12866   } else {
12867     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
12868   }
12869 }
12870 
12871 void Assembler::rcrq(Register dst, int imm8) {
12872   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12873   int encode = prefixq_and_encode(dst->encoding());
12874   if (imm8 == 1) {
12875     emit_int16((unsigned char)0xD1, (0xD8 | encode));
12876   } else {
12877     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
12878   }
12879 }
12880 
12881 
12882 void Assembler::rorxq(Register dst, Register src, int imm8) {
12883   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12884   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12885   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
12886   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
12887 }
12888 
12889 void Assembler::rorxd(Register dst, Register src, int imm8) {
12890   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12891   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12892   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
12893   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
12894 }
12895 
12896 #ifdef _LP64
12897 void Assembler::salq(Address dst, int imm8) {
12898   InstructionMark im(this);
12899   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12900   if (imm8 == 1) {
12901     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
12902     emit_operand(as_Register(4), dst);
12903   }
12904   else {
12905     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
12906     emit_operand(as_Register(4), dst);
12907     emit_int8(imm8);
12908   }
12909 }
12910 
12911 void Assembler::salq(Address dst) {
12912   InstructionMark im(this);
12913   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
12914   emit_operand(as_Register(4), dst);
12915 }
12916 
12917 void Assembler::salq(Register dst, int imm8) {
12918   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12919   int encode = prefixq_and_encode(dst->encoding());
12920   if (imm8 == 1) {
12921     emit_int16((unsigned char)0xD1, (0xE0 | encode));
12922   } else {
12923     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
12924   }
12925 }
12926 
12927 void Assembler::salq(Register dst) {
12928   int encode = prefixq_and_encode(dst->encoding());
12929   emit_int16((unsigned char)0xD3, (0xE0 | encode));
12930 }
12931 
12932 void Assembler::sarq(Address dst, int imm8) {
12933   InstructionMark im(this);
12934   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12935   if (imm8 == 1) {
12936     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
12937     emit_operand(as_Register(7), dst);
12938   }
12939   else {
12940     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
12941     emit_operand(as_Register(7), dst);
12942     emit_int8(imm8);
12943   }
12944 }
12945 
12946 void Assembler::sarq(Address dst) {
12947   InstructionMark im(this);
12948   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
12949   emit_operand(as_Register(7), dst);
12950 }
12951 
12952 void Assembler::sarq(Register dst, int imm8) {
12953   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12954   int encode = prefixq_and_encode(dst->encoding());
12955   if (imm8 == 1) {
12956     emit_int16((unsigned char)0xD1, (0xF8 | encode));
12957   } else {
12958     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
12959   }
12960 }
12961 
12962 void Assembler::sarq(Register dst) {
12963   int encode = prefixq_and_encode(dst->encoding());
12964   emit_int16((unsigned char)0xD3, (0xF8 | encode));
12965 }
12966 #endif
12967 
12968 void Assembler::sbbq(Address dst, int32_t imm32) {
12969   InstructionMark im(this);
12970   prefixq(dst);
12971   emit_arith_operand(0x81, rbx, dst, imm32);
12972 }
12973 
12974 void Assembler::sbbq(Register dst, int32_t imm32) {
12975   (void) prefixq_and_encode(dst->encoding());
12976   emit_arith(0x81, 0xD8, dst, imm32);
12977 }
12978 
12979 void Assembler::sbbq(Register dst, Address src) {
12980   InstructionMark im(this);
12981   emit_int16(get_prefixq(src, dst), 0x1B);
12982   emit_operand(dst, src);
12983 }
12984 
12985 void Assembler::sbbq(Register dst, Register src) {
12986   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12987   emit_arith(0x1B, 0xC0, dst, src);
12988 }
12989 
12990 void Assembler::shlq(Register dst, int imm8) {
12991   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12992   int encode = prefixq_and_encode(dst->encoding());
12993   if (imm8 == 1) {
12994     emit_int16((unsigned char)0xD1, (0xE0 | encode));
12995   } else {
12996     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
12997   }
12998 }
12999 
13000 void Assembler::shlq(Register dst) {
13001   int encode = prefixq_and_encode(dst->encoding());
13002   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13003 }
13004 
13005 void Assembler::shrq(Register dst, int imm8) {
13006   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13007   int encode = prefixq_and_encode(dst->encoding());
13008   if (imm8 == 1) {
13009     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13010   }
13011   else {
13012     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13013   }
13014 }
13015 
13016 void Assembler::shrq(Register dst) {
13017   int encode = prefixq_and_encode(dst->encoding());
13018   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13019 }
13020 
13021 void Assembler::shrq(Address dst) {
13022   InstructionMark im(this);
13023   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13024   emit_operand(as_Register(5), dst);
13025 }
13026 
13027 void Assembler::shrq(Address dst, int imm8) {
13028   InstructionMark im(this);
13029   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13030   if (imm8 == 1) {
13031     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13032     emit_operand(as_Register(5), dst);
13033   }
13034   else {
13035     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13036     emit_operand(as_Register(5), dst);
13037     emit_int8(imm8);
13038   }
13039 }
13040 
13041 void Assembler::subq(Address dst, int32_t imm32) {
13042   InstructionMark im(this);
13043   prefixq(dst);
13044   emit_arith_operand(0x81, rbp, dst, imm32);
13045 }
13046 
13047 void Assembler::subq(Address dst, Register src) {
13048   InstructionMark im(this);
13049   emit_int16(get_prefixq(dst, src), 0x29);
13050   emit_operand(src, dst);
13051 }
13052 
13053 void Assembler::subq(Register dst, int32_t imm32) {
13054   (void) prefixq_and_encode(dst->encoding());
13055   emit_arith(0x81, 0xE8, dst, imm32);
13056 }
13057 
13058 // Force generation of a 4 byte immediate value even if it fits into 8bit
13059 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13060   (void) prefixq_and_encode(dst->encoding());
13061   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13062 }
13063 
13064 void Assembler::subq(Register dst, Address src) {
13065   InstructionMark im(this);
13066   emit_int16(get_prefixq(src, dst), 0x2B);
13067   emit_operand(dst, src);
13068 }
13069 
13070 void Assembler::subq(Register dst, Register src) {
13071   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13072   emit_arith(0x2B, 0xC0, dst, src);
13073 }
13074 
13075 void Assembler::testq(Address dst, int32_t imm32) {
13076   InstructionMark im(this);
13077   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13078   emit_operand(as_Register(0), dst);
13079   emit_int32(imm32);
13080 }
13081 
13082 void Assembler::testq(Register dst, int32_t imm32) {
13083   // not using emit_arith because test
13084   // doesn't support sign-extension of
13085   // 8bit operands
13086   int encode = dst->encoding();
13087   encode = prefixq_and_encode(encode);
13088   emit_int16((unsigned char)0xF7, (0xC0 | encode));
13089   emit_int32(imm32);
13090 }
13091 
13092 void Assembler::testq(Register dst, Register src) {
13093   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13094   emit_arith(0x85, 0xC0, dst, src);
13095 }
13096 
13097 void Assembler::testq(Register dst, Address src) {
13098   InstructionMark im(this);
13099   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13100   emit_operand(dst, src);
13101 }
13102 
13103 void Assembler::xaddq(Address dst, Register src) {
13104   InstructionMark im(this);
13105   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13106   emit_operand(src, dst);
13107 }
13108 
13109 void Assembler::xchgq(Register dst, Address src) {
13110   InstructionMark im(this);
13111   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13112   emit_operand(dst, src);
13113 }
13114 
13115 void Assembler::xchgq(Register dst, Register src) {
13116   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13117   emit_int16((unsigned char)0x87, (0xc0 | encode));
13118 }
13119 
13120 void Assembler::xorq(Register dst, Register src) {
13121   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13122   emit_arith(0x33, 0xC0, dst, src);
13123 }
13124 
13125 void Assembler::xorq(Register dst, Address src) {
13126   InstructionMark im(this);
13127   emit_int16(get_prefixq(src, dst), 0x33);
13128   emit_operand(dst, src);
13129 }
13130 
13131 void Assembler::xorq(Register dst, int32_t imm32) {
13132   (void) prefixq_and_encode(dst->encoding());
13133   emit_arith(0x81, 0xF0, dst, imm32);
13134 }
13135 
13136 void Assembler::xorq(Address dst, int32_t imm32) {
13137   InstructionMark im(this);
13138   prefixq(dst);
13139   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13140 }
13141 
13142 void Assembler::xorq(Address dst, Register src) {
13143   InstructionMark im(this);
13144   emit_int16(get_prefixq(dst, src), 0x31);
13145   emit_operand(src, dst);
13146 }
13147 
13148 #endif // !LP64
13149 
13150 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13151   if (VM_Version::supports_evex()) {
13152     _tuple_type = tuple_type;
13153     _input_size_in_bits = input_size_in_bits;
13154   }
13155 }
--- EOF ---