1 /*
    2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec = RelocationHolder::none;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert(op1 == 0x81, "Unexpected opcode");
  304   if (is8bit(imm32)) {
  305     emit_int24(op1 | 0x02,        // set sign bit
  306                op2 | encode(dst),
  307                imm32 & 0xFF);
  308   } else if (dst == rax) {
  309     switch (op2) {
  310       case 0xD0: emit_int8(0x15); break; // adc
  311       case 0xC0: emit_int8(0x05); break; // add
  312       case 0xE0: emit_int8(0x25); break; // and
  313       case 0xF8: emit_int8(0x3D); break; // cmp
  314       case 0xC8: emit_int8(0x0D); break; // or
  315       case 0xD8: emit_int8(0x1D); break; // sbb
  316       case 0xE8: emit_int8(0x2D); break; // sub
  317       case 0xF0: emit_int8(0x35); break; // xor
  318       default: ShouldNotReachHere();
  319     }
  320     emit_int32(imm32);
  321   } else {
  322     emit_int16(op1, (op2 | encode(dst)));
  323     emit_int32(imm32);
  324   }
  325 }
  326 
  327 // Force generation of a 4 byte immediate value even if it fits into 8bit
  328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  329   assert(isByte(op1) && isByte(op2), "wrong opcode");
  330   assert((op1 & 0x01) == 1, "should be 32bit operation");
  331   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  332   emit_int16(op1, (op2 | encode(dst)));
  333   emit_int32(imm32);
  334 }
  335 
  336 // immediate-to-memory forms
  337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  338   assert((op1 & 0x01) == 1, "should be 32bit operation");
  339   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  340   if (is8bit(imm32)) {
  341     emit_int8(op1 | 0x02); // set sign bit
  342     emit_operand(rm, adr, 1);
  343     emit_int8(imm32 & 0xFF);
  344   } else {
  345     emit_int8(op1);
  346     emit_operand(rm, adr, 4);
  347     emit_int32(imm32);
  348   }
  349 }
  350 
  351 
  352 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  353   assert(isByte(op1) && isByte(op2), "wrong opcode");
  354   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  355 }
  356 
  357 
  358 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  359                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  360   int mod_idx = 0;
  361   // We will test if the displacement fits the compressed format and if so
  362   // apply the compression to the displacement iff the result is8bit.
  363   if (VM_Version::supports_evex() && is_evex_inst) {
  364     switch (cur_tuple_type) {
  365     case EVEX_FV:
  366       if ((cur_encoding & VEX_W) == VEX_W) {
  367         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  368       } else {
  369         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  370       }
  371       break;
  372 
  373     case EVEX_HV:
  374       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  375       break;
  376 
  377     case EVEX_FVM:
  378       break;
  379 
  380     case EVEX_T1S:
  381       switch (in_size_in_bits) {
  382       case EVEX_8bit:
  383         break;
  384 
  385       case EVEX_16bit:
  386         mod_idx = 1;
  387         break;
  388 
  389       case EVEX_32bit:
  390         mod_idx = 2;
  391         break;
  392 
  393       case EVEX_64bit:
  394         mod_idx = 3;
  395         break;
  396       }
  397       break;
  398 
  399     case EVEX_T1F:
  400     case EVEX_T2:
  401     case EVEX_T4:
  402       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  403       break;
  404 
  405     case EVEX_T8:
  406       break;
  407 
  408     case EVEX_HVM:
  409       break;
  410 
  411     case EVEX_QVM:
  412       break;
  413 
  414     case EVEX_OVM:
  415       break;
  416 
  417     case EVEX_M128:
  418       break;
  419 
  420     case EVEX_DUP:
  421       break;
  422 
  423     default:
  424       assert(0, "no valid evex tuple_table entry");
  425       break;
  426     }
  427 
  428     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  429       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  430       if ((disp % disp_factor) == 0) {
  431         int new_disp = disp / disp_factor;
  432         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  433           disp = new_disp;
  434         }
  435       } else {
  436         return false;
  437       }
  438     }
  439   }
  440   return (-0x80 <= disp && disp < 0x80);
  441 }
  442 
  443 
  444 bool Assembler::emit_compressed_disp_byte(int &disp) {
  445   int mod_idx = 0;
  446   // We will test if the displacement fits the compressed format and if so
  447   // apply the compression to the displacement iff the result is8bit.
  448   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  449     int evex_encoding = _attributes->get_evex_encoding();
  450     int tuple_type = _attributes->get_tuple_type();
  451     switch (tuple_type) {
  452     case EVEX_FV:
  453       if ((evex_encoding & VEX_W) == VEX_W) {
  454         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  455       } else {
  456         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  457       }
  458       break;
  459 
  460     case EVEX_HV:
  461       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  462       break;
  463 
  464     case EVEX_FVM:
  465       break;
  466 
  467     case EVEX_T1S:
  468       switch (_attributes->get_input_size()) {
  469       case EVEX_8bit:
  470         break;
  471 
  472       case EVEX_16bit:
  473         mod_idx = 1;
  474         break;
  475 
  476       case EVEX_32bit:
  477         mod_idx = 2;
  478         break;
  479 
  480       case EVEX_64bit:
  481         mod_idx = 3;
  482         break;
  483       }
  484       break;
  485 
  486     case EVEX_T1F:
  487     case EVEX_T2:
  488     case EVEX_T4:
  489       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  490       break;
  491 
  492     case EVEX_T8:
  493       break;
  494 
  495     case EVEX_HVM:
  496       break;
  497 
  498     case EVEX_QVM:
  499       break;
  500 
  501     case EVEX_OVM:
  502       break;
  503 
  504     case EVEX_M128:
  505       break;
  506 
  507     case EVEX_DUP:
  508       break;
  509 
  510     default:
  511       assert(0, "no valid evex tuple_table entry");
  512       break;
  513     }
  514 
  515     int vector_len = _attributes->get_vector_len();
  516     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  517       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  518       if ((disp % disp_factor) == 0) {
  519         int new_disp = disp / disp_factor;
  520         if (is8bit(new_disp)) {
  521           disp = new_disp;
  522         }
  523       } else {
  524         return false;
  525       }
  526     }
  527   }
  528   return is8bit(disp);
  529 }
  530 
  531 static bool is_valid_encoding(int reg_enc) {
  532   return reg_enc >= 0;
  533 }
  534 
  535 static int raw_encode(Register reg) {
  536   assert(reg == noreg || reg->is_valid(), "sanity");
  537   int reg_enc = (intptr_t)reg;
  538   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  539   return reg_enc;
  540 }
  541 
  542 static int raw_encode(XMMRegister xmmreg) {
  543   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  544   int xmmreg_enc = (intptr_t)xmmreg;
  545   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  546   return xmmreg_enc;
  547 }
  548 
  549 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  550   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  551 }
  552 
  553 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  554   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  555 }
  556 
  557 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  558   assert((mod & 3) != 0b11, "forbidden");
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   emit_int8(modrm);
  561 }
  562 
  563 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  564                                         int disp) {
  565   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  566   emit_int16(modrm, disp & 0xFF);
  567 }
  568 
  569 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  570                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  571   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  572   int sib = sib_encoding(scale, index_enc, base_enc);
  573   emit_int16(modrm, sib);
  574 }
  575 
  576 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  577                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  578                                             int disp) {
  579   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  580   int sib = sib_encoding(scale, index_enc, base_enc);
  581   emit_int24(modrm, sib, disp & 0xFF);
  582 }
  583 
  584 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  585                                     Address::ScaleFactor scale, int disp,
  586                                     RelocationHolder const& rspec,
  587                                     int rip_relative_correction) {
  588   bool no_relocation = (rspec.type() == relocInfo::none);
  589 
  590   if (is_valid_encoding(base_enc)) {
  591     if (is_valid_encoding(index_enc)) {
  592       assert(scale != Address::no_scale, "inconsistent address");
  593       // [base + index*scale + disp]
  594       if (disp == 0 && no_relocation &&
  595           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  596         // [base + index*scale]
  597         // [00 reg 100][ss index base]
  598         emit_modrm_sib(0b00, reg_enc, 0b100,
  599                        scale, index_enc, base_enc);
  600       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  601         // [base + index*scale + imm8]
  602         // [01 reg 100][ss index base] imm8
  603         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  604                              scale, index_enc, base_enc,
  605                              disp);
  606       } else {
  607         // [base + index*scale + disp32]
  608         // [10 reg 100][ss index base] disp32
  609         emit_modrm_sib(0b10, reg_enc, 0b100,
  610                        scale, index_enc, base_enc);
  611         emit_data(disp, rspec, disp32_operand);
  612       }
  613     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  614       // [rsp + disp]
  615       if (disp == 0 && no_relocation) {
  616         // [rsp]
  617         // [00 reg 100][00 100 100]
  618         emit_modrm_sib(0b00, reg_enc, 0b100,
  619                        Address::times_1, 0b100, 0b100);
  620       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  621         // [rsp + imm8]
  622         // [01 reg 100][00 100 100] disp8
  623         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  624                              Address::times_1, 0b100, 0b100,
  625                              disp);
  626       } else {
  627         // [rsp + imm32]
  628         // [10 reg 100][00 100 100] disp32
  629         emit_modrm_sib(0b10, reg_enc, 0b100,
  630                        Address::times_1, 0b100, 0b100);
  631         emit_data(disp, rspec, disp32_operand);
  632       }
  633     } else {
  634       // [base + disp]
  635       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  636       if (disp == 0 && no_relocation &&
  637           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  638         // [base]
  639         // [00 reg base]
  640         emit_modrm(0, reg_enc, base_enc);
  641       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  642         // [base + disp8]
  643         // [01 reg base] disp8
  644         emit_modrm_disp8(0b01, reg_enc, base_enc,
  645                          disp);
  646       } else {
  647         // [base + disp32]
  648         // [10 reg base] disp32
  649         emit_modrm(0b10, reg_enc, base_enc);
  650         emit_data(disp, rspec, disp32_operand);
  651       }
  652     }
  653   } else {
  654     if (is_valid_encoding(index_enc)) {
  655       assert(scale != Address::no_scale, "inconsistent address");
  656       // base == noreg
  657       // [index*scale + disp]
  658       // [00 reg 100][ss index 101] disp32
  659       emit_modrm_sib(0b00, reg_enc, 0b100,
  660                      scale, index_enc, 0b101 /* no base */);
  661       emit_data(disp, rspec, disp32_operand);
  662     } else if (!no_relocation) {
  663       // base == noreg, index == noreg
  664       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  665       // [00 reg 101] disp32
  666 
  667       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  668       // Note that the RIP-rel. correction applies to the generated
  669       // disp field, but _not_ to the target address in the rspec.
  670 
  671       // disp was created by converting the target address minus the pc
  672       // at the start of the instruction. That needs more correction here.
  673       // intptr_t disp = target - next_ip;
  674       assert(inst_mark() != NULL, "must be inside InstructionMark");
  675       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  676       int64_t adjusted = disp;
  677       // Do rip-rel adjustment for 64bit
  678       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  679       assert(is_simm32(adjusted),
  680              "must be 32bit offset (RIP relative address)");
  681       emit_data((int32_t) adjusted, rspec, disp32_operand);
  682 
  683     } else {
  684       // base == noreg, index == noreg, no_relocation == true
  685       // 32bit never did this, did everything as the rip-rel/disp code above
  686       // [disp] ABSOLUTE
  687       // [00 reg 100][00 100 101] disp32
  688       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  689                      Address::times_1, 0b100, 0b101);
  690       emit_data(disp, rspec, disp32_operand);
  691     }
  692   }
  693 }
  694 
  695 void Assembler::emit_operand(Register reg, Register base, Register index,
  696                              Address::ScaleFactor scale, int disp,
  697                              RelocationHolder const& rspec,
  698                              int rip_relative_correction) {
  699   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  700   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  701                       scale, disp, rspec, rip_relative_correction);
  702 
  703 }
  704 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  705                              Address::ScaleFactor scale, int disp,
  706                              RelocationHolder const& rspec) {
  707   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  708   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  709   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  710                       scale, disp, rspec);
  711 }
  712 
  713 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  714                              Address::ScaleFactor scale, int disp,
  715                              RelocationHolder const& rspec) {
  716   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  717   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  718   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  719                       scale, disp, rspec, /* rip_relative_correction */ 0);
  720 }
  721 
  722 // Secret local extension to Assembler::WhichOperand:
  723 #define end_pc_operand (_WhichOperand_limit)
  724 
  725 address Assembler::locate_operand(address inst, WhichOperand which) {
  726   // Decode the given instruction, and return the address of
  727   // an embedded 32-bit operand word.
  728 
  729   // If "which" is disp32_operand, selects the displacement portion
  730   // of an effective address specifier.
  731   // If "which" is imm64_operand, selects the trailing immediate constant.
  732   // If "which" is call32_operand, selects the displacement of a call or jump.
  733   // Caller is responsible for ensuring that there is such an operand,
  734   // and that it is 32/64 bits wide.
  735 
  736   // If "which" is end_pc_operand, find the end of the instruction.
  737 
  738   address ip = inst;
  739   bool is_64bit = false;
  740 
  741   debug_only(bool has_disp32 = false);
  742   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  743 
  744   again_after_prefix:
  745   switch (0xFF & *ip++) {
  746 
  747   // These convenience macros generate groups of "case" labels for the switch.
  748 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  749 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  750              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  751 #define REP16(x) REP8((x)+0): \
  752               case REP8((x)+8)
  753 
  754   case CS_segment:
  755   case SS_segment:
  756   case DS_segment:
  757   case ES_segment:
  758   case FS_segment:
  759   case GS_segment:
  760     // Seems dubious
  761     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  762     assert(ip == inst+1, "only one prefix allowed");
  763     goto again_after_prefix;
  764 
  765   case 0x67:
  766   case REX:
  767   case REX_B:
  768   case REX_X:
  769   case REX_XB:
  770   case REX_R:
  771   case REX_RB:
  772   case REX_RX:
  773   case REX_RXB:
  774     NOT_LP64(assert(false, "64bit prefixes"));
  775     goto again_after_prefix;
  776 
  777   case REX_W:
  778   case REX_WB:
  779   case REX_WX:
  780   case REX_WXB:
  781   case REX_WR:
  782   case REX_WRB:
  783   case REX_WRX:
  784   case REX_WRXB:
  785     NOT_LP64(assert(false, "64bit prefixes"));
  786     is_64bit = true;
  787     goto again_after_prefix;
  788 
  789   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  790   case 0x88: // movb a, r
  791   case 0x89: // movl a, r
  792   case 0x8A: // movb r, a
  793   case 0x8B: // movl r, a
  794   case 0x8F: // popl a
  795     debug_only(has_disp32 = true);
  796     break;
  797 
  798   case 0x68: // pushq #32
  799     if (which == end_pc_operand) {
  800       return ip + 4;
  801     }
  802     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  803     return ip;                  // not produced by emit_operand
  804 
  805   case 0x66: // movw ... (size prefix)
  806     again_after_size_prefix2:
  807     switch (0xFF & *ip++) {
  808     case REX:
  809     case REX_B:
  810     case REX_X:
  811     case REX_XB:
  812     case REX_R:
  813     case REX_RB:
  814     case REX_RX:
  815     case REX_RXB:
  816     case REX_W:
  817     case REX_WB:
  818     case REX_WX:
  819     case REX_WXB:
  820     case REX_WR:
  821     case REX_WRB:
  822     case REX_WRX:
  823     case REX_WRXB:
  824       NOT_LP64(assert(false, "64bit prefix found"));
  825       goto again_after_size_prefix2;
  826     case 0x8B: // movw r, a
  827     case 0x89: // movw a, r
  828       debug_only(has_disp32 = true);
  829       break;
  830     case 0xC7: // movw a, #16
  831       debug_only(has_disp32 = true);
  832       tail_size = 2;  // the imm16
  833       break;
  834     case 0x0F: // several SSE/SSE2 variants
  835       ip--;    // reparse the 0x0F
  836       goto again_after_prefix;
  837     default:
  838       ShouldNotReachHere();
  839     }
  840     break;
  841 
  842   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  843     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  844     // these asserts are somewhat nonsensical
  845 #ifndef _LP64
  846     assert(which == imm_operand || which == disp32_operand,
  847            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  848 #else
  849     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  850            which == narrow_oop_operand && !is_64bit,
  851            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  852 #endif // _LP64
  853     return ip;
  854 
  855   case 0x69: // imul r, a, #32
  856   case 0xC7: // movl a, #32(oop?)
  857     tail_size = 4;
  858     debug_only(has_disp32 = true); // has both kinds of operands!
  859     break;
  860 
  861   case 0x0F: // movx..., etc.
  862     switch (0xFF & *ip++) {
  863     case 0x3A: // pcmpestri
  864       tail_size = 1;
  865     case 0x38: // ptest, pmovzxbw
  866       ip++; // skip opcode
  867       debug_only(has_disp32 = true); // has both kinds of operands!
  868       break;
  869 
  870     case 0x70: // pshufd r, r/a, #8
  871       debug_only(has_disp32 = true); // has both kinds of operands!
  872     case 0x73: // psrldq r, #8
  873       tail_size = 1;
  874       break;
  875 
  876     case 0x12: // movlps
  877     case 0x28: // movaps
  878     case 0x2E: // ucomiss
  879     case 0x2F: // comiss
  880     case 0x54: // andps
  881     case 0x55: // andnps
  882     case 0x56: // orps
  883     case 0x57: // xorps
  884     case 0x58: // addpd
  885     case 0x59: // mulpd
  886     case 0x6E: // movd
  887     case 0x7E: // movd
  888     case 0x6F: // movdq
  889     case 0x7F: // movdq
  890     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  891     case 0xFE: // paddd
  892       debug_only(has_disp32 = true);
  893       break;
  894 
  895     case 0xAD: // shrd r, a, %cl
  896     case 0xAF: // imul r, a
  897     case 0xBE: // movsbl r, a (movsxb)
  898     case 0xBF: // movswl r, a (movsxw)
  899     case 0xB6: // movzbl r, a (movzxb)
  900     case 0xB7: // movzwl r, a (movzxw)
  901     case REP16(0x40): // cmovl cc, r, a
  902     case 0xB0: // cmpxchgb
  903     case 0xB1: // cmpxchg
  904     case 0xC1: // xaddl
  905     case 0xC7: // cmpxchg8
  906     case REP16(0x90): // setcc a
  907       debug_only(has_disp32 = true);
  908       // fall out of the switch to decode the address
  909       break;
  910 
  911     case 0xC4: // pinsrw r, a, #8
  912       debug_only(has_disp32 = true);
  913     case 0xC5: // pextrw r, r, #8
  914       tail_size = 1;  // the imm8
  915       break;
  916 
  917     case 0xAC: // shrd r, a, #8
  918       debug_only(has_disp32 = true);
  919       tail_size = 1;  // the imm8
  920       break;
  921 
  922     case REP16(0x80): // jcc rdisp32
  923       if (which == end_pc_operand)  return ip + 4;
  924       assert(which == call32_operand, "jcc has no disp32 or imm");
  925       return ip;
  926     default:
  927       ShouldNotReachHere();
  928     }
  929     break;
  930 
  931   case 0x81: // addl a, #32; addl r, #32
  932     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  933     // on 32bit in the case of cmpl, the imm might be an oop
  934     tail_size = 4;
  935     debug_only(has_disp32 = true); // has both kinds of operands!
  936     break;
  937 
  938   case 0x83: // addl a, #8; addl r, #8
  939     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  940     debug_only(has_disp32 = true); // has both kinds of operands!
  941     tail_size = 1;
  942     break;
  943 
  944   case 0x15: // adc rax, #32
  945   case 0x05: // add rax, #32
  946   case 0x25: // and rax, #32
  947   case 0x3D: // cmp rax, #32
  948   case 0x0D: // or  rax, #32
  949   case 0x1D: // sbb rax, #32
  950   case 0x2D: // sub rax, #32
  951   case 0x35: // xor rax, #32
  952     return which == end_pc_operand ? ip + 4 : ip;
  953 
  954   case 0x9B:
  955     switch (0xFF & *ip++) {
  956     case 0xD9: // fnstcw a
  957       debug_only(has_disp32 = true);
  958       break;
  959     default:
  960       ShouldNotReachHere();
  961     }
  962     break;
  963 
  964   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  965   case REP4(0x10): // adc...
  966   case REP4(0x20): // and...
  967   case REP4(0x30): // xor...
  968   case REP4(0x08): // or...
  969   case REP4(0x18): // sbb...
  970   case REP4(0x28): // sub...
  971   case 0xF7: // mull a
  972   case 0x8D: // lea r, a
  973   case 0x87: // xchg r, a
  974   case REP4(0x38): // cmp...
  975   case 0x85: // test r, a
  976     debug_only(has_disp32 = true); // has both kinds of operands!
  977     break;
  978 
  979   case 0xA8: // testb rax, #8
  980     return which == end_pc_operand ? ip + 1 : ip;
  981   case 0xA9: // testl/testq rax, #32
  982     return which == end_pc_operand ? ip + 4 : ip;
  983 
  984   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  985   case 0xC6: // movb a, #8
  986   case 0x80: // cmpb a, #8
  987   case 0x6B: // imul r, a, #8
  988     debug_only(has_disp32 = true); // has both kinds of operands!
  989     tail_size = 1; // the imm8
  990     break;
  991 
  992   case 0xC4: // VEX_3bytes
  993   case 0xC5: // VEX_2bytes
  994     assert((UseAVX > 0), "shouldn't have VEX prefix");
  995     assert(ip == inst+1, "no prefixes allowed");
  996     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  997     // but they have prefix 0x0F and processed when 0x0F processed above.
  998     //
  999     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1000     // instructions (these instructions are not supported in 64-bit mode).
 1001     // To distinguish them bits [7:6] are set in the VEX second byte since
 1002     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1003     // those VEX bits REX and vvvv bits are inverted.
 1004     //
 1005     // Fortunately C2 doesn't generate these instructions so we don't need
 1006     // to check for them in product version.
 1007 
 1008     // Check second byte
 1009     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1010 
 1011     int vex_opcode;
 1012     // First byte
 1013     if ((0xFF & *inst) == VEX_3bytes) {
 1014       vex_opcode = VEX_OPCODE_MASK & *ip;
 1015       ip++; // third byte
 1016       is_64bit = ((VEX_W & *ip) == VEX_W);
 1017     } else {
 1018       vex_opcode = VEX_OPCODE_0F;
 1019     }
 1020     ip++; // opcode
 1021     // To find the end of instruction (which == end_pc_operand).
 1022     switch (vex_opcode) {
 1023       case VEX_OPCODE_0F:
 1024         switch (0xFF & *ip) {
 1025         case 0x70: // pshufd r, r/a, #8
 1026         case 0x71: // ps[rl|ra|ll]w r, #8
 1027         case 0x72: // ps[rl|ra|ll]d r, #8
 1028         case 0x73: // ps[rl|ra|ll]q r, #8
 1029         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1030         case 0xC4: // pinsrw r, r, r/a, #8
 1031         case 0xC5: // pextrw r/a, r, #8
 1032         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1033           tail_size = 1;  // the imm8
 1034           break;
 1035         }
 1036         break;
 1037       case VEX_OPCODE_0F_3A:
 1038         tail_size = 1;
 1039         break;
 1040     }
 1041     ip++; // skip opcode
 1042     debug_only(has_disp32 = true); // has both kinds of operands!
 1043     break;
 1044 
 1045   case 0x62: // EVEX_4bytes
 1046     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1047     assert(ip == inst+1, "no prefixes allowed");
 1048     // no EVEX collisions, all instructions that have 0x62 opcodes
 1049     // have EVEX versions and are subopcodes of 0x66
 1050     ip++; // skip P0 and examine W in P1
 1051     is_64bit = ((VEX_W & *ip) == VEX_W);
 1052     ip++; // move to P2
 1053     ip++; // skip P2, move to opcode
 1054     // To find the end of instruction (which == end_pc_operand).
 1055     switch (0xFF & *ip) {
 1056     case 0x22: // pinsrd r, r/a, #8
 1057     case 0x61: // pcmpestri r, r/a, #8
 1058     case 0x70: // pshufd r, r/a, #8
 1059     case 0x73: // psrldq r, #8
 1060     case 0x1f: // evpcmpd/evpcmpq
 1061     case 0x3f: // evpcmpb/evpcmpw
 1062       tail_size = 1;  // the imm8
 1063       break;
 1064     default:
 1065       break;
 1066     }
 1067     ip++; // skip opcode
 1068     debug_only(has_disp32 = true); // has both kinds of operands!
 1069     break;
 1070 
 1071   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1072   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1073   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1074   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1075   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1076   case 0xDF: // fild_d a; fistp_d a
 1077   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1078   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1079   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1080     debug_only(has_disp32 = true);
 1081     break;
 1082 
 1083   case 0xE8: // call rdisp32
 1084   case 0xE9: // jmp  rdisp32
 1085     if (which == end_pc_operand)  return ip + 4;
 1086     assert(which == call32_operand, "call has no disp32 or imm");
 1087     return ip;
 1088 
 1089   case 0xF0:                    // Lock
 1090     goto again_after_prefix;
 1091 
 1092   case 0xF3:                    // For SSE
 1093   case 0xF2:                    // For SSE2
 1094     switch (0xFF & *ip++) {
 1095     case REX:
 1096     case REX_B:
 1097     case REX_X:
 1098     case REX_XB:
 1099     case REX_R:
 1100     case REX_RB:
 1101     case REX_RX:
 1102     case REX_RXB:
 1103     case REX_W:
 1104     case REX_WB:
 1105     case REX_WX:
 1106     case REX_WXB:
 1107     case REX_WR:
 1108     case REX_WRB:
 1109     case REX_WRX:
 1110     case REX_WRXB:
 1111       NOT_LP64(assert(false, "found 64bit prefix"));
 1112       ip++;
 1113     default:
 1114       ip++;
 1115     }
 1116     debug_only(has_disp32 = true); // has both kinds of operands!
 1117     break;
 1118 
 1119   default:
 1120     ShouldNotReachHere();
 1121 
 1122 #undef REP8
 1123 #undef REP16
 1124   }
 1125 
 1126   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1127 #ifdef _LP64
 1128   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1129 #else
 1130   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1131   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1132 #endif // LP64
 1133   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1134 
 1135   // parse the output of emit_operand
 1136   int op2 = 0xFF & *ip++;
 1137   int base = op2 & 0x07;
 1138   int op3 = -1;
 1139   const int b100 = 4;
 1140   const int b101 = 5;
 1141   if (base == b100 && (op2 >> 6) != 3) {
 1142     op3 = 0xFF & *ip++;
 1143     base = op3 & 0x07;   // refetch the base
 1144   }
 1145   // now ip points at the disp (if any)
 1146 
 1147   switch (op2 >> 6) {
 1148   case 0:
 1149     // [00 reg  100][ss index base]
 1150     // [00 reg  100][00   100  esp]
 1151     // [00 reg base]
 1152     // [00 reg  100][ss index  101][disp32]
 1153     // [00 reg  101]               [disp32]
 1154 
 1155     if (base == b101) {
 1156       if (which == disp32_operand)
 1157         return ip;              // caller wants the disp32
 1158       ip += 4;                  // skip the disp32
 1159     }
 1160     break;
 1161 
 1162   case 1:
 1163     // [01 reg  100][ss index base][disp8]
 1164     // [01 reg  100][00   100  esp][disp8]
 1165     // [01 reg base]               [disp8]
 1166     ip += 1;                    // skip the disp8
 1167     break;
 1168 
 1169   case 2:
 1170     // [10 reg  100][ss index base][disp32]
 1171     // [10 reg  100][00   100  esp][disp32]
 1172     // [10 reg base]               [disp32]
 1173     if (which == disp32_operand)
 1174       return ip;                // caller wants the disp32
 1175     ip += 4;                    // skip the disp32
 1176     break;
 1177 
 1178   case 3:
 1179     // [11 reg base]  (not a memory addressing mode)
 1180     break;
 1181   }
 1182 
 1183   if (which == end_pc_operand) {
 1184     return ip + tail_size;
 1185   }
 1186 
 1187 #ifdef _LP64
 1188   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1189 #else
 1190   assert(which == imm_operand, "instruction has only an imm field");
 1191 #endif // LP64
 1192   return ip;
 1193 }
 1194 
 1195 address Assembler::locate_next_instruction(address inst) {
 1196   // Secretly share code with locate_operand:
 1197   return locate_operand(inst, end_pc_operand);
 1198 }
 1199 
 1200 
 1201 #ifdef ASSERT
 1202 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1203   address inst = inst_mark();
 1204   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1205   address opnd;
 1206 
 1207   Relocation* r = rspec.reloc();
 1208   if (r->type() == relocInfo::none) {
 1209     return;
 1210   } else if (r->is_call() || format == call32_operand) {
 1211     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1212     opnd = locate_operand(inst, call32_operand);
 1213   } else if (r->is_data()) {
 1214     assert(format == imm_operand || format == disp32_operand
 1215            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1216     opnd = locate_operand(inst, (WhichOperand)format);
 1217   } else {
 1218     assert(format == imm_operand, "cannot specify a format");
 1219     return;
 1220   }
 1221   assert(opnd == pc(), "must put operand where relocs can find it");
 1222 }
 1223 #endif // ASSERT
 1224 
 1225 void Assembler::emit_operand(Register reg, Address adr,
 1226                              int rip_relative_correction) {
 1227   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1228                adr._rspec,
 1229                rip_relative_correction);
 1230 }
 1231 
 1232 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1233     if (adr.isxmmindex()) {
 1234        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1235     } else {
 1236        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1237        adr._rspec);
 1238     }
 1239 }
 1240 
 1241 // Now the Assembler instructions (identical for 32/64 bits)
 1242 
 1243 void Assembler::adcl(Address dst, int32_t imm32) {
 1244   InstructionMark im(this);
 1245   prefix(dst);
 1246   emit_arith_operand(0x81, rdx, dst, imm32);
 1247 }
 1248 
 1249 void Assembler::adcl(Address dst, Register src) {
 1250   InstructionMark im(this);
 1251   prefix(dst, src);
 1252   emit_int8(0x11);
 1253   emit_operand(src, dst);
 1254 }
 1255 
 1256 void Assembler::adcl(Register dst, int32_t imm32) {
 1257   prefix(dst);
 1258   emit_arith(0x81, 0xD0, dst, imm32);
 1259 }
 1260 
 1261 void Assembler::adcl(Register dst, Address src) {
 1262   InstructionMark im(this);
 1263   prefix(src, dst);
 1264   emit_int8(0x13);
 1265   emit_operand(dst, src);
 1266 }
 1267 
 1268 void Assembler::adcl(Register dst, Register src) {
 1269   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1270   emit_arith(0x13, 0xC0, dst, src);
 1271 }
 1272 
 1273 void Assembler::addl(Address dst, int32_t imm32) {
 1274   InstructionMark im(this);
 1275   prefix(dst);
 1276   emit_arith_operand(0x81, rax, dst, imm32);
 1277 }
 1278 
 1279 void Assembler::addb(Address dst, int imm8) {
 1280   InstructionMark im(this);
 1281   prefix(dst);
 1282   emit_int8((unsigned char)0x80);
 1283   emit_operand(rax, dst, 1);
 1284   emit_int8(imm8);
 1285 }
 1286 
 1287 void Assembler::addw(Register dst, Register src) {
 1288   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1289   emit_arith(0x03, 0xC0, dst, src);
 1290 }
 1291 
 1292 void Assembler::addw(Address dst, int imm16) {
 1293   InstructionMark im(this);
 1294   emit_int8(0x66);
 1295   prefix(dst);
 1296   emit_int8((unsigned char)0x81);
 1297   emit_operand(rax, dst, 2);
 1298   emit_int16(imm16);
 1299 }
 1300 
 1301 void Assembler::addl(Address dst, Register src) {
 1302   InstructionMark im(this);
 1303   prefix(dst, src);
 1304   emit_int8(0x01);
 1305   emit_operand(src, dst);
 1306 }
 1307 
 1308 void Assembler::addl(Register dst, int32_t imm32) {
 1309   prefix(dst);
 1310   emit_arith(0x81, 0xC0, dst, imm32);
 1311 }
 1312 
 1313 void Assembler::addl(Register dst, Address src) {
 1314   InstructionMark im(this);
 1315   prefix(src, dst);
 1316   emit_int8(0x03);
 1317   emit_operand(dst, src);
 1318 }
 1319 
 1320 void Assembler::addl(Register dst, Register src) {
 1321   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1322   emit_arith(0x03, 0xC0, dst, src);
 1323 }
 1324 
 1325 void Assembler::addr_nop_4() {
 1326   assert(UseAddressNop, "no CPU support");
 1327   // 4 bytes: NOP DWORD PTR [EAX+0]
 1328   emit_int32(0x0F,
 1329              0x1F,
 1330              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1331              0);   // 8-bits offset (1 byte)
 1332 }
 1333 
 1334 void Assembler::addr_nop_5() {
 1335   assert(UseAddressNop, "no CPU support");
 1336   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1337   emit_int32(0x0F,
 1338              0x1F,
 1339              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1340              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1341   emit_int8(0);     // 8-bits offset (1 byte)
 1342 }
 1343 
 1344 void Assembler::addr_nop_7() {
 1345   assert(UseAddressNop, "no CPU support");
 1346   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1347   emit_int24(0x0F,
 1348              0x1F,
 1349              (unsigned char)0x80);
 1350                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1351   emit_int32(0);   // 32-bits offset (4 bytes)
 1352 }
 1353 
 1354 void Assembler::addr_nop_8() {
 1355   assert(UseAddressNop, "no CPU support");
 1356   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1357   emit_int32(0x0F,
 1358              0x1F,
 1359              (unsigned char)0x84,
 1360                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1361              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1362   emit_int32(0);    // 32-bits offset (4 bytes)
 1363 }
 1364 
 1365 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_rex_vex_w_reverted();
 1369   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1370   emit_int16(0x58, (0xC0 | encode));
 1371 }
 1372 
 1373 void Assembler::addsd(XMMRegister dst, Address src) {
 1374   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1375   InstructionMark im(this);
 1376   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1377   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1378   attributes.set_rex_vex_w_reverted();
 1379   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1380   emit_int8(0x58);
 1381   emit_operand(dst, src);
 1382 }
 1383 
 1384 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1385   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1387   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1388   emit_int16(0x58, (0xC0 | encode));
 1389 }
 1390 
 1391 void Assembler::addss(XMMRegister dst, Address src) {
 1392   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1393   InstructionMark im(this);
 1394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1395   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1396   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1397   emit_int8(0x58);
 1398   emit_operand(dst, src);
 1399 }
 1400 
 1401 void Assembler::aesdec(XMMRegister dst, Address src) {
 1402   assert(VM_Version::supports_aes(), "");
 1403   InstructionMark im(this);
 1404   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1405   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1406   emit_int8((unsigned char)0xDE);
 1407   emit_operand(dst, src);
 1408 }
 1409 
 1410 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1411   assert(VM_Version::supports_aes(), "");
 1412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1413   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1414   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1415 }
 1416 
 1417 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1418   assert(VM_Version::supports_avx512_vaes(), "");
 1419   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1420   attributes.set_is_evex_instruction();
 1421   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1422   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1423 }
 1424 
 1425 
 1426 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1427   assert(VM_Version::supports_aes(), "");
 1428   InstructionMark im(this);
 1429   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1430   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1431   emit_int8((unsigned char)0xDF);
 1432   emit_operand(dst, src);
 1433 }
 1434 
 1435 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1436   assert(VM_Version::supports_aes(), "");
 1437   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1438   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1439   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1440 }
 1441 
 1442 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1443   assert(VM_Version::supports_avx512_vaes(), "");
 1444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1445   attributes.set_is_evex_instruction();
 1446   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1447   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1448 }
 1449 
 1450 void Assembler::aesenc(XMMRegister dst, Address src) {
 1451   assert(VM_Version::supports_aes(), "");
 1452   InstructionMark im(this);
 1453   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1454   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1455   emit_int8((unsigned char)0xDC);
 1456   emit_operand(dst, src);
 1457 }
 1458 
 1459 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1460   assert(VM_Version::supports_aes(), "");
 1461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1462   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1463   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1464 }
 1465 
 1466 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1467   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1469   attributes.set_is_evex_instruction();
 1470   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1471   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1472 }
 1473 
 1474 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1475   assert(VM_Version::supports_aes(), "");
 1476   InstructionMark im(this);
 1477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1478   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1479   emit_int8((unsigned char)0xDD);
 1480   emit_operand(dst, src);
 1481 }
 1482 
 1483 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1484   assert(VM_Version::supports_aes(), "");
 1485   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1486   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1487   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1488 }
 1489 
 1490 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1491   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1493   attributes.set_is_evex_instruction();
 1494   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1495   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1496 }
 1497 
 1498 void Assembler::andb(Address dst, Register src) {
 1499   InstructionMark im(this);
 1500   prefix(dst, src, true);
 1501   emit_int8(0x20);
 1502   emit_operand(src, dst);
 1503 }
 1504 
 1505 void Assembler::andw(Register dst, Register src) {
 1506   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1507   emit_arith(0x23, 0xC0, dst, src);
 1508 }
 1509 
 1510 void Assembler::andl(Address dst, int32_t imm32) {
 1511   InstructionMark im(this);
 1512   prefix(dst);
 1513   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1514 }
 1515 
 1516 void Assembler::andl(Register dst, int32_t imm32) {
 1517   prefix(dst);
 1518   emit_arith(0x81, 0xE0, dst, imm32);
 1519 }
 1520 
 1521 void Assembler::andl(Address dst, Register src) {
 1522   InstructionMark im(this);
 1523   prefix(dst, src);
 1524   emit_int8(0x21);
 1525   emit_operand(src, dst);
 1526 }
 1527 
 1528 void Assembler::andl(Register dst, Address src) {
 1529   InstructionMark im(this);
 1530   prefix(src, dst);
 1531   emit_int8(0x23);
 1532   emit_operand(dst, src);
 1533 }
 1534 
 1535 void Assembler::andl(Register dst, Register src) {
 1536   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1537   emit_arith(0x23, 0xC0, dst, src);
 1538 }
 1539 
 1540 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1541   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1543   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1544   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1545 }
 1546 
 1547 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1548   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1549   InstructionMark im(this);
 1550   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int8((unsigned char)0xF2);
 1553   emit_operand(dst, src2);
 1554 }
 1555 
 1556 void Assembler::bsfl(Register dst, Register src) {
 1557   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1558   emit_int24(0x0F,
 1559              (unsigned char)0xBC,
 1560              0xC0 | encode);
 1561 }
 1562 
 1563 void Assembler::bsrl(Register dst, Register src) {
 1564   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1565   emit_int24(0x0F,
 1566              (unsigned char)0xBD,
 1567              0xC0 | encode);
 1568 }
 1569 
 1570 void Assembler::bswapl(Register reg) { // bswap
 1571   int encode = prefix_and_encode(reg->encoding());
 1572   emit_int16(0x0F, (0xC8 | encode));
 1573 }
 1574 
 1575 void Assembler::blsil(Register dst, Register src) {
 1576   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1577   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1578   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1579   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1580 }
 1581 
 1582 void Assembler::blsil(Register dst, Address src) {
 1583   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1584   InstructionMark im(this);
 1585   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1586   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1587   emit_int8((unsigned char)0xF3);
 1588   emit_operand(rbx, src);
 1589 }
 1590 
 1591 void Assembler::blsmskl(Register dst, Register src) {
 1592   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1593   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1594   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1595   emit_int16((unsigned char)0xF3,
 1596              0xC0 | encode);
 1597 }
 1598 
 1599 void Assembler::blsmskl(Register dst, Address src) {
 1600   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1601   InstructionMark im(this);
 1602   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1603   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1604   emit_int8((unsigned char)0xF3);
 1605   emit_operand(rdx, src);
 1606 }
 1607 
 1608 void Assembler::blsrl(Register dst, Register src) {
 1609   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1610   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1611   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1612   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1613 }
 1614 
 1615 void Assembler::blsrl(Register dst, Address src) {
 1616   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1617   InstructionMark im(this);
 1618   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1619   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1620   emit_int8((unsigned char)0xF3);
 1621   emit_operand(rcx, src);
 1622 }
 1623 
 1624 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1625   // suspect disp32 is always good
 1626   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1627 
 1628   if (L.is_bound()) {
 1629     const int long_size = 5;
 1630     int offs = (int)( target(L) - pc() );
 1631     assert(offs <= 0, "assembler error");
 1632     InstructionMark im(this);
 1633     // 1110 1000 #32-bit disp
 1634     emit_int8((unsigned char)0xE8);
 1635     emit_data(offs - long_size, rtype, operand);
 1636   } else {
 1637     InstructionMark im(this);
 1638     // 1110 1000 #32-bit disp
 1639     L.add_patch_at(code(), locator());
 1640 
 1641     emit_int8((unsigned char)0xE8);
 1642     emit_data(int(0), rtype, operand);
 1643   }
 1644 }
 1645 
 1646 void Assembler::call(Register dst) {
 1647   int encode = prefix_and_encode(dst->encoding());
 1648   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1649 }
 1650 
 1651 
 1652 void Assembler::call(Address adr) {
 1653   InstructionMark im(this);
 1654   prefix(adr);
 1655   emit_int8((unsigned char)0xFF);
 1656   emit_operand(rdx, adr);
 1657 }
 1658 
 1659 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1660   InstructionMark im(this);
 1661   emit_int8((unsigned char)0xE8);
 1662   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1663   // Entry is NULL in case of a scratch emit.
 1664   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1665   // Technically, should use call32_operand, but this format is
 1666   // implied by the fact that we're emitting a call instruction.
 1667 
 1668   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1669   emit_data((int) disp, rspec, operand);
 1670 }
 1671 
 1672 void Assembler::cdql() {
 1673   emit_int8((unsigned char)0x99);
 1674 }
 1675 
 1676 void Assembler::cld() {
 1677   emit_int8((unsigned char)0xFC);
 1678 }
 1679 
 1680 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1681   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1682   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1683   emit_int24(0x0F,
 1684              0x40 | cc,
 1685              0xC0 | encode);
 1686 }
 1687 
 1688 
 1689 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1690   InstructionMark im(this);
 1691   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1692   prefix(src, dst);
 1693   emit_int16(0x0F, (0x40 | cc));
 1694   emit_operand(dst, src);
 1695 }
 1696 
 1697 void Assembler::cmpb(Address dst, int imm8) {
 1698   InstructionMark im(this);
 1699   prefix(dst);
 1700   emit_int8((unsigned char)0x80);
 1701   emit_operand(rdi, dst, 1);
 1702   emit_int8(imm8);
 1703 }
 1704 
 1705 void Assembler::cmpl(Address dst, int32_t imm32) {
 1706   InstructionMark im(this);
 1707   prefix(dst);
 1708   emit_int8((unsigned char)0x81);
 1709   emit_operand(rdi, dst, 4);
 1710   emit_int32(imm32);
 1711 }
 1712 
 1713 void Assembler::cmpl(Register dst, int32_t imm32) {
 1714   prefix(dst);
 1715   emit_arith(0x81, 0xF8, dst, imm32);
 1716 }
 1717 
 1718 void Assembler::cmpl(Register dst, Register src) {
 1719   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1720   emit_arith(0x3B, 0xC0, dst, src);
 1721 }
 1722 
 1723 void Assembler::cmpl(Register dst, Address  src) {
 1724   InstructionMark im(this);
 1725   prefix(src, dst);
 1726   emit_int8(0x3B);
 1727   emit_operand(dst, src);
 1728 }
 1729 
 1730 void Assembler::cmpw(Address dst, int imm16) {
 1731   InstructionMark im(this);
 1732   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1733   emit_int16(0x66, (unsigned char)0x81);
 1734   emit_operand(rdi, dst, 2);
 1735   emit_int16(imm16);
 1736 }
 1737 
 1738 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1739 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1740 // The ZF is set if the compared values were equal, and cleared otherwise.
 1741 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1742   InstructionMark im(this);
 1743   prefix(adr, reg);
 1744   emit_int16(0x0F, (unsigned char)0xB1);
 1745   emit_operand(reg, adr);
 1746 }
 1747 
 1748 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1749   InstructionMark im(this);
 1750   size_prefix();
 1751   prefix(adr, reg);
 1752   emit_int16(0x0F, (unsigned char)0xB1);
 1753   emit_operand(reg, adr);
 1754 }
 1755 
 1756 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1757 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1758 // The ZF is set if the compared values were equal, and cleared otherwise.
 1759 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1760   InstructionMark im(this);
 1761   prefix(adr, reg, true);
 1762   emit_int16(0x0F, (unsigned char)0xB0);
 1763   emit_operand(reg, adr);
 1764 }
 1765 
 1766 void Assembler::comisd(XMMRegister dst, Address src) {
 1767   // NOTE: dbx seems to decode this as comiss even though the
 1768   // 0x66 is there. Strangely ucomisd comes out correct
 1769   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1770   InstructionMark im(this);
 1771   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1772   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1773   attributes.set_rex_vex_w_reverted();
 1774   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1775   emit_int8(0x2F);
 1776   emit_operand(dst, src);
 1777 }
 1778 
 1779 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1780   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1781   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1782   attributes.set_rex_vex_w_reverted();
 1783   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1784   emit_int16(0x2F, (0xC0 | encode));
 1785 }
 1786 
 1787 void Assembler::comiss(XMMRegister dst, Address src) {
 1788   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1789   InstructionMark im(this);
 1790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1791   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1792   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1793   emit_int8(0x2F);
 1794   emit_operand(dst, src);
 1795 }
 1796 
 1797 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1798   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1800   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1801   emit_int16(0x2F, (0xC0 | encode));
 1802 }
 1803 
 1804 void Assembler::cpuid() {
 1805   emit_int16(0x0F, (unsigned char)0xA2);
 1806 }
 1807 
 1808 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1809 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1810 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1811 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1812 //
 1813 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1814 //
 1815 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1816 //
 1817 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1818 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1819   assert(VM_Version::supports_sse4_2(), "");
 1820   int8_t w = 0x01;
 1821   Prefix p = Prefix_EMPTY;
 1822 
 1823   emit_int8((unsigned char)0xF2);
 1824   switch (sizeInBytes) {
 1825   case 1:
 1826     w = 0;
 1827     break;
 1828   case 2:
 1829   case 4:
 1830     break;
 1831   LP64_ONLY(case 8:)
 1832     // This instruction is not valid in 32 bits
 1833     // Note:
 1834     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1835     //
 1836     // Page B - 72   Vol. 2C says
 1837     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1838     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1839     //                                                                            F0!!!
 1840     // while 3 - 208 Vol. 2A
 1841     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1842     //
 1843     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1844     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1845     p = REX_W;
 1846     break;
 1847   default:
 1848     assert(0, "Unsupported value for a sizeInBytes argument");
 1849     break;
 1850   }
 1851   LP64_ONLY(prefix(crc, v, p);)
 1852   emit_int32(0x0F,
 1853              0x38,
 1854              0xF0 | w,
 1855              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1856 }
 1857 
 1858 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1859   assert(VM_Version::supports_sse4_2(), "");
 1860   InstructionMark im(this);
 1861   int8_t w = 0x01;
 1862   Prefix p = Prefix_EMPTY;
 1863 
 1864   emit_int8((int8_t)0xF2);
 1865   switch (sizeInBytes) {
 1866   case 1:
 1867     w = 0;
 1868     break;
 1869   case 2:
 1870   case 4:
 1871     break;
 1872   LP64_ONLY(case 8:)
 1873     // This instruction is not valid in 32 bits
 1874     p = REX_W;
 1875     break;
 1876   default:
 1877     assert(0, "Unsupported value for a sizeInBytes argument");
 1878     break;
 1879   }
 1880   LP64_ONLY(prefix(crc, adr, p);)
 1881   emit_int24(0x0F, 0x38, (0xF0 | w));
 1882   emit_operand(crc, adr);
 1883 }
 1884 
 1885 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1886   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1889   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1893   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1894   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1895   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1896   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1897 }
 1898 
 1899 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1900   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1901   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1902   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1903   emit_int16(0x5B, (0xC0 | encode));
 1904 }
 1905 
 1906 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1907   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1908   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1909   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1910   emit_int16(0x5B, (0xC0 | encode));
 1911 }
 1912 
 1913 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1914   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1915   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1916   attributes.set_rex_vex_w_reverted();
 1917   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1918   emit_int16(0x5A, (0xC0 | encode));
 1919 }
 1920 
 1921 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1922   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1923   InstructionMark im(this);
 1924   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1925   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1926   attributes.set_rex_vex_w_reverted();
 1927   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1928   emit_int8(0x5A);
 1929   emit_operand(dst, src);
 1930 }
 1931 
 1932 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1933   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1934   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1935   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1936   emit_int16(0x2A, (0xC0 | encode));
 1937 }
 1938 
 1939 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1940   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1941   InstructionMark im(this);
 1942   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1943   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1944   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1945   emit_int8(0x2A);
 1946   emit_operand(dst, src);
 1947 }
 1948 
 1949 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1950   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1951   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1952   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1953   emit_int16(0x2A, (0xC0 | encode));
 1954 }
 1955 
 1956 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1957   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1958   InstructionMark im(this);
 1959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1960   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1961   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1962   emit_int8(0x2A);
 1963   emit_operand(dst, src);
 1964 }
 1965 
 1966 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1967   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1968   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1969   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1970   emit_int16(0x2A, (0xC0 | encode));
 1971 }
 1972 
 1973 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1974   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1975   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1976   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1977   emit_int16(0x5A, (0xC0 | encode));
 1978 }
 1979 
 1980 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1981   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1982   InstructionMark im(this);
 1983   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1984   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1985   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1986   emit_int8(0x5A);
 1987   emit_operand(dst, src);
 1988 }
 1989 
 1990 
 1991 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1993   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1994   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1995   emit_int16(0x2C, (0xC0 | encode));
 1996 }
 1997 
 1998 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 1999   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2000   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2001   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2002   emit_int16(0x2D, (0xC0 | encode));
 2003 }
 2004 
 2005 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2006   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2007   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2008   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2009   emit_int16(0x2C, (0xC0 | encode));
 2010 }
 2011 
 2012 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2013   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2014   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2015   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2016   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2017   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2018 }
 2019 
 2020 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2021   assert(VM_Version::supports_ssse3(), "");
 2022   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2023   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2024   emit_int16(0x1C, (0xC0 | encode));
 2025 }
 2026 
 2027 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2028   assert(VM_Version::supports_ssse3(), "");
 2029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2031   emit_int16(0x1D, (0xC0 | encode));
 2032 }
 2033 
 2034 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2035   assert(VM_Version::supports_ssse3(), "");
 2036   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2037   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2038   emit_int16(0x1E, (0xC0 | encode));
 2039 }
 2040 
 2041 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2042   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2043          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2044          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2045   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2046   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2047   emit_int16(0x1C, (0xC0 | encode));
 2048 }
 2049 
 2050 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2051   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2052          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2053          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2054   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2055   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2056   emit_int16(0x1D, (0xC0 | encode));
 2057 }
 2058 
 2059 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2060   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2061   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2062   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2063   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2065   emit_int16(0x1E, (0xC0 | encode));
 2066 }
 2067 
 2068 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2069   assert(UseAVX > 2, "");
 2070   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2071   attributes.set_is_evex_instruction();
 2072   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2073   emit_int16(0x1F, (0xC0 | encode));
 2074 }
 2075 
 2076 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2077   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2078   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2079   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2080   emit_int16(0x5A, (0xC0 | encode));
 2081 }
 2082 
 2083 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2084   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2085   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2086   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2087   attributes.set_rex_vex_w_reverted();
 2088   emit_int16(0x5A, (0xC0 | encode));
 2089 }
 2090 
 2091 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2092   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2093   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2094   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2095   emit_int16(0x5B, (0xC0 | encode));
 2096 }
 2097 
 2098 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2099   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2100   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2101   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2102   emit_int16(0x5B, (0xC0 | encode));
 2103 }
 2104 
 2105 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2106   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2107   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2108   attributes.set_is_evex_instruction();
 2109   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2110   emit_int16(0x7A, (0xC0 | encode));
 2111 }
 2112 
 2113 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2114   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2115   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2116   attributes.set_is_evex_instruction();
 2117   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2118   emit_int16(0x7B, (0xC0 | encode));
 2119 }
 2120 
 2121 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2122   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2123   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2124   attributes.set_is_evex_instruction();
 2125   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2126   emit_int16(0x5B, (0xC0 | encode));
 2127 }
 2128 
 2129 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2130   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2131   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2132   attributes.set_is_evex_instruction();
 2133   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2134   emit_int16(0x7A, (0xC0 | encode));
 2135 }
 2136 
 2137 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2138   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2139   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2140   attributes.set_is_evex_instruction();
 2141   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2142   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2143 }
 2144 
 2145 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2146   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2147   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   attributes.set_is_evex_instruction();
 2149   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2150   emit_int16(0x30, (0xC0 | encode));
 2151 }
 2152 
 2153 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2154   assert(UseAVX > 2, "");
 2155   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2156   attributes.set_is_evex_instruction();
 2157   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2158   emit_int16(0x33, (0xC0 | encode));
 2159 }
 2160 
 2161 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2162   assert(UseAVX > 2, "");
 2163   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2164   attributes.set_is_evex_instruction();
 2165   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2166   emit_int16(0x31, (0xC0 | encode));
 2167 }
 2168 
 2169 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2170   assert(UseAVX > 2, "");
 2171   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2172   attributes.set_is_evex_instruction();
 2173   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2174   emit_int16(0x35, (0xC0 | encode));
 2175 }
 2176 
 2177 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2178   assert(UseAVX > 2, "");
 2179   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2180   attributes.set_is_evex_instruction();
 2181   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2182   emit_int16(0x32, (0xC0 | encode));
 2183 }
 2184 
 2185 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2186   assert(UseAVX > 2, "");
 2187   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2188   attributes.set_is_evex_instruction();
 2189   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2190   emit_int16(0x34, (0xC0 | encode));
 2191 }
 2192 
 2193 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2194   assert(UseAVX > 2, "");
 2195   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2196   attributes.set_is_evex_instruction();
 2197   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2198   emit_int16(0x25, (0xC0 | encode));
 2199 }
 2200 
 2201 void Assembler::decl(Address dst) {
 2202   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2203   InstructionMark im(this);
 2204   prefix(dst);
 2205   emit_int8((unsigned char)0xFF);
 2206   emit_operand(rcx, dst);
 2207 }
 2208 
 2209 void Assembler::divsd(XMMRegister dst, Address src) {
 2210   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2211   InstructionMark im(this);
 2212   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2213   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2214   attributes.set_rex_vex_w_reverted();
 2215   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2216   emit_int8(0x5E);
 2217   emit_operand(dst, src);
 2218 }
 2219 
 2220 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2221   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2222   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2223   attributes.set_rex_vex_w_reverted();
 2224   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2225   emit_int16(0x5E, (0xC0 | encode));
 2226 }
 2227 
 2228 void Assembler::divss(XMMRegister dst, Address src) {
 2229   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2230   InstructionMark im(this);
 2231   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2232   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2233   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2234   emit_int8(0x5E);
 2235   emit_operand(dst, src);
 2236 }
 2237 
 2238 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2239   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2240   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2241   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2242   emit_int16(0x5E, (0xC0 | encode));
 2243 }
 2244 
 2245 void Assembler::hlt() {
 2246   emit_int8((unsigned char)0xF4);
 2247 }
 2248 
 2249 void Assembler::idivl(Register src) {
 2250   int encode = prefix_and_encode(src->encoding());
 2251   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2252 }
 2253 
 2254 void Assembler::divl(Register src) { // Unsigned
 2255   int encode = prefix_and_encode(src->encoding());
 2256   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2257 }
 2258 
 2259 void Assembler::imull(Register src) {
 2260   int encode = prefix_and_encode(src->encoding());
 2261   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2262 }
 2263 
 2264 void Assembler::imull(Register dst, Register src) {
 2265   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2266   emit_int24(0x0F,
 2267              (unsigned char)0xAF,
 2268              (0xC0 | encode));
 2269 }
 2270 
 2271 void Assembler::imull(Register dst, Address src, int32_t value) {
 2272   InstructionMark im(this);
 2273   prefix(src, dst);
 2274   if (is8bit(value)) {
 2275     emit_int8((unsigned char)0x6B);
 2276     emit_operand(dst, src);
 2277     emit_int8(value);
 2278   } else {
 2279     emit_int8((unsigned char)0x69);
 2280     emit_operand(dst, src);
 2281     emit_int32(value);
 2282   }
 2283 }
 2284 
 2285 void Assembler::imull(Register dst, Register src, int value) {
 2286   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2287   if (is8bit(value)) {
 2288     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2289   } else {
 2290     emit_int16(0x69, (0xC0 | encode));
 2291     emit_int32(value);
 2292   }
 2293 }
 2294 
 2295 void Assembler::imull(Register dst, Address src) {
 2296   InstructionMark im(this);
 2297   prefix(src, dst);
 2298   emit_int16(0x0F, (unsigned char)0xAF);
 2299   emit_operand(dst, src);
 2300 }
 2301 
 2302 
 2303 void Assembler::incl(Address dst) {
 2304   // Don't use it directly. Use MacroAssembler::increment() instead.
 2305   InstructionMark im(this);
 2306   prefix(dst);
 2307   emit_int8((unsigned char)0xFF);
 2308   emit_operand(rax, dst);
 2309 }
 2310 
 2311 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2312   InstructionMark im(this);
 2313   assert((0 <= cc) && (cc < 16), "illegal cc");
 2314   if (L.is_bound()) {
 2315     address dst = target(L);
 2316     assert(dst != NULL, "jcc most probably wrong");
 2317 
 2318     const int short_size = 2;
 2319     const int long_size = 6;
 2320     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2321     if (maybe_short && is8bit(offs - short_size)) {
 2322       // 0111 tttn #8-bit disp
 2323       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2324     } else {
 2325       // 0000 1111 1000 tttn #32-bit disp
 2326       assert(is_simm32(offs - long_size),
 2327              "must be 32bit offset (call4)");
 2328       emit_int16(0x0F, (0x80 | cc));
 2329       emit_int32(offs - long_size);
 2330     }
 2331   } else {
 2332     // Note: could eliminate cond. jumps to this jump if condition
 2333     //       is the same however, seems to be rather unlikely case.
 2334     // Note: use jccb() if label to be bound is very close to get
 2335     //       an 8-bit displacement
 2336     L.add_patch_at(code(), locator());
 2337     emit_int16(0x0F, (0x80 | cc));
 2338     emit_int32(0);
 2339   }
 2340 }
 2341 
 2342 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2343   if (L.is_bound()) {
 2344     const int short_size = 2;
 2345     address entry = target(L);
 2346 #ifdef ASSERT
 2347     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2348     intptr_t delta = short_branch_delta();
 2349     if (delta != 0) {
 2350       dist += (dist < 0 ? (-delta) :delta);
 2351     }
 2352     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2353 #endif
 2354     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2355     // 0111 tttn #8-bit disp
 2356     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2357   } else {
 2358     InstructionMark im(this);
 2359     L.add_patch_at(code(), locator(), file, line);
 2360     emit_int16(0x70 | cc, 0);
 2361   }
 2362 }
 2363 
 2364 void Assembler::jmp(Address adr) {
 2365   InstructionMark im(this);
 2366   prefix(adr);
 2367   emit_int8((unsigned char)0xFF);
 2368   emit_operand(rsp, adr);
 2369 }
 2370 
 2371 void Assembler::jmp(Label& L, bool maybe_short) {
 2372   if (L.is_bound()) {
 2373     address entry = target(L);
 2374     assert(entry != NULL, "jmp most probably wrong");
 2375     InstructionMark im(this);
 2376     const int short_size = 2;
 2377     const int long_size = 5;
 2378     intptr_t offs = entry - pc();
 2379     if (maybe_short && is8bit(offs - short_size)) {
 2380       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2381     } else {
 2382       emit_int8((unsigned char)0xE9);
 2383       emit_int32(offs - long_size);
 2384     }
 2385   } else {
 2386     // By default, forward jumps are always 32-bit displacements, since
 2387     // we can't yet know where the label will be bound.  If you're sure that
 2388     // the forward jump will not run beyond 256 bytes, use jmpb to
 2389     // force an 8-bit displacement.
 2390     InstructionMark im(this);
 2391     L.add_patch_at(code(), locator());
 2392     emit_int8((unsigned char)0xE9);
 2393     emit_int32(0);
 2394   }
 2395 }
 2396 
 2397 void Assembler::jmp(Register entry) {
 2398   int encode = prefix_and_encode(entry->encoding());
 2399   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2400 }
 2401 
 2402 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2403   InstructionMark im(this);
 2404   emit_int8((unsigned char)0xE9);
 2405   assert(dest != NULL, "must have a target");
 2406   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2407   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2408   emit_data(disp, rspec.reloc(), call32_operand);
 2409 }
 2410 
 2411 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2412   if (L.is_bound()) {
 2413     const int short_size = 2;
 2414     address entry = target(L);
 2415     assert(entry != NULL, "jmp most probably wrong");
 2416 #ifdef ASSERT
 2417     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2418     intptr_t delta = short_branch_delta();
 2419     if (delta != 0) {
 2420       dist += (dist < 0 ? (-delta) :delta);
 2421     }
 2422     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2423 #endif
 2424     intptr_t offs = entry - pc();
 2425     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2426   } else {
 2427     InstructionMark im(this);
 2428     L.add_patch_at(code(), locator(), file, line);
 2429     emit_int16((unsigned char)0xEB, 0);
 2430   }
 2431 }
 2432 
 2433 void Assembler::ldmxcsr( Address src) {
 2434   if (UseAVX > 0 ) {
 2435     InstructionMark im(this);
 2436     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2437     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2438     emit_int8((unsigned char)0xAE);
 2439     emit_operand(as_Register(2), src);
 2440   } else {
 2441     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2442     InstructionMark im(this);
 2443     prefix(src);
 2444     emit_int16(0x0F, (unsigned char)0xAE);
 2445     emit_operand(as_Register(2), src);
 2446   }
 2447 }
 2448 
 2449 void Assembler::leal(Register dst, Address src) {
 2450   InstructionMark im(this);
 2451   prefix(src, dst);
 2452   emit_int8((unsigned char)0x8D);
 2453   emit_operand(dst, src);
 2454 }
 2455 
 2456 void Assembler::lfence() {
 2457   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2458 }
 2459 
 2460 void Assembler::lock() {
 2461   emit_int8((unsigned char)0xF0);
 2462 }
 2463 
 2464 void Assembler::size_prefix() {
 2465   emit_int8(0x66);
 2466 }
 2467 
 2468 void Assembler::lzcntl(Register dst, Register src) {
 2469   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2470   emit_int8((unsigned char)0xF3);
 2471   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2472   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2473 }
 2474 
 2475 void Assembler::lzcntl(Register dst, Address src) {
 2476   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2477   InstructionMark im(this);
 2478   emit_int8((unsigned char)0xF3);
 2479   prefix(src, dst);
 2480   emit_int16(0x0F, (unsigned char)0xBD);
 2481   emit_operand(dst, src);
 2482 }
 2483 
 2484 // Emit mfence instruction
 2485 void Assembler::mfence() {
 2486   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2487   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2488 }
 2489 
 2490 // Emit sfence instruction
 2491 void Assembler::sfence() {
 2492   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2493   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2494 }
 2495 
 2496 void Assembler::mov(Register dst, Register src) {
 2497   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2498 }
 2499 
 2500 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2501   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2502   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2503   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2504   attributes.set_rex_vex_w_reverted();
 2505   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2506   emit_int16(0x28, (0xC0 | encode));
 2507 }
 2508 
 2509 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2510   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2511   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2512   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2513   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2514   emit_int16(0x28, (0xC0 | encode));
 2515 }
 2516 
 2517 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2518   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2519   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2520   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2521   emit_int16(0x16, (0xC0 | encode));
 2522 }
 2523 
 2524 void Assembler::movb(Register dst, Address src) {
 2525   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2526   InstructionMark im(this);
 2527   prefix(src, dst, true);
 2528   emit_int8((unsigned char)0x8A);
 2529   emit_operand(dst, src);
 2530 }
 2531 
 2532 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2533   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2534   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2535   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2536   attributes.set_rex_vex_w_reverted();
 2537   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2538   emit_int16(0x12, 0xC0 | encode);
 2539 }
 2540 
 2541 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2542   assert(VM_Version::supports_avx(), "");
 2543   InstructionMark im(this);
 2544   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2545   attributes.set_rex_vex_w_reverted();
 2546   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2547   emit_int8(0x12);
 2548   emit_operand(dst, src);
 2549 }
 2550 
 2551 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2552   assert(VM_Version::supports_avx512dq(), "");
 2553   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2554   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2555   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2556 }
 2557 
 2558 void Assembler::kmovbl(KRegister dst, Register src) {
 2559   assert(VM_Version::supports_avx512dq(), "");
 2560   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2561   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2562   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2563 }
 2564 
 2565 void Assembler::kmovbl(Register dst, KRegister src) {
 2566   assert(VM_Version::supports_avx512dq(), "");
 2567   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2568   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2569   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2570 }
 2571 
 2572 void Assembler::kmovwl(KRegister dst, Register src) {
 2573   assert(VM_Version::supports_evex(), "");
 2574   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2575   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2576   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2577 }
 2578 
 2579 void Assembler::kmovwl(Register dst, KRegister src) {
 2580   assert(VM_Version::supports_evex(), "");
 2581   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2582   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2583   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2584 }
 2585 
 2586 void Assembler::kmovwl(KRegister dst, Address src) {
 2587   assert(VM_Version::supports_evex(), "");
 2588   InstructionMark im(this);
 2589   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2590   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2591   emit_int8((unsigned char)0x90);
 2592   emit_operand((Register)dst, src);
 2593 }
 2594 
 2595 void Assembler::kmovwl(Address dst, KRegister src) {
 2596   assert(VM_Version::supports_evex(), "");
 2597   InstructionMark im(this);
 2598   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2599   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2600   emit_int8((unsigned char)0x91);
 2601   emit_operand((Register)src, dst);
 2602 }
 2603 
 2604 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2605   assert(VM_Version::supports_evex(), "");
 2606   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2607   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2608   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2609 }
 2610 
 2611 void Assembler::kmovdl(KRegister dst, Register src) {
 2612   assert(VM_Version::supports_avx512bw(), "");
 2613   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2614   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2615   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2616 }
 2617 
 2618 void Assembler::kmovdl(Register dst, KRegister src) {
 2619   assert(VM_Version::supports_avx512bw(), "");
 2620   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2621   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2622   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2623 }
 2624 
 2625 void Assembler::kmovql(KRegister dst, KRegister src) {
 2626   assert(VM_Version::supports_avx512bw(), "");
 2627   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2628   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2629   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2630 }
 2631 
 2632 void Assembler::kmovql(KRegister dst, Address src) {
 2633   assert(VM_Version::supports_avx512bw(), "");
 2634   InstructionMark im(this);
 2635   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2636   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2637   emit_int8((unsigned char)0x90);
 2638   emit_operand((Register)dst, src);
 2639 }
 2640 
 2641 void Assembler::kmovql(Address dst, KRegister src) {
 2642   assert(VM_Version::supports_avx512bw(), "");
 2643   InstructionMark im(this);
 2644   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2645   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2646   emit_int8((unsigned char)0x91);
 2647   emit_operand((Register)src, dst);
 2648 }
 2649 
 2650 void Assembler::kmovql(KRegister dst, Register src) {
 2651   assert(VM_Version::supports_avx512bw(), "");
 2652   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2653   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2654   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2655 }
 2656 
 2657 void Assembler::kmovql(Register dst, KRegister src) {
 2658   assert(VM_Version::supports_avx512bw(), "");
 2659   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2660   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2661   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2662 }
 2663 
 2664 void Assembler::knotwl(KRegister dst, KRegister src) {
 2665   assert(VM_Version::supports_evex(), "");
 2666   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2667   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2668   emit_int16(0x44, (0xC0 | encode));
 2669 }
 2670 
 2671 void Assembler::knotbl(KRegister dst, KRegister src) {
 2672   assert(VM_Version::supports_evex(), "");
 2673   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2674   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2675   emit_int16(0x44, (0xC0 | encode));
 2676 }
 2677 
 2678 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2679   assert(VM_Version::supports_avx512dq(), "");
 2680   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2681   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2682   emit_int16(0x45, (0xC0 | encode));
 2683 }
 2684 
 2685 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2686   assert(VM_Version::supports_evex(), "");
 2687   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2688   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2689   emit_int16(0x45, (0xC0 | encode));
 2690 }
 2691 
 2692 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2693   assert(VM_Version::supports_avx512bw(), "");
 2694   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2695   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2696   emit_int16(0x45, (0xC0 | encode));
 2697 }
 2698 
 2699 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2700   assert(VM_Version::supports_avx512bw(), "");
 2701   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2702   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2703   emit_int16(0x45, (0xC0 | encode));
 2704 }
 2705 
 2706 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2707   assert(VM_Version::supports_avx512dq(), "");
 2708   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2709   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2710   emit_int16(0x47, (0xC0 | encode));
 2711 }
 2712 
 2713 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2714   assert(VM_Version::supports_evex(), "");
 2715   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2716   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2717   emit_int16(0x47, (0xC0 | encode));
 2718 }
 2719 
 2720 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2721   assert(VM_Version::supports_avx512bw(), "");
 2722   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2723   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2724   emit_int16(0x47, (0xC0 | encode));
 2725 }
 2726 
 2727 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2728   assert(VM_Version::supports_avx512bw(), "");
 2729   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2730   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2731   emit_int16(0x47, (0xC0 | encode));
 2732 }
 2733 
 2734 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2735   assert(VM_Version::supports_avx512dq(), "");
 2736   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2737   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2738   emit_int16(0x41, (0xC0 | encode));
 2739 }
 2740 
 2741 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2742   assert(VM_Version::supports_evex(), "");
 2743   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2744   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2745   emit_int16(0x41, (0xC0 | encode));
 2746 }
 2747 
 2748 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2749   assert(VM_Version::supports_avx512bw(), "");
 2750   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2751   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2752   emit_int16(0x41, (0xC0 | encode));
 2753 }
 2754 
 2755 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2756   assert(VM_Version::supports_avx512bw(), "");
 2757   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2758   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2759   emit_int16(0x41, (0xC0 | encode));
 2760 }
 2761 
 2762 void Assembler::knotdl(KRegister dst, KRegister src) {
 2763   assert(VM_Version::supports_avx512bw(), "");
 2764   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2765   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2766   emit_int16(0x44, (0xC0 | encode));
 2767 }
 2768 
 2769 void Assembler::knotql(KRegister dst, KRegister src) {
 2770   assert(VM_Version::supports_avx512bw(), "");
 2771   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2772   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2773   emit_int16(0x44, (0xC0 | encode));
 2774 }
 2775 
 2776 // This instruction produces ZF or CF flags
 2777 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2778   assert(VM_Version::supports_avx512dq(), "");
 2779   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2780   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2781   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2782 }
 2783 
 2784 // This instruction produces ZF or CF flags
 2785 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2786   assert(VM_Version::supports_evex(), "");
 2787   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2788   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2789   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2790 }
 2791 
 2792 // This instruction produces ZF or CF flags
 2793 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2794   assert(VM_Version::supports_avx512bw(), "");
 2795   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2796   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2797   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2798 }
 2799 
 2800 // This instruction produces ZF or CF flags
 2801 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2802   assert(VM_Version::supports_avx512bw(), "");
 2803   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2804   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2805   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2806 }
 2807 
 2808 // This instruction produces ZF or CF flags
 2809 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2810   assert(VM_Version::supports_avx512bw(), "");
 2811   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2812   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2813   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2814 }
 2815 
 2816 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2817   assert(VM_Version::supports_avx512bw(), "");
 2818   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2819   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2820   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2821 }
 2822 
 2823 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2824   assert(VM_Version::supports_avx512dq(), "");
 2825   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2826   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2827   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2828 }
 2829 
 2830 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2831   assert(VM_Version::supports_avx512dq(), "");
 2832   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2833   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2834   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2835 }
 2836 
 2837 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2838   assert(VM_Version::supports_avx512bw(), "");
 2839   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2840   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2841   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2842 }
 2843 
 2844 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2845   assert(VM_Version::supports_avx512bw(), "");
 2846   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2847   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2848   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2849 }
 2850 
 2851 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2852   assert(VM_Version::supports_avx512dq(), "");
 2853   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2854   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2855   emit_int16(0x46, (0xC0 | encode));
 2856 }
 2857 
 2858 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2859   assert(VM_Version::supports_avx512dq(), "");
 2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2861   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2862   emit_int16(0x32, (0xC0 | encode));
 2863   emit_int8(imm8);
 2864 }
 2865 
 2866 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2867   assert(VM_Version::supports_avx512bw(), "");
 2868   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2869   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2870   emit_int16(0x33, (0xC0 | encode));
 2871   emit_int8(imm8);
 2872 }
 2873 
 2874 
 2875 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2876   assert(VM_Version::supports_avx512dq(), "");
 2877   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2878   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2879   emit_int16(0x30, (0xC0 | encode));
 2880 }
 2881 
 2882 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2883   assert(VM_Version::supports_evex(), "");
 2884   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2885   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2886   emit_int16(0x30, (0xC0 | encode));
 2887   emit_int8(imm8);
 2888 }
 2889 
 2890 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2891   assert(VM_Version::supports_avx512bw(), "");
 2892   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2893   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2894   emit_int16(0x31, (0xC0 | encode));
 2895   emit_int8(imm8);
 2896 }
 2897 
 2898 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2899   assert(VM_Version::supports_avx512bw(), "");
 2900   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2901   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2902   emit_int16(0x31, (0xC0 | encode));
 2903   emit_int8(imm8);
 2904 }
 2905 
 2906 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 2907   assert(VM_Version::supports_avx512bw(), "");
 2908   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2909   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2910   emit_int16(0x4B, (0xC0 | encode));
 2911 }
 2912 
 2913 void Assembler::movb(Address dst, int imm8) {
 2914   InstructionMark im(this);
 2915    prefix(dst);
 2916   emit_int8((unsigned char)0xC6);
 2917   emit_operand(rax, dst, 1);
 2918   emit_int8(imm8);
 2919 }
 2920 
 2921 
 2922 void Assembler::movb(Address dst, Register src) {
 2923   assert(src->has_byte_register(), "must have byte register");
 2924   InstructionMark im(this);
 2925   prefix(dst, src, true);
 2926   emit_int8((unsigned char)0x88);
 2927   emit_operand(src, dst);
 2928 }
 2929 
 2930 void Assembler::movdl(XMMRegister dst, Register src) {
 2931   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2932   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2933   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2934   emit_int16(0x6E, (0xC0 | encode));
 2935 }
 2936 
 2937 void Assembler::movdl(Register dst, XMMRegister src) {
 2938   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2939   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2940   // swap src/dst to get correct prefix
 2941   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2942   emit_int16(0x7E, (0xC0 | encode));
 2943 }
 2944 
 2945 void Assembler::movdl(XMMRegister dst, Address src) {
 2946   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2947   InstructionMark im(this);
 2948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2949   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2950   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2951   emit_int8(0x6E);
 2952   emit_operand(dst, src);
 2953 }
 2954 
 2955 void Assembler::movdl(Address dst, XMMRegister src) {
 2956   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2957   InstructionMark im(this);
 2958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2959   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2960   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2961   emit_int8(0x7E);
 2962   emit_operand(src, dst);
 2963 }
 2964 
 2965 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2966   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2967   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2968   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2969   emit_int16(0x6F, (0xC0 | encode));
 2970 }
 2971 
 2972 void Assembler::movdqa(XMMRegister dst, Address src) {
 2973   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2974   InstructionMark im(this);
 2975   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2976   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2977   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2978   emit_int8(0x6F);
 2979   emit_operand(dst, src);
 2980 }
 2981 
 2982 void Assembler::movdqu(XMMRegister dst, Address src) {
 2983   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2984   InstructionMark im(this);
 2985   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2986   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2987   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2988   emit_int8(0x6F);
 2989   emit_operand(dst, src);
 2990 }
 2991 
 2992 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2993   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2995   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2996   emit_int16(0x6F, (0xC0 | encode));
 2997 }
 2998 
 2999 void Assembler::movdqu(Address dst, XMMRegister src) {
 3000   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3001   InstructionMark im(this);
 3002   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3003   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3004   attributes.reset_is_clear_context();
 3005   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3006   emit_int8(0x7F);
 3007   emit_operand(src, dst);
 3008 }
 3009 
 3010 // Move Unaligned 256bit Vector
 3011 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 3012   assert(UseAVX > 0, "");
 3013   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3014   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3015   emit_int16(0x6F, (0xC0 | encode));
 3016 }
 3017 
 3018 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3019   assert(UseAVX > 0, "");
 3020   InstructionMark im(this);
 3021   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3022   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3023   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3024   emit_int8(0x6F);
 3025   emit_operand(dst, src);
 3026 }
 3027 
 3028 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3029   assert(UseAVX > 0, "");
 3030   InstructionMark im(this);
 3031   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3032   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3033   attributes.reset_is_clear_context();
 3034   // swap src<->dst for encoding
 3035   assert(src != xnoreg, "sanity");
 3036   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3037   emit_int8(0x7F);
 3038   emit_operand(src, dst);
 3039 }
 3040 
 3041 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3042   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3043   InstructionMark im(this);
 3044   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3045   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3046   emit_int8((unsigned char)0x8C);
 3047   emit_operand(dst, src);
 3048 }
 3049 
 3050 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3051   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3052   InstructionMark im(this);
 3053   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3054   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3055   emit_int8((unsigned char)0x8C);
 3056   emit_operand(dst, src);
 3057 }
 3058 
 3059 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3060   assert(UseAVX > 0, "requires some form of AVX");
 3061   InstructionMark im(this);
 3062   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3063   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3064   emit_int8(0x2C);
 3065   emit_operand(dst, src);
 3066 }
 3067 
 3068 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3069   assert(UseAVX > 0, "requires some form of AVX");
 3070   InstructionMark im(this);
 3071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3072   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3073   emit_int8(0x2D);
 3074   emit_operand(dst, src);
 3075 }
 3076 
 3077 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3078   assert(UseAVX > 0, "");
 3079   InstructionMark im(this);
 3080   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3081   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3082   emit_int8(0x2E);
 3083   emit_operand(src, dst);
 3084 }
 3085 
 3086 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3087   assert(UseAVX > 0, "");
 3088   InstructionMark im(this);
 3089   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3090   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3091   emit_int8(0x2F);
 3092   emit_operand(src, dst);
 3093 }
 3094 
 3095 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3096 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3097   assert(VM_Version::supports_avx512vlbw(), "");
 3098   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3099   attributes.set_embedded_opmask_register_specifier(mask);
 3100   attributes.set_is_evex_instruction();
 3101   if (merge) {
 3102     attributes.reset_is_clear_context();
 3103   }
 3104   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3105   emit_int16(0x6F, (0xC0 | encode));
 3106 }
 3107 
 3108 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
 3109   // Unmasked instruction
 3110   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3111 }
 3112 
 3113 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3114   assert(VM_Version::supports_avx512vlbw(), "");
 3115   InstructionMark im(this);
 3116   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3117   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3118   attributes.set_embedded_opmask_register_specifier(mask);
 3119   attributes.set_is_evex_instruction();
 3120   if (merge) {
 3121     attributes.reset_is_clear_context();
 3122   }
 3123   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3124   emit_int8(0x6F);
 3125   emit_operand(dst, src);
 3126 }
 3127 
 3128 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
 3129   // Unmasked instruction
 3130   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3131 }
 3132 
 3133 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3134   assert(VM_Version::supports_avx512vlbw(), "");
 3135   assert(src != xnoreg, "sanity");
 3136   InstructionMark im(this);
 3137   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3138   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3139   attributes.set_embedded_opmask_register_specifier(mask);
 3140   attributes.set_is_evex_instruction();
 3141   if (merge) {
 3142     attributes.reset_is_clear_context();
 3143   }
 3144   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3145   emit_int8(0x7F);
 3146   emit_operand(src, dst);
 3147 }
 3148 
 3149 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
 3150   // Unmasked instruction
 3151   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3152 }
 3153 
 3154 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3155   assert(VM_Version::supports_avx512vlbw(), "");
 3156   InstructionMark im(this);
 3157   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3158   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3159   attributes.set_embedded_opmask_register_specifier(mask);
 3160   attributes.set_is_evex_instruction();
 3161   if (merge) {
 3162     attributes.reset_is_clear_context();
 3163   }
 3164   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3165   emit_int8(0x6F);
 3166   emit_operand(dst, src);
 3167 }
 3168 
 3169 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
 3170   // Unmasked instruction
 3171   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3172 }
 3173 
 3174 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3175   assert(VM_Version::supports_avx512vlbw(), "");
 3176   assert(src != xnoreg, "sanity");
 3177   InstructionMark im(this);
 3178   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3179   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3180   attributes.set_embedded_opmask_register_specifier(mask);
 3181   attributes.set_is_evex_instruction();
 3182   if (merge) {
 3183     attributes.reset_is_clear_context();
 3184   }
 3185   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3186   emit_int8(0x7F);
 3187   emit_operand(src, dst);
 3188 }
 3189 
 3190 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3191   // Unmasked instruction
 3192   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3193 }
 3194 
 3195 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3196   assert(VM_Version::supports_evex(), "");
 3197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3198   attributes.set_embedded_opmask_register_specifier(mask);
 3199   attributes.set_is_evex_instruction();
 3200   if (merge) {
 3201     attributes.reset_is_clear_context();
 3202   }
 3203   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3204   emit_int16(0x6F, (0xC0 | encode));
 3205 }
 3206 
 3207 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3208   // Unmasked instruction
 3209   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3210 }
 3211 
 3212 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3213   assert(VM_Version::supports_evex(), "");
 3214   InstructionMark im(this);
 3215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3216   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3217   attributes.set_embedded_opmask_register_specifier(mask);
 3218   attributes.set_is_evex_instruction();
 3219   if (merge) {
 3220     attributes.reset_is_clear_context();
 3221   }
 3222   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3223   emit_int8(0x6F);
 3224   emit_operand(dst, src);
 3225 }
 3226 
 3227 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3228   // Unmasked isntruction
 3229   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3230 }
 3231 
 3232 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3233   assert(VM_Version::supports_evex(), "");
 3234   assert(src != xnoreg, "sanity");
 3235   InstructionMark im(this);
 3236   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3237   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3238   attributes.set_embedded_opmask_register_specifier(mask);
 3239   attributes.set_is_evex_instruction();
 3240   if (merge) {
 3241     attributes.reset_is_clear_context();
 3242   }
 3243   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3244   emit_int8(0x7F);
 3245   emit_operand(src, dst);
 3246 }
 3247 
 3248 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3249   // Unmasked instruction
 3250   if (dst->encoding() == src->encoding()) return;
 3251   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3252 }
 3253 
 3254 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3255   assert(VM_Version::supports_evex(), "");
 3256   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3257   attributes.set_embedded_opmask_register_specifier(mask);
 3258   attributes.set_is_evex_instruction();
 3259   if (merge) {
 3260     attributes.reset_is_clear_context();
 3261   }
 3262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3263   emit_int16(0x6F, (0xC0 | encode));
 3264 }
 3265 
 3266 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3267   // Unmasked instruction
 3268   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3269 }
 3270 
 3271 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3272   assert(VM_Version::supports_evex(), "");
 3273   InstructionMark im(this);
 3274   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3275   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3276   attributes.set_embedded_opmask_register_specifier(mask);
 3277   attributes.set_is_evex_instruction();
 3278   if (merge) {
 3279     attributes.reset_is_clear_context();
 3280   }
 3281   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3282   emit_int8(0x6F);
 3283   emit_operand(dst, src);
 3284 }
 3285 
 3286 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3287   // Unmasked instruction
 3288   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3289 }
 3290 
 3291 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3292   assert(VM_Version::supports_evex(), "");
 3293   assert(src != xnoreg, "sanity");
 3294   InstructionMark im(this);
 3295   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3296   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3297   attributes.set_embedded_opmask_register_specifier(mask);
 3298   if (merge) {
 3299     attributes.reset_is_clear_context();
 3300   }
 3301   attributes.set_is_evex_instruction();
 3302   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3303   emit_int8(0x7F);
 3304   emit_operand(src, dst);
 3305 }
 3306 
 3307 // Uses zero extension on 64bit
 3308 
 3309 void Assembler::movl(Register dst, int32_t imm32) {
 3310   int encode = prefix_and_encode(dst->encoding());
 3311   emit_int8(0xB8 | encode);
 3312   emit_int32(imm32);
 3313 }
 3314 
 3315 void Assembler::movl(Register dst, Register src) {
 3316   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3317   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3318 }
 3319 
 3320 void Assembler::movl(Register dst, Address src) {
 3321   InstructionMark im(this);
 3322   prefix(src, dst);
 3323   emit_int8((unsigned char)0x8B);
 3324   emit_operand(dst, src);
 3325 }
 3326 
 3327 void Assembler::movl(Address dst, int32_t imm32) {
 3328   InstructionMark im(this);
 3329   prefix(dst);
 3330   emit_int8((unsigned char)0xC7);
 3331   emit_operand(rax, dst, 4);
 3332   emit_int32(imm32);
 3333 }
 3334 
 3335 void Assembler::movl(Address dst, Register src) {
 3336   InstructionMark im(this);
 3337   prefix(dst, src);
 3338   emit_int8((unsigned char)0x89);
 3339   emit_operand(src, dst);
 3340 }
 3341 
 3342 // New cpus require to use movsd and movss to avoid partial register stall
 3343 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3344 // The selection is done in MacroAssembler::movdbl() and movflt().
 3345 void Assembler::movlpd(XMMRegister dst, Address src) {
 3346   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3347   InstructionMark im(this);
 3348   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3349   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3350   attributes.set_rex_vex_w_reverted();
 3351   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3352   emit_int8(0x12);
 3353   emit_operand(dst, src);
 3354 }
 3355 
 3356 void Assembler::movq(XMMRegister dst, Address src) {
 3357   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3358   InstructionMark im(this);
 3359   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3360   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3361   attributes.set_rex_vex_w_reverted();
 3362   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3363   emit_int8(0x7E);
 3364   emit_operand(dst, src);
 3365 }
 3366 
 3367 void Assembler::movq(Address dst, XMMRegister src) {
 3368   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3369   InstructionMark im(this);
 3370   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3371   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3372   attributes.set_rex_vex_w_reverted();
 3373   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3374   emit_int8((unsigned char)0xD6);
 3375   emit_operand(src, dst);
 3376 }
 3377 
 3378 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3379   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3380   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3381   attributes.set_rex_vex_w_reverted();
 3382   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3383   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3384 }
 3385 
 3386 void Assembler::movq(Register dst, XMMRegister src) {
 3387   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3388   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3389   // swap src/dst to get correct prefix
 3390   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3391   emit_int16(0x7E, (0xC0 | encode));
 3392 }
 3393 
 3394 void Assembler::movq(XMMRegister dst, Register src) {
 3395   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3396   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3397   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3398   emit_int16(0x6E, (0xC0 | encode));
 3399 }
 3400 
 3401 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3402   InstructionMark im(this);
 3403   prefix(src, dst);
 3404   emit_int16(0x0F, (unsigned char)0xBE);
 3405   emit_operand(dst, src);
 3406 }
 3407 
 3408 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3409   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3410   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3411   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3412 }
 3413 
 3414 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3415   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3416   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3417   attributes.set_rex_vex_w_reverted();
 3418   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3419   emit_int16(0x10, (0xC0 | encode));
 3420 }
 3421 
 3422 void Assembler::movsd(XMMRegister dst, Address src) {
 3423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3424   InstructionMark im(this);
 3425   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3426   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3427   attributes.set_rex_vex_w_reverted();
 3428   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3429   emit_int8(0x10);
 3430   emit_operand(dst, src);
 3431 }
 3432 
 3433 void Assembler::movsd(Address dst, XMMRegister src) {
 3434   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3435   InstructionMark im(this);
 3436   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3437   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3438   attributes.reset_is_clear_context();
 3439   attributes.set_rex_vex_w_reverted();
 3440   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3441   emit_int8(0x11);
 3442   emit_operand(src, dst);
 3443 }
 3444 
 3445 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3446   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3447   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3448   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3449   emit_int16(0x10, (0xC0 | encode));
 3450 }
 3451 
 3452 void Assembler::movss(XMMRegister dst, Address src) {
 3453   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3454   InstructionMark im(this);
 3455   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3456   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3457   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3458   emit_int8(0x10);
 3459   emit_operand(dst, src);
 3460 }
 3461 
 3462 void Assembler::movss(Address dst, XMMRegister src) {
 3463   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3464   InstructionMark im(this);
 3465   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3466   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3467   attributes.reset_is_clear_context();
 3468   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3469   emit_int8(0x11);
 3470   emit_operand(src, dst);
 3471 }
 3472 
 3473 void Assembler::movswl(Register dst, Address src) { // movsxw
 3474   InstructionMark im(this);
 3475   prefix(src, dst);
 3476   emit_int16(0x0F, (unsigned char)0xBF);
 3477   emit_operand(dst, src);
 3478 }
 3479 
 3480 void Assembler::movswl(Register dst, Register src) { // movsxw
 3481   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3482   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3483 }
 3484 
 3485 void Assembler::movw(Address dst, int imm16) {
 3486   InstructionMark im(this);
 3487 
 3488   emit_int8(0x66); // switch to 16-bit mode
 3489   prefix(dst);
 3490   emit_int8((unsigned char)0xC7);
 3491   emit_operand(rax, dst, 2);
 3492   emit_int16(imm16);
 3493 }
 3494 
 3495 void Assembler::movw(Register dst, Address src) {
 3496   InstructionMark im(this);
 3497   emit_int8(0x66);
 3498   prefix(src, dst);
 3499   emit_int8((unsigned char)0x8B);
 3500   emit_operand(dst, src);
 3501 }
 3502 
 3503 void Assembler::movw(Address dst, Register src) {
 3504   InstructionMark im(this);
 3505   emit_int8(0x66);
 3506   prefix(dst, src);
 3507   emit_int8((unsigned char)0x89);
 3508   emit_operand(src, dst);
 3509 }
 3510 
 3511 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3512   InstructionMark im(this);
 3513   prefix(src, dst);
 3514   emit_int16(0x0F, (unsigned char)0xB6);
 3515   emit_operand(dst, src);
 3516 }
 3517 
 3518 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3519   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3520   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3521   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3522 }
 3523 
 3524 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3525   InstructionMark im(this);
 3526   prefix(src, dst);
 3527   emit_int16(0x0F, (unsigned char)0xB7);
 3528   emit_operand(dst, src);
 3529 }
 3530 
 3531 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3532   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3533   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3534 }
 3535 
 3536 void Assembler::mull(Address src) {
 3537   InstructionMark im(this);
 3538   prefix(src);
 3539   emit_int8((unsigned char)0xF7);
 3540   emit_operand(rsp, src);
 3541 }
 3542 
 3543 void Assembler::mull(Register src) {
 3544   int encode = prefix_and_encode(src->encoding());
 3545   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3546 }
 3547 
 3548 void Assembler::mulsd(XMMRegister dst, Address src) {
 3549   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3550   InstructionMark im(this);
 3551   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3552   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3553   attributes.set_rex_vex_w_reverted();
 3554   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3555   emit_int8(0x59);
 3556   emit_operand(dst, src);
 3557 }
 3558 
 3559 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3560   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3561   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3562   attributes.set_rex_vex_w_reverted();
 3563   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3564   emit_int16(0x59, (0xC0 | encode));
 3565 }
 3566 
 3567 void Assembler::mulss(XMMRegister dst, Address src) {
 3568   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3569   InstructionMark im(this);
 3570   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3571   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3572   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3573   emit_int8(0x59);
 3574   emit_operand(dst, src);
 3575 }
 3576 
 3577 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3578   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3579   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3580   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3581   emit_int16(0x59, (0xC0 | encode));
 3582 }
 3583 
 3584 void Assembler::negl(Register dst) {
 3585   int encode = prefix_and_encode(dst->encoding());
 3586   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3587 }
 3588 
 3589 void Assembler::negl(Address dst) {
 3590   InstructionMark im(this);
 3591   prefix(dst);
 3592   emit_int8((unsigned char)0xF7);
 3593   emit_operand(as_Register(3), dst);
 3594 }
 3595 
 3596 void Assembler::nop(int i) {
 3597 #ifdef ASSERT
 3598   assert(i > 0, " ");
 3599   // The fancy nops aren't currently recognized by debuggers making it a
 3600   // pain to disassemble code while debugging. If asserts are on clearly
 3601   // speed is not an issue so simply use the single byte traditional nop
 3602   // to do alignment.
 3603 
 3604   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3605   return;
 3606 
 3607 #endif // ASSERT
 3608 
 3609   if (UseAddressNop && VM_Version::is_intel()) {
 3610     //
 3611     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3612     //  1: 0x90
 3613     //  2: 0x66 0x90
 3614     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3615     //  4: 0x0F 0x1F 0x40 0x00
 3616     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3617     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3618     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3619     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3620     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3621     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3622     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3623 
 3624     // The rest coding is Intel specific - don't use consecutive address nops
 3625 
 3626     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3627     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3628     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3629     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3630 
 3631     while(i >= 15) {
 3632       // For Intel don't generate consecutive address nops (mix with regular nops)
 3633       i -= 15;
 3634       emit_int24(0x66, 0x66, 0x66);
 3635       addr_nop_8();
 3636       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3637     }
 3638     switch (i) {
 3639       case 14:
 3640         emit_int8(0x66); // size prefix
 3641       case 13:
 3642         emit_int8(0x66); // size prefix
 3643       case 12:
 3644         addr_nop_8();
 3645         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3646         break;
 3647       case 11:
 3648         emit_int8(0x66); // size prefix
 3649       case 10:
 3650         emit_int8(0x66); // size prefix
 3651       case 9:
 3652         emit_int8(0x66); // size prefix
 3653       case 8:
 3654         addr_nop_8();
 3655         break;
 3656       case 7:
 3657         addr_nop_7();
 3658         break;
 3659       case 6:
 3660         emit_int8(0x66); // size prefix
 3661       case 5:
 3662         addr_nop_5();
 3663         break;
 3664       case 4:
 3665         addr_nop_4();
 3666         break;
 3667       case 3:
 3668         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3669         emit_int8(0x66); // size prefix
 3670       case 2:
 3671         emit_int8(0x66); // size prefix
 3672       case 1:
 3673         emit_int8((unsigned char)0x90);
 3674                          // nop
 3675         break;
 3676       default:
 3677         assert(i == 0, " ");
 3678     }
 3679     return;
 3680   }
 3681   if (UseAddressNop && VM_Version::is_amd_family()) {
 3682     //
 3683     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3684     //  1: 0x90
 3685     //  2: 0x66 0x90
 3686     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3687     //  4: 0x0F 0x1F 0x40 0x00
 3688     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3689     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3690     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3691     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3692     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3693     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3694     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3695 
 3696     // The rest coding is AMD specific - use consecutive address nops
 3697 
 3698     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3699     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3700     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3701     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3702     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3703     //     Size prefixes (0x66) are added for larger sizes
 3704 
 3705     while(i >= 22) {
 3706       i -= 11;
 3707       emit_int24(0x66, 0x66, 0x66);
 3708       addr_nop_8();
 3709     }
 3710     // Generate first nop for size between 21-12
 3711     switch (i) {
 3712       case 21:
 3713         i -= 1;
 3714         emit_int8(0x66); // size prefix
 3715       case 20:
 3716       case 19:
 3717         i -= 1;
 3718         emit_int8(0x66); // size prefix
 3719       case 18:
 3720       case 17:
 3721         i -= 1;
 3722         emit_int8(0x66); // size prefix
 3723       case 16:
 3724       case 15:
 3725         i -= 8;
 3726         addr_nop_8();
 3727         break;
 3728       case 14:
 3729       case 13:
 3730         i -= 7;
 3731         addr_nop_7();
 3732         break;
 3733       case 12:
 3734         i -= 6;
 3735         emit_int8(0x66); // size prefix
 3736         addr_nop_5();
 3737         break;
 3738       default:
 3739         assert(i < 12, " ");
 3740     }
 3741 
 3742     // Generate second nop for size between 11-1
 3743     switch (i) {
 3744       case 11:
 3745         emit_int8(0x66); // size prefix
 3746       case 10:
 3747         emit_int8(0x66); // size prefix
 3748       case 9:
 3749         emit_int8(0x66); // size prefix
 3750       case 8:
 3751         addr_nop_8();
 3752         break;
 3753       case 7:
 3754         addr_nop_7();
 3755         break;
 3756       case 6:
 3757         emit_int8(0x66); // size prefix
 3758       case 5:
 3759         addr_nop_5();
 3760         break;
 3761       case 4:
 3762         addr_nop_4();
 3763         break;
 3764       case 3:
 3765         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3766         emit_int8(0x66); // size prefix
 3767       case 2:
 3768         emit_int8(0x66); // size prefix
 3769       case 1:
 3770         emit_int8((unsigned char)0x90);
 3771                          // nop
 3772         break;
 3773       default:
 3774         assert(i == 0, " ");
 3775     }
 3776     return;
 3777   }
 3778 
 3779   if (UseAddressNop && VM_Version::is_zx()) {
 3780     //
 3781     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3782     //  1: 0x90
 3783     //  2: 0x66 0x90
 3784     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3785     //  4: 0x0F 0x1F 0x40 0x00
 3786     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3787     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3788     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3789     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3790     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3791     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3792     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3793 
 3794     // The rest coding is ZX specific - don't use consecutive address nops
 3795 
 3796     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3797     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3798     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3799     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3800 
 3801     while (i >= 15) {
 3802       // For ZX don't generate consecutive address nops (mix with regular nops)
 3803       i -= 15;
 3804       emit_int24(0x66, 0x66, 0x66);
 3805       addr_nop_8();
 3806       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3807     }
 3808     switch (i) {
 3809       case 14:
 3810         emit_int8(0x66); // size prefix
 3811       case 13:
 3812         emit_int8(0x66); // size prefix
 3813       case 12:
 3814         addr_nop_8();
 3815         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3816         break;
 3817       case 11:
 3818         emit_int8(0x66); // size prefix
 3819       case 10:
 3820         emit_int8(0x66); // size prefix
 3821       case 9:
 3822         emit_int8(0x66); // size prefix
 3823       case 8:
 3824         addr_nop_8();
 3825         break;
 3826       case 7:
 3827         addr_nop_7();
 3828         break;
 3829       case 6:
 3830         emit_int8(0x66); // size prefix
 3831       case 5:
 3832         addr_nop_5();
 3833         break;
 3834       case 4:
 3835         addr_nop_4();
 3836         break;
 3837       case 3:
 3838         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3839         emit_int8(0x66); // size prefix
 3840       case 2:
 3841         emit_int8(0x66); // size prefix
 3842       case 1:
 3843         emit_int8((unsigned char)0x90);
 3844                          // nop
 3845         break;
 3846       default:
 3847         assert(i == 0, " ");
 3848     }
 3849     return;
 3850   }
 3851 
 3852   // Using nops with size prefixes "0x66 0x90".
 3853   // From AMD Optimization Guide:
 3854   //  1: 0x90
 3855   //  2: 0x66 0x90
 3856   //  3: 0x66 0x66 0x90
 3857   //  4: 0x66 0x66 0x66 0x90
 3858   //  5: 0x66 0x66 0x90 0x66 0x90
 3859   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3860   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3861   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3862   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3863   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3864   //
 3865   while (i > 12) {
 3866     i -= 4;
 3867     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3868   }
 3869   // 1 - 12 nops
 3870   if (i > 8) {
 3871     if (i > 9) {
 3872       i -= 1;
 3873       emit_int8(0x66);
 3874     }
 3875     i -= 3;
 3876     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3877   }
 3878   // 1 - 8 nops
 3879   if (i > 4) {
 3880     if (i > 6) {
 3881       i -= 1;
 3882       emit_int8(0x66);
 3883     }
 3884     i -= 3;
 3885     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3886   }
 3887   switch (i) {
 3888     case 4:
 3889       emit_int8(0x66);
 3890     case 3:
 3891       emit_int8(0x66);
 3892     case 2:
 3893       emit_int8(0x66);
 3894     case 1:
 3895       emit_int8((unsigned char)0x90);
 3896       break;
 3897     default:
 3898       assert(i == 0, " ");
 3899   }
 3900 }
 3901 
 3902 void Assembler::notl(Register dst) {
 3903   int encode = prefix_and_encode(dst->encoding());
 3904   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 3905 }
 3906 
 3907 void Assembler::orw(Register dst, Register src) {
 3908   (void)prefix_and_encode(dst->encoding(), src->encoding());
 3909   emit_arith(0x0B, 0xC0, dst, src);
 3910 }
 3911 
 3912 void Assembler::orl(Address dst, int32_t imm32) {
 3913   InstructionMark im(this);
 3914   prefix(dst);
 3915   emit_arith_operand(0x81, rcx, dst, imm32);
 3916 }
 3917 
 3918 void Assembler::orl(Register dst, int32_t imm32) {
 3919   prefix(dst);
 3920   emit_arith(0x81, 0xC8, dst, imm32);
 3921 }
 3922 
 3923 void Assembler::orl(Register dst, Address src) {
 3924   InstructionMark im(this);
 3925   prefix(src, dst);
 3926   emit_int8(0x0B);
 3927   emit_operand(dst, src);
 3928 }
 3929 
 3930 void Assembler::orl(Register dst, Register src) {
 3931   (void) prefix_and_encode(dst->encoding(), src->encoding());
 3932   emit_arith(0x0B, 0xC0, dst, src);
 3933 }
 3934 
 3935 void Assembler::orl(Address dst, Register src) {
 3936   InstructionMark im(this);
 3937   prefix(dst, src);
 3938   emit_int8(0x09);
 3939   emit_operand(src, dst);
 3940 }
 3941 
 3942 void Assembler::orb(Address dst, int imm8) {
 3943   InstructionMark im(this);
 3944   prefix(dst);
 3945   emit_int8((unsigned char)0x80);
 3946   emit_operand(rcx, dst, 1);
 3947   emit_int8(imm8);
 3948 }
 3949 
 3950 void Assembler::orb(Address dst, Register src) {
 3951   InstructionMark im(this);
 3952   prefix(dst, src, true);
 3953   emit_int8(0x08);
 3954   emit_operand(src, dst);
 3955 }
 3956 
 3957 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 3958   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3960   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3961   emit_int16(0x63, (0xC0 | encode));
 3962 }
 3963 
 3964 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3965   assert(UseAVX > 0, "some form of AVX must be enabled");
 3966   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3967   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3968   emit_int16(0x63, (0xC0 | encode));
 3969 }
 3970 
 3971 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 3972   assert(VM_Version::supports_sse2(), "");
 3973   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3974   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3975   emit_int16(0x6B, (0xC0 | encode));
 3976 }
 3977 
 3978 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3979   assert(UseAVX > 0, "some form of AVX must be enabled");
 3980   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3981   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3982   emit_int16(0x6B, (0xC0 | encode));
 3983 }
 3984 
 3985 void Assembler::packuswb(XMMRegister dst, Address src) {
 3986   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3987   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 3988   InstructionMark im(this);
 3989   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3990   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 3991   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3992   emit_int8(0x67);
 3993   emit_operand(dst, src);
 3994 }
 3995 
 3996 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 3997   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3998   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3999   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4000   emit_int16(0x67, (0xC0 | encode));
 4001 }
 4002 
 4003 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4004   assert(UseAVX > 0, "some form of AVX must be enabled");
 4005   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4006   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4007   emit_int16(0x67, (0xC0 | encode));
 4008 }
 4009 
 4010 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4011   assert(VM_Version::supports_sse4_1(), "");
 4012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4013   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4014   emit_int16(0x2B, (0xC0 | encode));
 4015 }
 4016 
 4017 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4018   assert(UseAVX > 0, "some form of AVX must be enabled");
 4019   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4020   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4021   emit_int16(0x2B, (0xC0 | encode));
 4022 }
 4023 
 4024 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4025   assert(VM_Version::supports_avx2(), "");
 4026   assert(vector_len != AVX_128bit, "");
 4027   // VEX.256.66.0F3A.W1 00 /r ib
 4028   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4029   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4030   emit_int24(0x00, (0xC0 | encode), imm8);
 4031 }
 4032 
 4033 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4034   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4035          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4036   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4037   attributes.set_is_evex_instruction();
 4038   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4039   emit_int16(0x36, (0xC0 | encode));
 4040 }
 4041 
 4042 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4043   assert(VM_Version::supports_avx512_vbmi(), "");
 4044   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4045   attributes.set_is_evex_instruction();
 4046   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4047   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4048 }
 4049 
 4050 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4051   assert(VM_Version::supports_avx512_vbmi(), "");
 4052   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4053   attributes.set_is_evex_instruction();
 4054   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4055   emit_int8((unsigned char)0x8D);
 4056   emit_operand(dst, src);
 4057 }
 4058 
 4059 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4060   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4061          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4062          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4063   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4064   attributes.set_is_evex_instruction();
 4065   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4066   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4067 }
 4068 
 4069 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4070   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4071   // VEX.NDS.256.66.0F38.W0 36 /r
 4072   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4073   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4074   emit_int16(0x36, (0xC0 | encode));
 4075 }
 4076 
 4077 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4078   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4079   // VEX.NDS.256.66.0F38.W0 36 /r
 4080   InstructionMark im(this);
 4081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4082   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4083   emit_int8(0x36);
 4084   emit_operand(dst, src);
 4085 }
 4086 
 4087 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4088   assert(VM_Version::supports_avx2(), "");
 4089   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4090   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4091   emit_int24(0x46, (0xC0 | encode), imm8);
 4092 }
 4093 
 4094 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4095   assert(VM_Version::supports_avx(), "");
 4096   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4097   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4098   emit_int24(0x06, (0xC0 | encode), imm8);
 4099 }
 4100 
 4101 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4102   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4103   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4104   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4105   emit_int24(0x04, (0xC0 | encode), imm8);
 4106 }
 4107 
 4108 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4109   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4110   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4111   attributes.set_rex_vex_w_reverted();
 4112   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4113   emit_int24(0x05, (0xC0 | encode), imm8);
 4114 }
 4115 
 4116 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4117   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4118   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4119   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4120   emit_int24(0x01, (0xC0 | encode), imm8);
 4121 }
 4122 
 4123 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4124   assert(VM_Version::supports_evex(), "");
 4125   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4126   attributes.set_is_evex_instruction();
 4127   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4128   emit_int16(0x76, (0xC0 | encode));
 4129 }
 4130 
 4131 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4132   assert(VM_Version::supports_avx512_vbmi(), "");
 4133   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4134   attributes.set_is_evex_instruction();
 4135   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4136   emit_int16(0x7D, (0xC0 | encode));
 4137 }
 4138 
 4139 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4140   assert(VM_Version::supports_avx512_vbmi(), "");
 4141   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4142   attributes.set_is_evex_instruction();
 4143   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4144   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4145 }
 4146 
 4147 void Assembler::pause() {
 4148   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4149 }
 4150 
 4151 void Assembler::ud2() {
 4152   emit_int16(0x0F, 0x0B);
 4153 }
 4154 
 4155 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4156   assert(VM_Version::supports_sse4_2(), "");
 4157   InstructionMark im(this);
 4158   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4159   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4160   emit_int8(0x61);
 4161   emit_operand(dst, src);
 4162   emit_int8(imm8);
 4163 }
 4164 
 4165 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4166   assert(VM_Version::supports_sse4_2(), "");
 4167   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4169   emit_int24(0x61, (0xC0 | encode), imm8);
 4170 }
 4171 
 4172 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4173 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4174   assert(VM_Version::supports_sse2(), "");
 4175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4176   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4177   emit_int16(0x74, (0xC0 | encode));
 4178 }
 4179 
 4180 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4181   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4182   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4183   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4184   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4185   emit_int16(cond_encoding, (0xC0 | encode));
 4186 }
 4187 
 4188 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4189 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4190   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4191   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4192   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4193   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4194   emit_int16(0x74, (0xC0 | encode));
 4195 }
 4196 
 4197 // In this context, kdst is written the mask used to process the equal components
 4198 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4199   assert(VM_Version::supports_avx512bw(), "");
 4200   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4201   attributes.set_is_evex_instruction();
 4202   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4203   emit_int16(0x74, (0xC0 | encode));
 4204 }
 4205 
 4206 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4207   assert(VM_Version::supports_avx512vlbw(), "");
 4208   InstructionMark im(this);
 4209   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4210   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4211   attributes.set_is_evex_instruction();
 4212   int dst_enc = kdst->encoding();
 4213   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4214   emit_int8(0x64);
 4215   emit_operand(as_Register(dst_enc), src);
 4216 }
 4217 
 4218 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4219   assert(VM_Version::supports_avx512vlbw(), "");
 4220   InstructionMark im(this);
 4221   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4222   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4223   attributes.reset_is_clear_context();
 4224   attributes.set_embedded_opmask_register_specifier(mask);
 4225   attributes.set_is_evex_instruction();
 4226   int dst_enc = kdst->encoding();
 4227   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4228   emit_int8(0x64);
 4229   emit_operand(as_Register(dst_enc), src);
 4230 }
 4231 
 4232 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4233   assert(VM_Version::supports_avx512vlbw(), "");
 4234   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4235   attributes.set_is_evex_instruction();
 4236   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4237   emit_int24(0x3E, (0xC0 | encode), vcc);
 4238 }
 4239 
 4240 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4241   assert(VM_Version::supports_avx512vlbw(), "");
 4242   InstructionMark im(this);
 4243   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4244   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4245   attributes.set_is_evex_instruction();
 4246   int dst_enc = kdst->encoding();
 4247   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4248   emit_int8(0x3E);
 4249   emit_operand(as_Register(dst_enc), src);
 4250   emit_int8(vcc);
 4251 }
 4252 
 4253 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4254   assert(VM_Version::supports_avx512bw(), "");
 4255   InstructionMark im(this);
 4256   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4257   attributes.set_is_evex_instruction();
 4258   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4259   int dst_enc = kdst->encoding();
 4260   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4261   emit_int8(0x74);
 4262   emit_operand(as_Register(dst_enc), src);
 4263 }
 4264 
 4265 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4266   assert(VM_Version::supports_avx512vlbw(), "");
 4267   InstructionMark im(this);
 4268   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4269   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4270   attributes.reset_is_clear_context();
 4271   attributes.set_embedded_opmask_register_specifier(mask);
 4272   attributes.set_is_evex_instruction();
 4273   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4274   emit_int8(0x74);
 4275   emit_operand(as_Register(kdst->encoding()), src);
 4276 }
 4277 
 4278 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4279 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4280   assert(VM_Version::supports_sse2(), "");
 4281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4282   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4283   emit_int16(0x75, (0xC0 | encode));
 4284 }
 4285 
 4286 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4287 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4288   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4289   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4290   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4291   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4292   emit_int16(0x75, (0xC0 | encode));
 4293 }
 4294 
 4295 // In this context, kdst is written the mask used to process the equal components
 4296 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4297   assert(VM_Version::supports_avx512bw(), "");
 4298   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4299   attributes.set_is_evex_instruction();
 4300   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4301   emit_int16(0x75, (0xC0 | encode));
 4302 }
 4303 
 4304 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4305   assert(VM_Version::supports_avx512bw(), "");
 4306   InstructionMark im(this);
 4307   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4308   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4309   attributes.set_is_evex_instruction();
 4310   int dst_enc = kdst->encoding();
 4311   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4312   emit_int8(0x75);
 4313   emit_operand(as_Register(dst_enc), src);
 4314 }
 4315 
 4316 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4317 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4318   assert(VM_Version::supports_sse2(), "");
 4319   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4320   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4321   emit_int16(0x76, (0xC0 | encode));
 4322 }
 4323 
 4324 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4325 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4326   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4327   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4329   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4330   emit_int16(0x76, (0xC0 | encode));
 4331 }
 4332 
 4333 // In this context, kdst is written the mask used to process the equal components
 4334 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4335   assert(VM_Version::supports_evex(), "");
 4336   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4337   attributes.set_is_evex_instruction();
 4338   attributes.reset_is_clear_context();
 4339   attributes.set_embedded_opmask_register_specifier(mask);
 4340   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4341   emit_int16(0x76, (0xC0 | encode));
 4342 }
 4343 
 4344 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4345   assert(VM_Version::supports_evex(), "");
 4346   InstructionMark im(this);
 4347   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4348   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4349   attributes.set_is_evex_instruction();
 4350   attributes.reset_is_clear_context();
 4351   attributes.set_embedded_opmask_register_specifier(mask);
 4352   int dst_enc = kdst->encoding();
 4353   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4354   emit_int8(0x76);
 4355   emit_operand(as_Register(dst_enc), src);
 4356 }
 4357 
 4358 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4359 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4360   assert(VM_Version::supports_sse4_1(), "");
 4361   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4362   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4363   emit_int16(0x29, (0xC0 | encode));
 4364 }
 4365 
 4366 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4367   assert(VM_Version::supports_evex(), "");
 4368   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4369   attributes.set_is_evex_instruction();
 4370   attributes.reset_is_clear_context();
 4371   attributes.set_embedded_opmask_register_specifier(mask);
 4372   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4373   emit_int16(0x29, (0xC0 | encode));
 4374 }
 4375 
 4376 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4377   assert(VM_Version::supports_avx(), "");
 4378   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4379   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4380   emit_int16(cond_encoding, (0xC0 | encode));
 4381 }
 4382 
 4383 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4384 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4385   assert(VM_Version::supports_avx(), "");
 4386   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4387   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4388   emit_int16(0x29, (0xC0 | encode));
 4389 }
 4390 
 4391 // In this context, kdst is written the mask used to process the equal components
 4392 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4393   assert(VM_Version::supports_evex(), "");
 4394   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4395   attributes.reset_is_clear_context();
 4396   attributes.set_is_evex_instruction();
 4397   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4398   emit_int16(0x29, (0xC0 | encode));
 4399 }
 4400 
 4401 // In this context, kdst is written the mask used to process the equal components
 4402 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4403   assert(VM_Version::supports_evex(), "");
 4404   InstructionMark im(this);
 4405   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4406   attributes.reset_is_clear_context();
 4407   attributes.set_is_evex_instruction();
 4408   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4409   int dst_enc = kdst->encoding();
 4410   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4411   emit_int8(0x29);
 4412   emit_operand(as_Register(dst_enc), src);
 4413 }
 4414 
 4415 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4416   assert(VM_Version::supports_sse4_1(), "");
 4417   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4418   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4419   emit_int16(0x37, (0xC0 | encode));
 4420 }
 4421 
 4422 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4423   assert(VM_Version::supports_sse2(), "");
 4424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4425   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4426   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4427 }
 4428 
 4429 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4430   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4431          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4432   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4433   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4434   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4435 }
 4436 
 4437 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4438   assert(VM_Version::supports_avx(), "");
 4439   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4440   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4441   emit_int16(0x50, (0xC0 | encode));
 4442 }
 4443 
 4444 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4445   assert(VM_Version::supports_avx(), "");
 4446   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4447   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4448   emit_int16(0x50, (0xC0 | encode));
 4449 }
 4450 
 4451 
 4452 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4453   assert(VM_Version::supports_sse4_1(), "");
 4454   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4455   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4456   emit_int24(0x16, (0xC0 | encode), imm8);
 4457 }
 4458 
 4459 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4460   assert(VM_Version::supports_sse4_1(), "");
 4461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4462   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4463   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4464   emit_int8(0x16);
 4465   emit_operand(src, dst);
 4466   emit_int8(imm8);
 4467 }
 4468 
 4469 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4470   assert(VM_Version::supports_sse4_1(), "");
 4471   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4472   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4473   emit_int24(0x16, (0xC0 | encode), imm8);
 4474 }
 4475 
 4476 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4477   assert(VM_Version::supports_sse4_1(), "");
 4478   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4479   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4480   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4481   emit_int8(0x16);
 4482   emit_operand(src, dst);
 4483   emit_int8(imm8);
 4484 }
 4485 
 4486 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4487   assert(VM_Version::supports_sse2(), "");
 4488   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4489   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4490   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4491 }
 4492 
 4493 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4494   assert(VM_Version::supports_sse4_1(), "");
 4495   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4496   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4497   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4498   emit_int8(0x15);
 4499   emit_operand(src, dst);
 4500   emit_int8(imm8);
 4501 }
 4502 
 4503 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4504   assert(VM_Version::supports_sse4_1(), "");
 4505   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4506   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4507   emit_int24(0x14, (0xC0 | encode), imm8);
 4508 }
 4509 
 4510 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4511   assert(VM_Version::supports_sse4_1(), "");
 4512   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4513   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4514   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4515   emit_int8(0x14);
 4516   emit_operand(src, dst);
 4517   emit_int8(imm8);
 4518 }
 4519 
 4520 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4521   assert(VM_Version::supports_sse4_1(), "");
 4522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4523   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4524   emit_int24(0x22, (0xC0 | encode), imm8);
 4525 }
 4526 
 4527 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4528   assert(VM_Version::supports_sse4_1(), "");
 4529   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4530   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4531   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4532   emit_int8(0x22);
 4533   emit_operand(dst,src);
 4534   emit_int8(imm8);
 4535 }
 4536 
 4537 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4538   assert(VM_Version::supports_avx(), "");
 4539   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4540   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4541   emit_int24(0x22, (0xC0 | encode), imm8);
 4542 }
 4543 
 4544 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4545   assert(VM_Version::supports_sse4_1(), "");
 4546   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4547   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4548   emit_int24(0x22, (0xC0 | encode), imm8);
 4549 }
 4550 
 4551 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4552   assert(VM_Version::supports_sse4_1(), "");
 4553   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4554   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4555   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4556   emit_int8(0x22);
 4557   emit_operand(dst, src);
 4558   emit_int8(imm8);
 4559 }
 4560 
 4561 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4562   assert(VM_Version::supports_avx(), "");
 4563   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4564   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4565   emit_int24(0x22, (0xC0 | encode), imm8);
 4566 }
 4567 
 4568 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4569   assert(VM_Version::supports_sse2(), "");
 4570   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4571   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4572   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4573 }
 4574 
 4575 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4576   assert(VM_Version::supports_sse2(), "");
 4577   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4578   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4579   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4580   emit_int8((unsigned char)0xC4);
 4581   emit_operand(dst, src);
 4582   emit_int8(imm8);
 4583 }
 4584 
 4585 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4586   assert(VM_Version::supports_avx(), "");
 4587   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4588   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4589   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4590 }
 4591 
 4592 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4593   assert(VM_Version::supports_sse4_1(), "");
 4594   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4595   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4596   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4597   emit_int8(0x20);
 4598   emit_operand(dst, src);
 4599   emit_int8(imm8);
 4600 }
 4601 
 4602 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4603   assert(VM_Version::supports_sse4_1(), "");
 4604   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4605   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4606   emit_int24(0x20, (0xC0 | encode), imm8);
 4607 }
 4608 
 4609 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4610   assert(VM_Version::supports_avx(), "");
 4611   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4613   emit_int24(0x20, (0xC0 | encode), imm8);
 4614 }
 4615 
 4616 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4617   assert(VM_Version::supports_sse4_1(), "");
 4618   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4619   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4620   emit_int24(0x21, (0xC0 | encode), imm8);
 4621 }
 4622 
 4623 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4624   assert(VM_Version::supports_avx(), "");
 4625   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4626   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4627   emit_int24(0x21, (0xC0 | encode), imm8);
 4628 }
 4629 
 4630 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4631   assert(VM_Version::supports_sse4_1(), "");
 4632   InstructionMark im(this);
 4633   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4634   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4635   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4636   emit_int8(0x30);
 4637   emit_operand(dst, src);
 4638 }
 4639 
 4640 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4641   assert(VM_Version::supports_sse4_1(), "");
 4642   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4643   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4644   emit_int16(0x30, (0xC0 | encode));
 4645 }
 4646 
 4647 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4648   assert(VM_Version::supports_sse4_1(), "");
 4649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4650   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4651   emit_int16(0x20, (0xC0 | encode));
 4652 }
 4653 
 4654 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4655   assert(VM_Version::supports_sse4_1(), "");
 4656   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4657   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4658   emit_int16(0x35, (0xC0 | encode));
 4659 }
 4660 
 4661 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4662   assert(VM_Version::supports_sse4_1(), "");
 4663   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4664   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4665   emit_int16(0x21, (0xC0 | encode));
 4666 }
 4667 
 4668 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4669   assert(VM_Version::supports_sse4_1(), "");
 4670   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4671   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4672   emit_int16(0x31, (0xC0 | encode));
 4673 }
 4674 
 4675 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4676   assert(VM_Version::supports_sse4_1(), "");
 4677   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4678   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4679   emit_int16(0x22, (0xC0 | encode));
 4680 }
 4681 
 4682 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4683   assert(VM_Version::supports_sse4_1(), "");
 4684   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4685   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4686   emit_int16(0x23, (0xC0 | encode));
 4687 }
 4688 
 4689 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4690   assert(VM_Version::supports_avx(), "");
 4691   InstructionMark im(this);
 4692   assert(dst != xnoreg, "sanity");
 4693   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4694   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4695   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4696   emit_int8(0x30);
 4697   emit_operand(dst, src);
 4698 }
 4699 
 4700 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4701   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4702   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4703   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4704   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4705   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4706   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4707 }
 4708 
 4709 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4710   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4711   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4712   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4713   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4714   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4715   emit_int16(0x20, (0xC0 | encode));
 4716 }
 4717 
 4718 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4719   assert(VM_Version::supports_avx512vlbw(), "");
 4720   assert(dst != xnoreg, "sanity");
 4721   InstructionMark im(this);
 4722   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4723   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4724   attributes.set_embedded_opmask_register_specifier(mask);
 4725   attributes.set_is_evex_instruction();
 4726   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4727   emit_int8(0x30);
 4728   emit_operand(dst, src);
 4729 }
 4730 
 4731 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4732   assert(VM_Version::supports_evex(), "");
 4733   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4734   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4735   attributes.set_is_evex_instruction();
 4736   attributes.set_embedded_opmask_register_specifier(mask);
 4737   if (merge) {
 4738     attributes.reset_is_clear_context();
 4739   }
 4740   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4741   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4742 }
 4743 
 4744 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4745   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4746   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4747   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4748   emit_int16(0x35, (0xC0 | encode));
 4749 }
 4750 
 4751 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4752   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4753   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4754   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4755   emit_int16(0x31, (0xC0 | encode));
 4756 }
 4757 
 4758 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4759   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4760   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4761   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4762   emit_int16(0x32, (0xC0 | encode));
 4763 }
 4764 
 4765 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4766   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4767          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4768              VM_Version::supports_evex(), "");
 4769   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4770   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4771   emit_int16(0x21, (0xC0 | encode));
 4772 }
 4773 
 4774 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4775   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4776          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4777              VM_Version::supports_evex(), "");
 4778   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4779   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4780   emit_int16(0x22, (0xC0 | encode));
 4781 }
 4782 
 4783 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4784   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4785          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4786              VM_Version::supports_evex(), "");
 4787   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4788   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4789   emit_int16(0x23, (0xC0 | encode));
 4790 }
 4791 
 4792 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4793   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4794          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4795              VM_Version::supports_evex(), "");
 4796   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4797   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4798   emit_int16(0x24, (0xC0 | encode));
 4799 }
 4800 
 4801 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4802   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4803          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4804              VM_Version::supports_evex(), "");
 4805   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4806   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4807   emit_int16(0x25, (0xC0 | encode));
 4808 }
 4809 
 4810 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4811   assert(VM_Version::supports_avx512vlbw(), "");
 4812   assert(src != xnoreg, "sanity");
 4813   InstructionMark im(this);
 4814   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4815   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4816   attributes.set_is_evex_instruction();
 4817   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4818   emit_int8(0x30);
 4819   emit_operand(src, dst);
 4820 }
 4821 
 4822 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4823   assert(VM_Version::supports_avx512vlbw(), "");
 4824   assert(src != xnoreg, "sanity");
 4825   InstructionMark im(this);
 4826   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4827   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4828   attributes.reset_is_clear_context();
 4829   attributes.set_embedded_opmask_register_specifier(mask);
 4830   attributes.set_is_evex_instruction();
 4831   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4832   emit_int8(0x30);
 4833   emit_operand(src, dst);
 4834 }
 4835 
 4836 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4837   assert(VM_Version::supports_evex(), "");
 4838   assert(src != xnoreg, "sanity");
 4839   InstructionMark im(this);
 4840   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4841   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4842   attributes.set_is_evex_instruction();
 4843   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4844   emit_int8(0x31);
 4845   emit_operand(src, dst);
 4846 }
 4847 
 4848 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4849   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4850   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4851   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4852   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4853   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4854   emit_int16(0x33, (0xC0 | encode));
 4855 }
 4856 
 4857 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4858   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4859   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4860   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4861   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4862   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4863   emit_int16(0x34, (0xC0 | encode));
 4864 }
 4865 
 4866 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4869   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4870   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4871 }
 4872 
 4873 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4874   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4875     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4876     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4877   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4878   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4879   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4880 }
 4881 
 4882 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4883 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4884        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4885        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4886   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4887   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4888   emit_int16(0x04, (0xC0 | encode));
 4889 }
 4890 
 4891 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4892   assert(VM_Version::supports_evex(), "");
 4893   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4894   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4895   attributes.set_is_evex_instruction();
 4896   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4897   emit_int16(0x52, (0xC0 | encode));
 4898 }
 4899 
 4900 // generic
 4901 void Assembler::pop(Register dst) {
 4902   int encode = prefix_and_encode(dst->encoding());
 4903   emit_int8(0x58 | encode);
 4904 }
 4905 
 4906 void Assembler::popcntl(Register dst, Address src) {
 4907   assert(VM_Version::supports_popcnt(), "must support");
 4908   InstructionMark im(this);
 4909   emit_int8((unsigned char)0xF3);
 4910   prefix(src, dst);
 4911   emit_int16(0x0F, (unsigned char)0xB8);
 4912   emit_operand(dst, src);
 4913 }
 4914 
 4915 void Assembler::popcntl(Register dst, Register src) {
 4916   assert(VM_Version::supports_popcnt(), "must support");
 4917   emit_int8((unsigned char)0xF3);
 4918   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 4919   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 4920 }
 4921 
 4922 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4923   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 4924   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4926   attributes.set_embedded_opmask_register_specifier(mask);
 4927   attributes.set_is_evex_instruction();
 4928   if (merge) {
 4929     attributes.reset_is_clear_context();
 4930   }
 4931   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4932   emit_int16(0x54, (0xC0 | encode));
 4933 }
 4934 
 4935 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4936   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 4937   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4938   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4939   attributes.set_is_evex_instruction();
 4940   attributes.set_embedded_opmask_register_specifier(mask);
 4941   if (merge) {
 4942     attributes.reset_is_clear_context();
 4943   }
 4944   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4945   emit_int16(0x54, (0xC0 | encode));
 4946 }
 4947 
 4948 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4949   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4950   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4951   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4952   attributes.set_is_evex_instruction();
 4953   attributes.set_embedded_opmask_register_specifier(mask);
 4954   if (merge) {
 4955     attributes.reset_is_clear_context();
 4956   }
 4957   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4958   emit_int16(0x55, (0xC0 | encode));
 4959 }
 4960 
 4961 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4962   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4963   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4964   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4965   attributes.set_is_evex_instruction();
 4966   attributes.set_embedded_opmask_register_specifier(mask);
 4967   if (merge) {
 4968     attributes.reset_is_clear_context();
 4969   }
 4970   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4971   emit_int16(0x55, (0xC0 | encode));
 4972 }
 4973 
 4974 void Assembler::popf() {
 4975   emit_int8((unsigned char)0x9D);
 4976 }
 4977 
 4978 #ifndef _LP64 // no 32bit push/pop on amd64
 4979 void Assembler::popl(Address dst) {
 4980   // NOTE: this will adjust stack by 8byte on 64bits
 4981   InstructionMark im(this);
 4982   prefix(dst);
 4983   emit_int8((unsigned char)0x8F);
 4984   emit_operand(rax, dst);
 4985 }
 4986 #endif
 4987 
 4988 void Assembler::prefetchnta(Address src) {
 4989   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4990   InstructionMark im(this);
 4991   prefix(src);
 4992   emit_int16(0x0F, 0x18);
 4993   emit_operand(rax, src); // 0, src
 4994 }
 4995 
 4996 void Assembler::prefetchr(Address src) {
 4997   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4998   InstructionMark im(this);
 4999   prefix(src);
 5000   emit_int16(0x0F, 0x0D);
 5001   emit_operand(rax, src); // 0, src
 5002 }
 5003 
 5004 void Assembler::prefetcht0(Address src) {
 5005   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5006   InstructionMark im(this);
 5007   prefix(src);
 5008   emit_int16(0x0F, 0x18);
 5009   emit_operand(rcx, src); // 1, src
 5010 }
 5011 
 5012 void Assembler::prefetcht1(Address src) {
 5013   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5014   InstructionMark im(this);
 5015   prefix(src);
 5016   emit_int16(0x0F, 0x18);
 5017   emit_operand(rdx, src); // 2, src
 5018 }
 5019 
 5020 void Assembler::prefetcht2(Address src) {
 5021   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5022   InstructionMark im(this);
 5023   prefix(src);
 5024   emit_int16(0x0F, 0x18);
 5025   emit_operand(rbx, src); // 3, src
 5026 }
 5027 
 5028 void Assembler::prefetchw(Address src) {
 5029   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5030   InstructionMark im(this);
 5031   prefix(src);
 5032   emit_int16(0x0F, 0x0D);
 5033   emit_operand(rcx, src); // 1, src
 5034 }
 5035 
 5036 void Assembler::prefix(Prefix p) {
 5037   emit_int8(p);
 5038 }
 5039 
 5040 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5041   assert(VM_Version::supports_ssse3(), "");
 5042   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5043   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5044   emit_int16(0x00, (0xC0 | encode));
 5045 }
 5046 
 5047 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5048   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5049          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5050          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5051   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5052   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5053   emit_int16(0x00, (0xC0 | encode));
 5054 }
 5055 
 5056 void Assembler::pshufb(XMMRegister dst, Address src) {
 5057   assert(VM_Version::supports_ssse3(), "");
 5058   InstructionMark im(this);
 5059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5060   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5061   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5062   emit_int8(0x00);
 5063   emit_operand(dst, src);
 5064 }
 5065 
 5066 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5067   assert(isByte(mode), "invalid value");
 5068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5069   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5070   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5071   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5072   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5073 }
 5074 
 5075 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5076   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5077          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5078          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5080   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5081   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5082   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5083 }
 5084 
 5085 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5086   assert(isByte(mode), "invalid value");
 5087   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5088   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5089   InstructionMark im(this);
 5090   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5091   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5092   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5093   emit_int8(0x70);
 5094   emit_operand(dst, src);
 5095   emit_int8(mode & 0xFF);
 5096 }
 5097 
 5098 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5099   assert(isByte(mode), "invalid value");
 5100   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5101   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5102   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5103   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5104 }
 5105 
 5106 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5107   assert(isByte(mode), "invalid value");
 5108   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5110   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5111   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5112 }
 5113 
 5114 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5115   assert(isByte(mode), "invalid value");
 5116   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5117   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5118   InstructionMark im(this);
 5119   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5120   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5121   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5122   emit_int8(0x70);
 5123   emit_operand(dst, src);
 5124   emit_int8(mode & 0xFF);
 5125 }
 5126 
 5127 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5128   assert(VM_Version::supports_evex(), "requires EVEX support");
 5129   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5130   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5131   attributes.set_is_evex_instruction();
 5132   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5133   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5134 }
 5135 
 5136 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5137   assert(isByte(imm8), "invalid value");
 5138   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5139   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5140   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5141   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5142 }
 5143 
 5144 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5145   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5146   attributes.set_rex_vex_w_reverted();
 5147   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5148   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5149 }
 5150 
 5151 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 5152   assert(isByte(imm8), "invalid value");
 5153   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5154   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5155   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5156   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5157 }
 5158 
 5159 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5160   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5161   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5162   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5163 }
 5164 
 5165 void Assembler::psrldq(XMMRegister dst, int shift) {
 5166   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5167   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5168   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5169   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5170   emit_int24(0x73, (0xC0 | encode), shift);
 5171 }
 5172 
 5173 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5174   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5175          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5176          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5177   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5178   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5179   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5180 }
 5181 
 5182 void Assembler::pslldq(XMMRegister dst, int shift) {
 5183   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5184   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5185   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5186   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5187   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5188   emit_int24(0x73, (0xC0 | encode), shift);
 5189 }
 5190 
 5191 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5192   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5193          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5194          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5195   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5196   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5197   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5198 }
 5199 
 5200 void Assembler::ptest(XMMRegister dst, Address src) {
 5201   assert(VM_Version::supports_sse4_1(), "");
 5202   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5203   InstructionMark im(this);
 5204   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5205   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5206   emit_int8(0x17);
 5207   emit_operand(dst, src);
 5208 }
 5209 
 5210 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5211   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5212   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5213   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5214   emit_int8(0x17);
 5215   emit_int8((0xC0 | encode));
 5216 }
 5217 
 5218 void Assembler::vptest(XMMRegister dst, Address src) {
 5219   assert(VM_Version::supports_avx(), "");
 5220   InstructionMark im(this);
 5221   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5222   assert(dst != xnoreg, "sanity");
 5223   // swap src<->dst for encoding
 5224   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5225   emit_int8(0x17);
 5226   emit_operand(dst, src);
 5227 }
 5228 
 5229 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5230   assert(VM_Version::supports_avx(), "");
 5231   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5233   emit_int16(0x17, (0xC0 | encode));
 5234 }
 5235 
 5236 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5237   assert(VM_Version::supports_avx(), "");
 5238   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5239   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5240   emit_int16(0x17, (0xC0 | encode));
 5241 }
 5242 
 5243 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5244   assert(VM_Version::supports_avx512vlbw(), "");
 5245   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5246   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5247   attributes.set_is_evex_instruction();
 5248   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5249   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5250 }
 5251 
 5252 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5253   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5254   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5255   InstructionMark im(this);
 5256   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5257   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5258   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5259   emit_int8(0x60);
 5260   emit_operand(dst, src);
 5261 }
 5262 
 5263 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5264   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5265   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5266   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5267   emit_int16(0x60, (0xC0 | encode));
 5268 }
 5269 
 5270 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5272   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5273   InstructionMark im(this);
 5274   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5275   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5276   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5277   emit_int8(0x62);
 5278   emit_operand(dst, src);
 5279 }
 5280 
 5281 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5282   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5284   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5285   emit_int16(0x62, (0xC0 | encode));
 5286 }
 5287 
 5288 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5289   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5290   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5291   attributes.set_rex_vex_w_reverted();
 5292   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5293   emit_int16(0x6C, (0xC0 | encode));
 5294 }
 5295 
 5296 void Assembler::push(int32_t imm32) {
 5297   // in 64bits we push 64bits onto the stack but only
 5298   // take a 32bit immediate
 5299   emit_int8(0x68);
 5300   emit_int32(imm32);
 5301 }
 5302 
 5303 void Assembler::push(Register src) {
 5304   int encode = prefix_and_encode(src->encoding());
 5305   emit_int8(0x50 | encode);
 5306 }
 5307 
 5308 void Assembler::pushf() {
 5309   emit_int8((unsigned char)0x9C);
 5310 }
 5311 
 5312 #ifndef _LP64 // no 32bit push/pop on amd64
 5313 void Assembler::pushl(Address src) {
 5314   // Note this will push 64bit on 64bit
 5315   InstructionMark im(this);
 5316   prefix(src);
 5317   emit_int8((unsigned char)0xFF);
 5318   emit_operand(rsi, src);
 5319 }
 5320 #endif
 5321 
 5322 void Assembler::rcll(Register dst, int imm8) {
 5323   assert(isShiftCount(imm8), "illegal shift count");
 5324   int encode = prefix_and_encode(dst->encoding());
 5325   if (imm8 == 1) {
 5326     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5327   } else {
 5328     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5329   }
 5330 }
 5331 
 5332 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5333   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5334   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5335   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5336   emit_int16(0x53, (0xC0 | encode));
 5337 }
 5338 
 5339 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5340   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5343   emit_int16(0x53, (0xC0 | encode));
 5344 }
 5345 
 5346 void Assembler::rdtsc() {
 5347   emit_int16(0x0F, 0x31);
 5348 }
 5349 
 5350 void Assembler::rdtscp() {
 5351   emit_int24(0x0F, 0x01, (unsigned char)0xF9);
 5352 }
 5353 
 5354 // copies data from [esi] to [edi] using rcx pointer sized words
 5355 // generic
 5356 void Assembler::rep_mov() {
 5357   // REP
 5358   // MOVSQ
 5359   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5360   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5361 }
 5362 
 5363 // sets rcx bytes with rax, value at [edi]
 5364 void Assembler::rep_stosb() {
 5365   // REP
 5366   // STOSB
 5367   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5368   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5369 }
 5370 
 5371 // sets rcx pointer sized words with rax, value at [edi]
 5372 // generic
 5373 void Assembler::rep_stos() {
 5374   // REP
 5375   // LP64:STOSQ, LP32:STOSD
 5376   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5377   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5378 }
 5379 
 5380 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5381 // generic
 5382 void Assembler::repne_scan() { // repne_scan
 5383   // SCASQ
 5384   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5385   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5386 }
 5387 
 5388 #ifdef _LP64
 5389 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5390 // generic
 5391 void Assembler::repne_scanl() { // repne_scan
 5392   // SCASL
 5393   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5394 }
 5395 #endif
 5396 
 5397 void Assembler::ret(int imm16) {
 5398   if (imm16 == 0) {
 5399     emit_int8((unsigned char)0xC3);
 5400   } else {
 5401     emit_int8((unsigned char)0xC2);
 5402     emit_int16(imm16);
 5403   }
 5404 }
 5405 
 5406 void Assembler::roll(Register dst, int imm8) {
 5407   assert(isShiftCount(imm8), "illegal shift count");
 5408   int encode = prefix_and_encode(dst->encoding());
 5409   if (imm8 == 1) {
 5410     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5411   } else {
 5412     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5413   }
 5414 }
 5415 
 5416 void Assembler::roll(Register dst) {
 5417   int encode = prefix_and_encode(dst->encoding());
 5418   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5419 }
 5420 
 5421 void Assembler::rorl(Register dst, int imm8) {
 5422   assert(isShiftCount(imm8), "illegal shift count");
 5423   int encode = prefix_and_encode(dst->encoding());
 5424   if (imm8 == 1) {
 5425     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5426   } else {
 5427     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5428   }
 5429 }
 5430 
 5431 void Assembler::rorl(Register dst) {
 5432   int encode = prefix_and_encode(dst->encoding());
 5433   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5434 }
 5435 
 5436 #ifdef _LP64
 5437 void Assembler::rorq(Register dst) {
 5438   int encode = prefixq_and_encode(dst->encoding());
 5439   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5440 }
 5441 
 5442 void Assembler::rorq(Register dst, int imm8) {
 5443   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5444   int encode = prefixq_and_encode(dst->encoding());
 5445   if (imm8 == 1) {
 5446     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5447   } else {
 5448     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5449   }
 5450 }
 5451 
 5452 void Assembler::rolq(Register dst) {
 5453   int encode = prefixq_and_encode(dst->encoding());
 5454   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5455 }
 5456 
 5457 void Assembler::rolq(Register dst, int imm8) {
 5458   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5459   int encode = prefixq_and_encode(dst->encoding());
 5460   if (imm8 == 1) {
 5461     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5462   } else {
 5463     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5464   }
 5465 }
 5466 #endif
 5467 
 5468 void Assembler::sahf() {
 5469 #ifdef _LP64
 5470   // Not supported in 64bit mode
 5471   ShouldNotReachHere();
 5472 #endif
 5473   emit_int8((unsigned char)0x9E);
 5474 }
 5475 
 5476 void Assembler::sall(Address dst, int imm8) {
 5477   InstructionMark im(this);
 5478   assert(isShiftCount(imm8), "illegal shift count");
 5479   prefix(dst);
 5480   if (imm8 == 1) {
 5481     emit_int8((unsigned char)0xD1);
 5482     emit_operand(as_Register(4), dst);
 5483   }
 5484   else {
 5485     emit_int8((unsigned char)0xC1);
 5486     emit_operand(as_Register(4), dst);
 5487     emit_int8(imm8);
 5488   }
 5489 }
 5490 
 5491 void Assembler::sall(Address dst) {
 5492   InstructionMark im(this);
 5493   prefix(dst);
 5494   emit_int8((unsigned char)0xD3);
 5495   emit_operand(as_Register(4), dst);
 5496 }
 5497 
 5498 void Assembler::sall(Register dst, int imm8) {
 5499   assert(isShiftCount(imm8), "illegal shift count");
 5500   int encode = prefix_and_encode(dst->encoding());
 5501   if (imm8 == 1) {
 5502     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5503   } else {
 5504     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5505   }
 5506 }
 5507 
 5508 void Assembler::sall(Register dst) {
 5509   int encode = prefix_and_encode(dst->encoding());
 5510   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5511 }
 5512 
 5513 void Assembler::sarl(Address dst, int imm8) {
 5514   assert(isShiftCount(imm8), "illegal shift count");
 5515   InstructionMark im(this);
 5516   prefix(dst);
 5517   if (imm8 == 1) {
 5518     emit_int8((unsigned char)0xD1);
 5519     emit_operand(as_Register(7), dst);
 5520   }
 5521   else {
 5522     emit_int8((unsigned char)0xC1);
 5523     emit_operand(as_Register(7), dst);
 5524     emit_int8(imm8);
 5525   }
 5526 }
 5527 
 5528 void Assembler::sarl(Address dst) {
 5529   InstructionMark im(this);
 5530   prefix(dst);
 5531   emit_int8((unsigned char)0xD3);
 5532   emit_operand(as_Register(7), dst);
 5533 }
 5534 
 5535 void Assembler::sarl(Register dst, int imm8) {
 5536   int encode = prefix_and_encode(dst->encoding());
 5537   assert(isShiftCount(imm8), "illegal shift count");
 5538   if (imm8 == 1) {
 5539     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5540   } else {
 5541     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5542   }
 5543 }
 5544 
 5545 void Assembler::sarl(Register dst) {
 5546   int encode = prefix_and_encode(dst->encoding());
 5547   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5548 }
 5549 
 5550 void Assembler::sbbl(Address dst, int32_t imm32) {
 5551   InstructionMark im(this);
 5552   prefix(dst);
 5553   emit_arith_operand(0x81, rbx, dst, imm32);
 5554 }
 5555 
 5556 void Assembler::sbbl(Register dst, int32_t imm32) {
 5557   prefix(dst);
 5558   emit_arith(0x81, 0xD8, dst, imm32);
 5559 }
 5560 
 5561 
 5562 void Assembler::sbbl(Register dst, Address src) {
 5563   InstructionMark im(this);
 5564   prefix(src, dst);
 5565   emit_int8(0x1B);
 5566   emit_operand(dst, src);
 5567 }
 5568 
 5569 void Assembler::sbbl(Register dst, Register src) {
 5570   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5571   emit_arith(0x1B, 0xC0, dst, src);
 5572 }
 5573 
 5574 void Assembler::setb(Condition cc, Register dst) {
 5575   assert(0 <= cc && cc < 16, "illegal cc");
 5576   int encode = prefix_and_encode(dst->encoding(), true);
 5577   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5578 }
 5579 
 5580 void Assembler::sete(Register dst) {
 5581   int encode = prefix_and_encode(dst->encoding(), true);
 5582   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5583 }
 5584 
 5585 void Assembler::setl(Register dst) {
 5586   int encode = prefix_and_encode(dst->encoding(), true);
 5587   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5588 }
 5589 
 5590 void Assembler::setne(Register dst) {
 5591   int encode = prefix_and_encode(dst->encoding(), true);
 5592   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5593 }
 5594 
 5595 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5596   assert(VM_Version::supports_ssse3(), "");
 5597   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5598   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5599   emit_int24(0x0F, (0xC0 | encode), imm8);
 5600 }
 5601 
 5602 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5603   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5604          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5605          0, "");
 5606   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5607   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5608   emit_int24(0x0F, (0xC0 | encode), imm8);
 5609 }
 5610 
 5611 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5612   assert(VM_Version::supports_evex(), "");
 5613   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5614   attributes.set_is_evex_instruction();
 5615   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5616   emit_int24(0x3, (0xC0 | encode), imm8);
 5617 }
 5618 
 5619 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5620   assert(VM_Version::supports_sse4_1(), "");
 5621   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5622   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5623   emit_int24(0x0E, (0xC0 | encode), imm8);
 5624 }
 5625 
 5626 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5627   assert(VM_Version::supports_sha(), "");
 5628   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5629   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5630 }
 5631 
 5632 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5633   assert(VM_Version::supports_sha(), "");
 5634   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5635   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5636 }
 5637 
 5638 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5639   assert(VM_Version::supports_sha(), "");
 5640   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5641   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5642 }
 5643 
 5644 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5645   assert(VM_Version::supports_sha(), "");
 5646   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5647   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5648 }
 5649 
 5650 // xmm0 is implicit additional source to this instruction.
 5651 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5652   assert(VM_Version::supports_sha(), "");
 5653   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5654   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5655 }
 5656 
 5657 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5658   assert(VM_Version::supports_sha(), "");
 5659   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5660   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5661 }
 5662 
 5663 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5664   assert(VM_Version::supports_sha(), "");
 5665   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5666   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5667 }
 5668 
 5669 
 5670 void Assembler::shll(Register dst, int imm8) {
 5671   assert(isShiftCount(imm8), "illegal shift count");
 5672   int encode = prefix_and_encode(dst->encoding());
 5673   if (imm8 == 1 ) {
 5674     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5675   } else {
 5676     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5677   }
 5678 }
 5679 
 5680 void Assembler::shll(Register dst) {
 5681   int encode = prefix_and_encode(dst->encoding());
 5682   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5683 }
 5684 
 5685 void Assembler::shrl(Register dst, int imm8) {
 5686   assert(isShiftCount(imm8), "illegal shift count");
 5687   int encode = prefix_and_encode(dst->encoding());
 5688   if (imm8 == 1) {
 5689     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5690   }
 5691   else {
 5692     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5693   }
 5694 }
 5695 
 5696 void Assembler::shrl(Register dst) {
 5697   int encode = prefix_and_encode(dst->encoding());
 5698   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5699 }
 5700 
 5701 void Assembler::shrl(Address dst) {
 5702   InstructionMark im(this);
 5703   prefix(dst);
 5704   emit_int8((unsigned char)0xD3);
 5705   emit_operand(as_Register(5), dst);
 5706 }
 5707 
 5708 void Assembler::shrl(Address dst, int imm8) {
 5709   InstructionMark im(this);
 5710   assert(isShiftCount(imm8), "illegal shift count");
 5711   prefix(dst);
 5712   if (imm8 == 1) {
 5713     emit_int8((unsigned char)0xD1);
 5714     emit_operand(as_Register(5), dst);
 5715   }
 5716   else {
 5717     emit_int8((unsigned char)0xC1);
 5718     emit_operand(as_Register(5), dst);
 5719     emit_int8(imm8);
 5720   }
 5721 }
 5722 
 5723 
 5724 void Assembler::shldl(Register dst, Register src) {
 5725   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5726   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5727 }
 5728 
 5729 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5730   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5731   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5732 }
 5733 
 5734 void Assembler::shrdl(Register dst, Register src) {
 5735   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5736   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5737 }
 5738 
 5739 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5740   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5741   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5742 }
 5743 
 5744 // copies a single word from [esi] to [edi]
 5745 void Assembler::smovl() {
 5746   emit_int8((unsigned char)0xA5);
 5747 }
 5748 
 5749 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5750   assert(VM_Version::supports_sse4_1(), "");
 5751   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5752   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5753   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5754 }
 5755 
 5756 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5757   assert(VM_Version::supports_sse4_1(), "");
 5758   InstructionMark im(this);
 5759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5760   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5761   emit_int8(0x0B);
 5762   emit_operand(dst, src);
 5763   emit_int8((unsigned char)rmode);
 5764 }
 5765 
 5766 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5767   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5768   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5769   attributes.set_rex_vex_w_reverted();
 5770   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5771   emit_int16(0x51, (0xC0 | encode));
 5772 }
 5773 
 5774 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5776   InstructionMark im(this);
 5777   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5778   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5779   attributes.set_rex_vex_w_reverted();
 5780   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5781   emit_int8(0x51);
 5782   emit_operand(dst, src);
 5783 }
 5784 
 5785 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5786   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5787   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5788   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5789   emit_int16(0x51, (0xC0 | encode));
 5790 }
 5791 
 5792 void Assembler::std() {
 5793   emit_int8((unsigned char)0xFD);
 5794 }
 5795 
 5796 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5797   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5798   InstructionMark im(this);
 5799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5800   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5801   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5802   emit_int8(0x51);
 5803   emit_operand(dst, src);
 5804 }
 5805 
 5806 void Assembler::stmxcsr( Address dst) {
 5807   if (UseAVX > 0 ) {
 5808     assert(VM_Version::supports_avx(), "");
 5809     InstructionMark im(this);
 5810     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5811     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5812     emit_int8((unsigned char)0xAE);
 5813     emit_operand(as_Register(3), dst);
 5814   } else {
 5815     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5816     InstructionMark im(this);
 5817     prefix(dst);
 5818     emit_int16(0x0F, (unsigned char)0xAE);
 5819     emit_operand(as_Register(3), dst);
 5820   }
 5821 }
 5822 
 5823 void Assembler::subl(Address dst, int32_t imm32) {
 5824   InstructionMark im(this);
 5825   prefix(dst);
 5826   emit_arith_operand(0x81, rbp, dst, imm32);
 5827 }
 5828 
 5829 void Assembler::subl(Address dst, Register src) {
 5830   InstructionMark im(this);
 5831   prefix(dst, src);
 5832   emit_int8(0x29);
 5833   emit_operand(src, dst);
 5834 }
 5835 
 5836 void Assembler::subl(Register dst, int32_t imm32) {
 5837   prefix(dst);
 5838   emit_arith(0x81, 0xE8, dst, imm32);
 5839 }
 5840 
 5841 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5842 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5843   prefix(dst);
 5844   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5845 }
 5846 
 5847 void Assembler::subl(Register dst, Address src) {
 5848   InstructionMark im(this);
 5849   prefix(src, dst);
 5850   emit_int8(0x2B);
 5851   emit_operand(dst, src);
 5852 }
 5853 
 5854 void Assembler::subl(Register dst, Register src) {
 5855   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5856   emit_arith(0x2B, 0xC0, dst, src);
 5857 }
 5858 
 5859 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5861   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5862   attributes.set_rex_vex_w_reverted();
 5863   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5864   emit_int16(0x5C, (0xC0 | encode));
 5865 }
 5866 
 5867 void Assembler::subsd(XMMRegister dst, Address src) {
 5868   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5869   InstructionMark im(this);
 5870   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5871   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5872   attributes.set_rex_vex_w_reverted();
 5873   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5874   emit_int8(0x5C);
 5875   emit_operand(dst, src);
 5876 }
 5877 
 5878 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5879   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5881   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5882   emit_int16(0x5C, (0xC0 | encode));
 5883 }
 5884 
 5885 void Assembler::subss(XMMRegister dst, Address src) {
 5886   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5887   InstructionMark im(this);
 5888   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5889   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5890   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5891   emit_int8(0x5C);
 5892   emit_operand(dst, src);
 5893 }
 5894 
 5895 void Assembler::testb(Register dst, int imm8) {
 5896   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5897   if (dst == rax) {
 5898     emit_int8((unsigned char)0xA8);
 5899     emit_int8(imm8);
 5900   } else {
 5901     (void) prefix_and_encode(dst->encoding(), true);
 5902     emit_arith_b(0xF6, 0xC0, dst, imm8);
 5903   }
 5904 }
 5905 
 5906 void Assembler::testb(Address dst, int imm8) {
 5907   InstructionMark im(this);
 5908   prefix(dst);
 5909   emit_int8((unsigned char)0xF6);
 5910   emit_operand(rax, dst, 1);
 5911   emit_int8(imm8);
 5912 }
 5913 
 5914 void Assembler::testl(Address dst, int32_t imm32) {
 5915   if (imm32 >= 0 && is8bit(imm32)) {
 5916     testb(dst, imm32);
 5917     return;
 5918   }
 5919   InstructionMark im(this);
 5920   emit_int8((unsigned char)0xF7);
 5921   emit_operand(as_Register(0), dst);
 5922   emit_int32(imm32);
 5923 }
 5924 
 5925 void Assembler::testl(Register dst, int32_t imm32) {
 5926   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 5927     testb(dst, imm32);
 5928     return;
 5929   }
 5930   // not using emit_arith because test
 5931   // doesn't support sign-extension of
 5932   // 8bit operands
 5933   if (dst == rax) {
 5934     emit_int8((unsigned char)0xA9);
 5935     emit_int32(imm32);
 5936   } else {
 5937     int encode = dst->encoding();
 5938     encode = prefix_and_encode(encode);
 5939     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5940     emit_int32(imm32);
 5941   }
 5942 }
 5943 
 5944 void Assembler::testl(Register dst, Register src) {
 5945   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5946   emit_arith(0x85, 0xC0, dst, src);
 5947 }
 5948 
 5949 void Assembler::testl(Register dst, Address src) {
 5950   InstructionMark im(this);
 5951   prefix(src, dst);
 5952   emit_int8((unsigned char)0x85);
 5953   emit_operand(dst, src);
 5954 }
 5955 
 5956 void Assembler::tzcntl(Register dst, Register src) {
 5957   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5958   emit_int8((unsigned char)0xF3);
 5959   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5960   emit_int24(0x0F,
 5961              (unsigned char)0xBC,
 5962              0xC0 | encode);
 5963 }
 5964 
 5965 void Assembler::tzcntl(Register dst, Address src) {
 5966   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5967   InstructionMark im(this);
 5968   emit_int8((unsigned char)0xF3);
 5969   prefix(src, dst);
 5970   emit_int16(0x0F, (unsigned char)0xBC);
 5971   emit_operand(dst, src);
 5972 }
 5973 
 5974 void Assembler::tzcntq(Register dst, Register src) {
 5975   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5976   emit_int8((unsigned char)0xF3);
 5977   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5978   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5979 }
 5980 
 5981 void Assembler::tzcntq(Register dst, Address src) {
 5982   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5983   InstructionMark im(this);
 5984   emit_int8((unsigned char)0xF3);
 5985   prefixq(src, dst);
 5986   emit_int16(0x0F, (unsigned char)0xBC);
 5987   emit_operand(dst, src);
 5988 }
 5989 
 5990 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5991   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5992   InstructionMark im(this);
 5993   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5994   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5995   attributes.set_rex_vex_w_reverted();
 5996   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5997   emit_int8(0x2E);
 5998   emit_operand(dst, src);
 5999 }
 6000 
 6001 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6002   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6003   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6004   attributes.set_rex_vex_w_reverted();
 6005   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6006   emit_int16(0x2E, (0xC0 | encode));
 6007 }
 6008 
 6009 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6010   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6011   InstructionMark im(this);
 6012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6013   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6014   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6015   emit_int8(0x2E);
 6016   emit_operand(dst, src);
 6017 }
 6018 
 6019 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6020   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6021   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6022   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6023   emit_int16(0x2E, (0xC0 | encode));
 6024 }
 6025 
 6026 void Assembler::xabort(int8_t imm8) {
 6027   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6028 }
 6029 
 6030 void Assembler::xaddb(Address dst, Register src) {
 6031   InstructionMark im(this);
 6032   prefix(dst, src, true);
 6033   emit_int16(0x0F, (unsigned char)0xC0);
 6034   emit_operand(src, dst);
 6035 }
 6036 
 6037 void Assembler::xaddw(Address dst, Register src) {
 6038   InstructionMark im(this);
 6039   emit_int8(0x66);
 6040   prefix(dst, src);
 6041   emit_int16(0x0F, (unsigned char)0xC1);
 6042   emit_operand(src, dst);
 6043 }
 6044 
 6045 void Assembler::xaddl(Address dst, Register src) {
 6046   InstructionMark im(this);
 6047   prefix(dst, src);
 6048   emit_int16(0x0F, (unsigned char)0xC1);
 6049   emit_operand(src, dst);
 6050 }
 6051 
 6052 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6053   InstructionMark im(this);
 6054   relocate(rtype);
 6055   if (abort.is_bound()) {
 6056     address entry = target(abort);
 6057     assert(entry != NULL, "abort entry NULL");
 6058     intptr_t offset = entry - pc();
 6059     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6060     emit_int32(offset - 6); // 2 opcode + 4 address
 6061   } else {
 6062     abort.add_patch_at(code(), locator());
 6063     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6064     emit_int32(0);
 6065   }
 6066 }
 6067 
 6068 void Assembler::xchgb(Register dst, Address src) { // xchg
 6069   InstructionMark im(this);
 6070   prefix(src, dst, true);
 6071   emit_int8((unsigned char)0x86);
 6072   emit_operand(dst, src);
 6073 }
 6074 
 6075 void Assembler::xchgw(Register dst, Address src) { // xchg
 6076   InstructionMark im(this);
 6077   emit_int8(0x66);
 6078   prefix(src, dst);
 6079   emit_int8((unsigned char)0x87);
 6080   emit_operand(dst, src);
 6081 }
 6082 
 6083 void Assembler::xchgl(Register dst, Address src) { // xchg
 6084   InstructionMark im(this);
 6085   prefix(src, dst);
 6086   emit_int8((unsigned char)0x87);
 6087   emit_operand(dst, src);
 6088 }
 6089 
 6090 void Assembler::xchgl(Register dst, Register src) {
 6091   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6092   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6093 }
 6094 
 6095 void Assembler::xend() {
 6096   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6097 }
 6098 
 6099 void Assembler::xgetbv() {
 6100   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6101 }
 6102 
 6103 void Assembler::xorl(Address dst, int32_t imm32) {
 6104   InstructionMark im(this);
 6105   prefix(dst);
 6106   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6107 }
 6108 
 6109 void Assembler::xorl(Register dst, int32_t imm32) {
 6110   prefix(dst);
 6111   emit_arith(0x81, 0xF0, dst, imm32);
 6112 }
 6113 
 6114 void Assembler::xorl(Register dst, Address src) {
 6115   InstructionMark im(this);
 6116   prefix(src, dst);
 6117   emit_int8(0x33);
 6118   emit_operand(dst, src);
 6119 }
 6120 
 6121 void Assembler::xorl(Register dst, Register src) {
 6122   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6123   emit_arith(0x33, 0xC0, dst, src);
 6124 }
 6125 
 6126 void Assembler::xorl(Address dst, Register src) {
 6127   InstructionMark im(this);
 6128   prefix(dst, src);
 6129   emit_int8(0x31);
 6130   emit_operand(src, dst);
 6131 }
 6132 
 6133 void Assembler::xorb(Register dst, Address src) {
 6134   InstructionMark im(this);
 6135   prefix(src, dst);
 6136   emit_int8(0x32);
 6137   emit_operand(dst, src);
 6138 }
 6139 
 6140 void Assembler::xorb(Address dst, Register src) {
 6141   InstructionMark im(this);
 6142   prefix(dst, src, true);
 6143   emit_int8(0x30);
 6144   emit_operand(src, dst);
 6145 }
 6146 
 6147 void Assembler::xorw(Register dst, Register src) {
 6148   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6149   emit_arith(0x33, 0xC0, dst, src);
 6150 }
 6151 
 6152 // AVX 3-operands scalar float-point arithmetic instructions
 6153 
 6154 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6155   assert(VM_Version::supports_avx(), "");
 6156   InstructionMark im(this);
 6157   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6158   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6159   attributes.set_rex_vex_w_reverted();
 6160   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6161   emit_int8(0x58);
 6162   emit_operand(dst, src);
 6163 }
 6164 
 6165 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6166   assert(VM_Version::supports_avx(), "");
 6167   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6168   attributes.set_rex_vex_w_reverted();
 6169   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6170   emit_int16(0x58, (0xC0 | encode));
 6171 }
 6172 
 6173 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6174   assert(VM_Version::supports_avx(), "");
 6175   InstructionMark im(this);
 6176   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6177   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6178   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6179   emit_int8(0x58);
 6180   emit_operand(dst, src);
 6181 }
 6182 
 6183 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6184   assert(VM_Version::supports_avx(), "");
 6185   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6186   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6187   emit_int16(0x58, (0xC0 | encode));
 6188 }
 6189 
 6190 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6191   assert(VM_Version::supports_avx(), "");
 6192   InstructionMark im(this);
 6193   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6194   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6195   attributes.set_rex_vex_w_reverted();
 6196   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6197   emit_int8(0x5E);
 6198   emit_operand(dst, src);
 6199 }
 6200 
 6201 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6202   assert(VM_Version::supports_avx(), "");
 6203   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6204   attributes.set_rex_vex_w_reverted();
 6205   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6206   emit_int16(0x5E, (0xC0 | encode));
 6207 }
 6208 
 6209 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6210   assert(VM_Version::supports_avx(), "");
 6211   InstructionMark im(this);
 6212   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6213   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6214   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6215   emit_int8(0x5E);
 6216   emit_operand(dst, src);
 6217 }
 6218 
 6219 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6220   assert(VM_Version::supports_avx(), "");
 6221   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6222   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6223   emit_int16(0x5E, (0xC0 | encode));
 6224 }
 6225 
 6226 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6227   assert(VM_Version::supports_fma(), "");
 6228   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6229   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6230   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6231 }
 6232 
 6233 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6234   assert(VM_Version::supports_fma(), "");
 6235   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6236   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6237   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6238 }
 6239 
 6240 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6241   assert(VM_Version::supports_avx(), "");
 6242   InstructionMark im(this);
 6243   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6244   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6245   attributes.set_rex_vex_w_reverted();
 6246   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6247   emit_int8(0x59);
 6248   emit_operand(dst, src);
 6249 }
 6250 
 6251 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6252   assert(VM_Version::supports_avx(), "");
 6253   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6254   attributes.set_rex_vex_w_reverted();
 6255   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6256   emit_int16(0x59, (0xC0 | encode));
 6257 }
 6258 
 6259 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6260   assert(VM_Version::supports_avx(), "");
 6261   InstructionMark im(this);
 6262   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6263   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6264   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6265   emit_int8(0x59);
 6266   emit_operand(dst, src);
 6267 }
 6268 
 6269 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6270   assert(VM_Version::supports_avx(), "");
 6271   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6272   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6273   emit_int16(0x59, (0xC0 | encode));
 6274 }
 6275 
 6276 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6277   assert(VM_Version::supports_avx(), "");
 6278   InstructionMark im(this);
 6279   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6280   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6281   attributes.set_rex_vex_w_reverted();
 6282   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6283   emit_int8(0x5C);
 6284   emit_operand(dst, src);
 6285 }
 6286 
 6287 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6288   assert(VM_Version::supports_avx(), "");
 6289   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6290   attributes.set_rex_vex_w_reverted();
 6291   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6292   emit_int16(0x5C, (0xC0 | encode));
 6293 }
 6294 
 6295 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6296   assert(VM_Version::supports_avx(), "");
 6297   InstructionMark im(this);
 6298   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6299   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6300   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6301   emit_int8(0x5C);
 6302   emit_operand(dst, src);
 6303 }
 6304 
 6305 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6306   assert(VM_Version::supports_avx(), "");
 6307   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6308   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6309   emit_int16(0x5C, (0xC0 | encode));
 6310 }
 6311 
 6312 //====================VECTOR ARITHMETIC=====================================
 6313 
 6314 // Float-point vector arithmetic
 6315 
 6316 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6317   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6318   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6319   attributes.set_rex_vex_w_reverted();
 6320   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6321   emit_int16(0x58, (0xC0 | encode));
 6322 }
 6323 
 6324 void Assembler::addpd(XMMRegister dst, Address src) {
 6325   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6326   InstructionMark im(this);
 6327   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6328   attributes.set_rex_vex_w_reverted();
 6329   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6330   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6331   emit_int8(0x58);
 6332   emit_operand(dst, src);
 6333 }
 6334 
 6335 
 6336 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6337   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6338   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6340   emit_int16(0x58, (0xC0 | encode));
 6341 }
 6342 
 6343 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6344   assert(VM_Version::supports_avx(), "");
 6345   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6346   attributes.set_rex_vex_w_reverted();
 6347   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6348   emit_int16(0x58, (0xC0 | encode));
 6349 }
 6350 
 6351 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6352   assert(VM_Version::supports_avx(), "");
 6353   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6354   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6355   emit_int16(0x58, (0xC0 | encode));
 6356 }
 6357 
 6358 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6359   assert(VM_Version::supports_avx(), "");
 6360   InstructionMark im(this);
 6361   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6362   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6363   attributes.set_rex_vex_w_reverted();
 6364   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6365   emit_int8(0x58);
 6366   emit_operand(dst, src);
 6367 }
 6368 
 6369 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6370   assert(VM_Version::supports_avx(), "");
 6371   InstructionMark im(this);
 6372   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6373   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6374   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6375   emit_int8(0x58);
 6376   emit_operand(dst, src);
 6377 }
 6378 
 6379 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6380   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6381   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6382   attributes.set_rex_vex_w_reverted();
 6383   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6384   emit_int16(0x5C, (0xC0 | encode));
 6385 }
 6386 
 6387 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6388   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6389   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6390   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6391   emit_int16(0x5C, (0xC0 | encode));
 6392 }
 6393 
 6394 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6395   assert(VM_Version::supports_avx(), "");
 6396   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6397   attributes.set_rex_vex_w_reverted();
 6398   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6399   emit_int16(0x5C, (0xC0 | encode));
 6400 }
 6401 
 6402 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6403   assert(VM_Version::supports_avx(), "");
 6404   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6405   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6406   emit_int16(0x5C, (0xC0 | encode));
 6407 }
 6408 
 6409 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6410   assert(VM_Version::supports_avx(), "");
 6411   InstructionMark im(this);
 6412   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6413   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6414   attributes.set_rex_vex_w_reverted();
 6415   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6416   emit_int8(0x5C);
 6417   emit_operand(dst, src);
 6418 }
 6419 
 6420 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6421   assert(VM_Version::supports_avx(), "");
 6422   InstructionMark im(this);
 6423   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6424   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6425   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6426   emit_int8(0x5C);
 6427   emit_operand(dst, src);
 6428 }
 6429 
 6430 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6432   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6433   attributes.set_rex_vex_w_reverted();
 6434   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6435   emit_int16(0x59, (0xC0 | encode));
 6436 }
 6437 
 6438 void Assembler::mulpd(XMMRegister dst, Address src) {
 6439   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6440   InstructionMark im(this);
 6441   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6442   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6443   attributes.set_rex_vex_w_reverted();
 6444   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6445   emit_int8(0x59);
 6446   emit_operand(dst, src);
 6447 }
 6448 
 6449 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6450   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6451   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6452   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6453   emit_int16(0x59, (0xC0 | encode));
 6454 }
 6455 
 6456 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6457   assert(VM_Version::supports_avx(), "");
 6458   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6459   attributes.set_rex_vex_w_reverted();
 6460   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6461   emit_int16(0x59, (0xC0 | encode));
 6462 }
 6463 
 6464 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6465   assert(VM_Version::supports_avx(), "");
 6466   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6467   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6468   emit_int16(0x59, (0xC0 | encode));
 6469 }
 6470 
 6471 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6472   assert(VM_Version::supports_avx(), "");
 6473   InstructionMark im(this);
 6474   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6475   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6476   attributes.set_rex_vex_w_reverted();
 6477   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6478   emit_int8(0x59);
 6479   emit_operand(dst, src);
 6480 }
 6481 
 6482 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6483   assert(VM_Version::supports_avx(), "");
 6484   InstructionMark im(this);
 6485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6486   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6487   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6488   emit_int8(0x59);
 6489   emit_operand(dst, src);
 6490 }
 6491 
 6492 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6493   assert(VM_Version::supports_fma(), "");
 6494   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6495   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6496   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6497 }
 6498 
 6499 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6500   assert(VM_Version::supports_fma(), "");
 6501   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6502   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6503   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6504 }
 6505 
 6506 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6507   assert(VM_Version::supports_fma(), "");
 6508   InstructionMark im(this);
 6509   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6510   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6511   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6512   emit_int8((unsigned char)0xB8);
 6513   emit_operand(dst, src2);
 6514 }
 6515 
 6516 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6517   assert(VM_Version::supports_fma(), "");
 6518   InstructionMark im(this);
 6519   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6520   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6521   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6522   emit_int8((unsigned char)0xB8);
 6523   emit_operand(dst, src2);
 6524 }
 6525 
 6526 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6527   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6528   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6529   attributes.set_rex_vex_w_reverted();
 6530   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6531   emit_int16(0x5E, (0xC0 | encode));
 6532 }
 6533 
 6534 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6535   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6536   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6537   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6538   emit_int16(0x5E, (0xC0 | encode));
 6539 }
 6540 
 6541 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6542   assert(VM_Version::supports_avx(), "");
 6543   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6544   attributes.set_rex_vex_w_reverted();
 6545   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6546   emit_int16(0x5E, (0xC0 | encode));
 6547 }
 6548 
 6549 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6550   assert(VM_Version::supports_avx(), "");
 6551   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6552   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6553   emit_int16(0x5E, (0xC0 | encode));
 6554 }
 6555 
 6556 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6557   assert(VM_Version::supports_avx(), "");
 6558   InstructionMark im(this);
 6559   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6560   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6561   attributes.set_rex_vex_w_reverted();
 6562   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6563   emit_int8(0x5E);
 6564   emit_operand(dst, src);
 6565 }
 6566 
 6567 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6568   assert(VM_Version::supports_avx(), "");
 6569   InstructionMark im(this);
 6570   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6571   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6572   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6573   emit_int8(0x5E);
 6574   emit_operand(dst, src);
 6575 }
 6576 
 6577 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6578   assert(VM_Version::supports_avx(), "");
 6579   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6580   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6581   emit_int24(0x09, (0xC0 | encode), (rmode));
 6582 }
 6583 
 6584 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6585   assert(VM_Version::supports_avx(), "");
 6586   InstructionMark im(this);
 6587   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6588   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6589   emit_int8(0x09);
 6590   emit_operand(dst, src);
 6591   emit_int8((rmode));
 6592 }
 6593 
 6594 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6595   assert(VM_Version::supports_evex(), "requires EVEX support");
 6596   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6597   attributes.set_is_evex_instruction();
 6598   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6599   emit_int24(0x09, (0xC0 | encode), (rmode));
 6600 }
 6601 
 6602 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6603   assert(VM_Version::supports_evex(), "requires EVEX support");
 6604   assert(dst != xnoreg, "sanity");
 6605   InstructionMark im(this);
 6606   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6607   attributes.set_is_evex_instruction();
 6608   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6609   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6610   emit_int8(0x09);
 6611   emit_operand(dst, src);
 6612   emit_int8((rmode));
 6613 }
 6614 
 6615 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6616   assert(VM_Version::supports_avx(), "");
 6617   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6618   attributes.set_rex_vex_w_reverted();
 6619   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6620   emit_int16(0x51, (0xC0 | encode));
 6621 }
 6622 
 6623 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6624   assert(VM_Version::supports_avx(), "");
 6625   InstructionMark im(this);
 6626   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6627   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6628   attributes.set_rex_vex_w_reverted();
 6629   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6630   emit_int8(0x51);
 6631   emit_operand(dst, src);
 6632 }
 6633 
 6634 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6635   assert(VM_Version::supports_avx(), "");
 6636   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6637   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6638   emit_int16(0x51, (0xC0 | encode));
 6639 }
 6640 
 6641 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6642   assert(VM_Version::supports_avx(), "");
 6643   InstructionMark im(this);
 6644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6645   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6646   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6647   emit_int8(0x51);
 6648   emit_operand(dst, src);
 6649 }
 6650 
 6651 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6652   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6653   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6654   attributes.set_rex_vex_w_reverted();
 6655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6656   emit_int16(0x54, (0xC0 | encode));
 6657 }
 6658 
 6659 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6660   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6661   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6662   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6663   emit_int16(0x54, (0xC0 | encode));
 6664 }
 6665 
 6666 void Assembler::andps(XMMRegister dst, Address src) {
 6667   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6668   InstructionMark im(this);
 6669   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6670   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6671   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6672   emit_int8(0x54);
 6673   emit_operand(dst, src);
 6674 }
 6675 
 6676 void Assembler::andpd(XMMRegister dst, Address src) {
 6677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6678   InstructionMark im(this);
 6679   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6680   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6681   attributes.set_rex_vex_w_reverted();
 6682   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6683   emit_int8(0x54);
 6684   emit_operand(dst, src);
 6685 }
 6686 
 6687 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6688   assert(VM_Version::supports_avx(), "");
 6689   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6690   attributes.set_rex_vex_w_reverted();
 6691   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6692   emit_int16(0x54, (0xC0 | encode));
 6693 }
 6694 
 6695 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6696   assert(VM_Version::supports_avx(), "");
 6697   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6698   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6699   emit_int16(0x54, (0xC0 | encode));
 6700 }
 6701 
 6702 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6703   assert(VM_Version::supports_avx(), "");
 6704   InstructionMark im(this);
 6705   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6706   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6707   attributes.set_rex_vex_w_reverted();
 6708   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6709   emit_int8(0x54);
 6710   emit_operand(dst, src);
 6711 }
 6712 
 6713 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6714   assert(VM_Version::supports_avx(), "");
 6715   InstructionMark im(this);
 6716   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6717   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6718   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6719   emit_int8(0x54);
 6720   emit_operand(dst, src);
 6721 }
 6722 
 6723 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6724   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6725   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6726   attributes.set_rex_vex_w_reverted();
 6727   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6728   emit_int8(0x15);
 6729   emit_int8((0xC0 | encode));
 6730 }
 6731 
 6732 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6733   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6734   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6735   attributes.set_rex_vex_w_reverted();
 6736   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6737   emit_int16(0x14, (0xC0 | encode));
 6738 }
 6739 
 6740 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6742   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6743   attributes.set_rex_vex_w_reverted();
 6744   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6745   emit_int16(0x57, (0xC0 | encode));
 6746 }
 6747 
 6748 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6749   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6751   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6752   emit_int16(0x57, (0xC0 | encode));
 6753 }
 6754 
 6755 void Assembler::xorpd(XMMRegister dst, Address src) {
 6756   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6757   InstructionMark im(this);
 6758   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6759   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6760   attributes.set_rex_vex_w_reverted();
 6761   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6762   emit_int8(0x57);
 6763   emit_operand(dst, src);
 6764 }
 6765 
 6766 void Assembler::xorps(XMMRegister dst, Address src) {
 6767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6768   InstructionMark im(this);
 6769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6770   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6771   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6772   emit_int8(0x57);
 6773   emit_operand(dst, src);
 6774 }
 6775 
 6776 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6777   assert(VM_Version::supports_avx(), "");
 6778   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6779   attributes.set_rex_vex_w_reverted();
 6780   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6781   emit_int16(0x57, (0xC0 | encode));
 6782 }
 6783 
 6784 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6785   assert(VM_Version::supports_avx(), "");
 6786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6787   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6788   emit_int16(0x57, (0xC0 | encode));
 6789 }
 6790 
 6791 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6792   assert(VM_Version::supports_avx(), "");
 6793   InstructionMark im(this);
 6794   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6795   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6796   attributes.set_rex_vex_w_reverted();
 6797   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6798   emit_int8(0x57);
 6799   emit_operand(dst, src);
 6800 }
 6801 
 6802 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6803   assert(VM_Version::supports_avx(), "");
 6804   InstructionMark im(this);
 6805   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6806   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6807   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6808   emit_int8(0x57);
 6809   emit_operand(dst, src);
 6810 }
 6811 
 6812 // Integer vector arithmetic
 6813 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6814   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6815          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6816   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6817   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6818   emit_int16(0x01, (0xC0 | encode));
 6819 }
 6820 
 6821 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6822   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6823          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6824   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6825   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6826   emit_int16(0x02, (0xC0 | encode));
 6827 }
 6828 
 6829 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6830   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6831   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6832   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6833   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6834 }
 6835 
 6836 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6839   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6840   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6841 }
 6842 
 6843 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6844   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6845   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6846   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6847   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6848 }
 6849 
 6850 void Assembler::paddd(XMMRegister dst, Address src) {
 6851   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6852   InstructionMark im(this);
 6853   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6854   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6855   emit_int8((unsigned char)0xFE);
 6856   emit_operand(dst, src);
 6857 }
 6858 
 6859 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6861   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6862   attributes.set_rex_vex_w_reverted();
 6863   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6864   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6865 }
 6866 
 6867 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6868   assert(VM_Version::supports_sse3(), "");
 6869   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6870   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6871   emit_int16(0x01, (0xC0 | encode));
 6872 }
 6873 
 6874 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6875   assert(VM_Version::supports_sse3(), "");
 6876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6877   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6878   emit_int16(0x02, (0xC0 | encode));
 6879 }
 6880 
 6881 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6882   assert(UseAVX > 0, "requires some form of AVX");
 6883   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6884   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6885   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6886 }
 6887 
 6888 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6889   assert(UseAVX > 0, "requires some form of AVX");
 6890   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6891   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6892   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6893 }
 6894 
 6895 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6896   assert(UseAVX > 0, "requires some form of AVX");
 6897   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6898   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6899   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6900 }
 6901 
 6902 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6903   assert(UseAVX > 0, "requires some form of AVX");
 6904   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6905   attributes.set_rex_vex_w_reverted();
 6906   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6907   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6908 }
 6909 
 6910 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6911   assert(UseAVX > 0, "requires some form of AVX");
 6912   InstructionMark im(this);
 6913   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6914   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6915   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6916   emit_int8((unsigned char)0xFC);
 6917   emit_operand(dst, src);
 6918 }
 6919 
 6920 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6921   assert(UseAVX > 0, "requires some form of AVX");
 6922   InstructionMark im(this);
 6923   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6924   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6925   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6926   emit_int8((unsigned char)0xFD);
 6927   emit_operand(dst, src);
 6928 }
 6929 
 6930 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6931   assert(UseAVX > 0, "requires some form of AVX");
 6932   InstructionMark im(this);
 6933   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6934   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6935   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6936   emit_int8((unsigned char)0xFE);
 6937   emit_operand(dst, src);
 6938 }
 6939 
 6940 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6941   assert(UseAVX > 0, "requires some form of AVX");
 6942   InstructionMark im(this);
 6943   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6944   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6945   attributes.set_rex_vex_w_reverted();
 6946   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6947   emit_int8((unsigned char)0xD4);
 6948   emit_operand(dst, src);
 6949 }
 6950 
 6951 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6952   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6953   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6954   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6955   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6956 }
 6957 
 6958 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6959   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6960   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6961   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6962   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6963 }
 6964 
 6965 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6966   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6967   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6968   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6969 }
 6970 
 6971 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6972   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6973   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6974   attributes.set_rex_vex_w_reverted();
 6975   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6976   emit_int8((unsigned char)0xFB);
 6977   emit_int8((0xC0 | encode));
 6978 }
 6979 
 6980 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6981   assert(UseAVX > 0, "requires some form of AVX");
 6982   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6983   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6984   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6985 }
 6986 
 6987 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6988   assert(UseAVX > 0, "requires some form of AVX");
 6989   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6990   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6991   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6992 }
 6993 
 6994 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6995   assert(UseAVX > 0, "requires some form of AVX");
 6996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6997   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6998   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6999 }
 7000 
 7001 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7002   assert(UseAVX > 0, "requires some form of AVX");
 7003   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7004   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7005   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7006 }
 7007 
 7008 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7009   assert(UseAVX > 0, "requires some form of AVX");
 7010   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7011   attributes.set_rex_vex_w_reverted();
 7012   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7013   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7014 }
 7015 
 7016 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7017   assert(UseAVX > 0, "requires some form of AVX");
 7018   InstructionMark im(this);
 7019   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7020   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7021   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7022   emit_int8((unsigned char)0xF8);
 7023   emit_operand(dst, src);
 7024 }
 7025 
 7026 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7027   assert(UseAVX > 0, "requires some form of AVX");
 7028   InstructionMark im(this);
 7029   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7030   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7031   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7032   emit_int8((unsigned char)0xF9);
 7033   emit_operand(dst, src);
 7034 }
 7035 
 7036 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7037   assert(UseAVX > 0, "requires some form of AVX");
 7038   InstructionMark im(this);
 7039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7040   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7041   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7042   emit_int8((unsigned char)0xFA);
 7043   emit_operand(dst, src);
 7044 }
 7045 
 7046 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7047   assert(UseAVX > 0, "requires some form of AVX");
 7048   InstructionMark im(this);
 7049   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7050   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7051   attributes.set_rex_vex_w_reverted();
 7052   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7053   emit_int8((unsigned char)0xFB);
 7054   emit_operand(dst, src);
 7055 }
 7056 
 7057 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7060   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7061   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7062 }
 7063 
 7064 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7065   assert(VM_Version::supports_sse4_1(), "");
 7066   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7067   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7068   emit_int16(0x40, (0xC0 | encode));
 7069 }
 7070 
 7071 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7072   assert(VM_Version::supports_sse2(), "");
 7073   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7074   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7075   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7076 }
 7077 
 7078 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7079   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7080          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7081          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7082   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7083   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7084   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7085 }
 7086 
 7087 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7088   assert(UseAVX > 0, "requires some form of AVX");
 7089   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7090   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7091   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7092 }
 7093 
 7094 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7095   assert(UseAVX > 0, "requires some form of AVX");
 7096   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7097   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7098   emit_int16(0x40, (0xC0 | encode));
 7099 }
 7100 
 7101 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7102   assert(UseAVX > 2, "requires some form of EVEX");
 7103   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7104   attributes.set_is_evex_instruction();
 7105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7106   emit_int16(0x40, (0xC0 | encode));
 7107 }
 7108 
 7109 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7110   assert(UseAVX > 0, "requires some form of AVX");
 7111   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7112   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7113   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7114 }
 7115 
 7116 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7117   assert(UseAVX > 0, "requires some form of AVX");
 7118   InstructionMark im(this);
 7119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7120   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7121   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7122   emit_int8((unsigned char)0xD5);
 7123   emit_operand(dst, src);
 7124 }
 7125 
 7126 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7127   assert(UseAVX > 0, "requires some form of AVX");
 7128   InstructionMark im(this);
 7129   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7130   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7131   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7132   emit_int8(0x40);
 7133   emit_operand(dst, src);
 7134 }
 7135 
 7136 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7137   assert(UseAVX > 2, "requires some form of EVEX");
 7138   InstructionMark im(this);
 7139   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7140   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7141   attributes.set_is_evex_instruction();
 7142   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7143   emit_int8(0x40);
 7144   emit_operand(dst, src);
 7145 }
 7146 
 7147 // Min, max
 7148 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7149   assert(VM_Version::supports_sse4_1(), "");
 7150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7151   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7152   emit_int16(0x38, (0xC0 | encode));
 7153 }
 7154 
 7155 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7156   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7157         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7160   emit_int16(0x38, (0xC0 | encode));
 7161 }
 7162 
 7163 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7164   assert(VM_Version::supports_sse2(), "");
 7165   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7166   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7167   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7168 }
 7169 
 7170 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7171   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7172         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7173   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7174   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7175   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7176 }
 7177 
 7178 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7179   assert(VM_Version::supports_sse4_1(), "");
 7180   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7181   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7182   emit_int16(0x39, (0xC0 | encode));
 7183 }
 7184 
 7185 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7186   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7187         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7188   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7189   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7190   emit_int16(0x39, (0xC0 | encode));
 7191 }
 7192 
 7193 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7194   assert(UseAVX > 2, "requires AVX512F");
 7195   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7196   attributes.set_is_evex_instruction();
 7197   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7198   emit_int16(0x39, (0xC0 | encode));
 7199 }
 7200 
 7201 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7202   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7203   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7204   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7205   emit_int16(0x5D, (0xC0 | encode));
 7206 }
 7207 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7208   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7209   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7211   emit_int16(0x5D, (0xC0 | encode));
 7212 }
 7213 
 7214 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7215   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7216   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7217   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7218   emit_int16(0x5D, (0xC0 | encode));
 7219 }
 7220 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7221   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7222   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7223   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7224   emit_int16(0x5D, (0xC0 | encode));
 7225 }
 7226 
 7227 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7228   assert(VM_Version::supports_sse4_1(), "");
 7229   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7230   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7231   emit_int16(0x3C, (0xC0 | encode));
 7232 }
 7233 
 7234 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7235   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7236         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7237   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7238   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7239   emit_int16(0x3C, (0xC0 | encode));
 7240 }
 7241 
 7242 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7243   assert(VM_Version::supports_sse2(), "");
 7244   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7245   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7246   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7247 }
 7248 
 7249 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7250   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7251         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7252   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7253   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7254   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7255 }
 7256 
 7257 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7258   assert(VM_Version::supports_sse4_1(), "");
 7259   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7260   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7261   emit_int16(0x3D, (0xC0 | encode));
 7262 }
 7263 
 7264 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7265   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7266         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7268   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7269   emit_int16(0x3D, (0xC0 | encode));
 7270 }
 7271 
 7272 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7273   assert(UseAVX > 2, "requires AVX512F");
 7274   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7275   attributes.set_is_evex_instruction();
 7276   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7277   emit_int16(0x3D, (0xC0 | encode));
 7278 }
 7279 
 7280 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7281   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7282   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7283   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7284   emit_int16(0x5F, (0xC0 | encode));
 7285 }
 7286 
 7287 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7288   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7289   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7290   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7291   emit_int16(0x5F, (0xC0 | encode));
 7292 }
 7293 
 7294 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7295   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7297   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7298   emit_int16(0x5F, (0xC0 | encode));
 7299 }
 7300 
 7301 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7302   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7303   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7304   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7305   emit_int16(0x5F, (0xC0 | encode));
 7306 }
 7307 
 7308 // Shift packed integers left by specified number of bits.
 7309 void Assembler::psllw(XMMRegister dst, int shift) {
 7310   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7311   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7312   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7313   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7314   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7315 }
 7316 
 7317 void Assembler::pslld(XMMRegister dst, int shift) {
 7318   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7319   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7320   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7321   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7322   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7323 }
 7324 
 7325 void Assembler::psllq(XMMRegister dst, int shift) {
 7326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7327   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7328   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7329   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7330   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7331 }
 7332 
 7333 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7336   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7337   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7338 }
 7339 
 7340 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7341   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7342   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7343   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7344   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7345 }
 7346 
 7347 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7348   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7349   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7350   attributes.set_rex_vex_w_reverted();
 7351   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7352   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7353 }
 7354 
 7355 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7356   assert(UseAVX > 0, "requires some form of AVX");
 7357   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7358   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7359   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7360   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7361 }
 7362 
 7363 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7364   assert(UseAVX > 0, "requires some form of AVX");
 7365   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7366   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7367   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7368   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7369   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7370 }
 7371 
 7372 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7373   assert(UseAVX > 0, "requires some form of AVX");
 7374   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7375   attributes.set_rex_vex_w_reverted();
 7376   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7377   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7378   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7379 }
 7380 
 7381 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7382   assert(UseAVX > 0, "requires some form of AVX");
 7383   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7384   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7385   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7386 }
 7387 
 7388 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7389   assert(UseAVX > 0, "requires some form of AVX");
 7390   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7391   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7392   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7393 }
 7394 
 7395 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7396   assert(UseAVX > 0, "requires some form of AVX");
 7397   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7398   attributes.set_rex_vex_w_reverted();
 7399   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7400   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7401 }
 7402 
 7403 // Shift packed integers logically right by specified number of bits.
 7404 void Assembler::psrlw(XMMRegister dst, int shift) {
 7405   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7407   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7408   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7409   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7410 }
 7411 
 7412 void Assembler::psrld(XMMRegister dst, int shift) {
 7413   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7415   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7416   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7417   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7418 }
 7419 
 7420 void Assembler::psrlq(XMMRegister dst, int shift) {
 7421   // Do not confuse it with psrldq SSE2 instruction which
 7422   // shifts 128 bit value in xmm register by number of bytes.
 7423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7424   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7425   attributes.set_rex_vex_w_reverted();
 7426   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7427   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7428   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7429 }
 7430 
 7431 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7432   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7433   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7434   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7435   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7436 }
 7437 
 7438 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7439   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7440   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7441   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7442   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7443 }
 7444 
 7445 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7446   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7447   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7448   attributes.set_rex_vex_w_reverted();
 7449   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7450   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7451 }
 7452 
 7453 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7454   assert(UseAVX > 0, "requires some form of AVX");
 7455   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7456   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7457   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7458   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7459 }
 7460 
 7461 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7462   assert(UseAVX > 0, "requires some form of AVX");
 7463   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7464   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7465   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7466   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7467 }
 7468 
 7469 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7470   assert(UseAVX > 0, "requires some form of AVX");
 7471   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7472   attributes.set_rex_vex_w_reverted();
 7473   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7474   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7475   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7476 }
 7477 
 7478 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7479   assert(UseAVX > 0, "requires some form of AVX");
 7480   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7481   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7482   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7483 }
 7484 
 7485 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7486   assert(UseAVX > 0, "requires some form of AVX");
 7487   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7488   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7489   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7490 }
 7491 
 7492 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7493   assert(UseAVX > 0, "requires some form of AVX");
 7494   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7495   attributes.set_rex_vex_w_reverted();
 7496   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7497   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7498 }
 7499 
 7500 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7501   assert(VM_Version::supports_avx512bw(), "");
 7502   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7503   attributes.set_is_evex_instruction();
 7504   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7505   emit_int16(0x10, (0xC0 | encode));
 7506 }
 7507 
 7508 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7509   assert(VM_Version::supports_avx512bw(), "");
 7510   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7511   attributes.set_is_evex_instruction();
 7512   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7513   emit_int16(0x12, (0xC0 | encode));
 7514 }
 7515 
 7516 // Shift packed integers arithmetically right by specified number of bits.
 7517 void Assembler::psraw(XMMRegister dst, int shift) {
 7518   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7519   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7520   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7521   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7522   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7523 }
 7524 
 7525 void Assembler::psrad(XMMRegister dst, int shift) {
 7526   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7527   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7528   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7529   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7530   emit_int8(0x72);
 7531   emit_int8((0xC0 | encode));
 7532   emit_int8(shift & 0xFF);
 7533 }
 7534 
 7535 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7536   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7537   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7538   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7539   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7540 }
 7541 
 7542 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7543   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7544   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7545   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7546   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7547 }
 7548 
 7549 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7550   assert(UseAVX > 0, "requires some form of AVX");
 7551   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7552   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7553   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7554   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7555 }
 7556 
 7557 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7558   assert(UseAVX > 0, "requires some form of AVX");
 7559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7560   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7561   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7562   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7563 }
 7564 
 7565 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7566   assert(UseAVX > 0, "requires some form of AVX");
 7567   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7568   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7569   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7570 }
 7571 
 7572 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7573   assert(UseAVX > 0, "requires some form of AVX");
 7574   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7575   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7576   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7577 }
 7578 
 7579 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7580   assert(UseAVX > 2, "requires AVX512");
 7581   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7582   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7583   attributes.set_is_evex_instruction();
 7584   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7585   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7586 }
 7587 
 7588 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7589   assert(UseAVX > 2, "requires AVX512");
 7590   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7591   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7592   attributes.set_is_evex_instruction();
 7593   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7594   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7595 }
 7596 
 7597 // logical operations packed integers
 7598 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7599   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7600   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7601   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7602   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7603 }
 7604 
 7605 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7606   assert(UseAVX > 0, "requires some form of AVX");
 7607   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7608   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7609   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7610 }
 7611 
 7612 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7613   assert(UseAVX > 0, "requires some form of AVX");
 7614   InstructionMark im(this);
 7615   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7616   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7617   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7618   emit_int8((unsigned char)0xDB);
 7619   emit_operand(dst, src);
 7620 }
 7621 
 7622 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7623   assert(VM_Version::supports_evex(), "");
 7624   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7626   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7627 }
 7628 
 7629 //Variable Shift packed integers logically left.
 7630 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7631   assert(UseAVX > 1, "requires AVX2");
 7632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7633   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7634   emit_int16(0x47, (0xC0 | encode));
 7635 }
 7636 
 7637 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7638   assert(UseAVX > 1, "requires AVX2");
 7639   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7640   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7641   emit_int16(0x47, (0xC0 | encode));
 7642 }
 7643 
 7644 //Variable Shift packed integers logically right.
 7645 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7646   assert(UseAVX > 1, "requires AVX2");
 7647   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7648   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7649   emit_int16(0x45, (0xC0 | encode));
 7650 }
 7651 
 7652 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7653   assert(UseAVX > 1, "requires AVX2");
 7654   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7655   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7656   emit_int16(0x45, (0xC0 | encode));
 7657 }
 7658 
 7659 //Variable right Shift arithmetic packed integers .
 7660 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7661   assert(UseAVX > 1, "requires AVX2");
 7662   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7663   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7664   emit_int16(0x46, (0xC0 | encode));
 7665 }
 7666 
 7667 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7668   assert(VM_Version::supports_avx512bw(), "");
 7669   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7670   attributes.set_is_evex_instruction();
 7671   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7672   emit_int16(0x11, (0xC0 | encode));
 7673 }
 7674 
 7675 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7676   assert(UseAVX > 2, "requires AVX512");
 7677   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7678   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7679   attributes.set_is_evex_instruction();
 7680   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7681   emit_int16(0x46, (0xC0 | encode));
 7682 }
 7683 
 7684 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7685   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7686   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7687   attributes.set_is_evex_instruction();
 7688   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7689   emit_int16(0x71, (0xC0 | encode));
 7690 }
 7691 
 7692 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7693   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7694   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7695   attributes.set_is_evex_instruction();
 7696   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7697   emit_int16(0x73, (0xC0 | encode));
 7698 }
 7699 
 7700 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7701   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7702   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7703   attributes.set_rex_vex_w_reverted();
 7704   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7705   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7706 }
 7707 
 7708 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7709   assert(UseAVX > 0, "requires some form of AVX");
 7710   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7711   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7712   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7713 }
 7714 
 7715 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7716   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7717   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7718   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7719   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7720 }
 7721 
 7722 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7723   assert(UseAVX > 0, "requires some form of AVX");
 7724   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7725   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7726   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7727 }
 7728 
 7729 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7730   assert(UseAVX > 0, "requires some form of AVX");
 7731   InstructionMark im(this);
 7732   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7733   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7734   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7735   emit_int8((unsigned char)0xEB);
 7736   emit_operand(dst, src);
 7737 }
 7738 
 7739 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7740   assert(VM_Version::supports_evex(), "");
 7741   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7742   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7743   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7744 }
 7745 
 7746 
 7747 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7748   assert(VM_Version::supports_evex(), "");
 7749   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7750   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7751   attributes.set_is_evex_instruction();
 7752   attributes.set_embedded_opmask_register_specifier(mask);
 7753   if (merge) {
 7754     attributes.reset_is_clear_context();
 7755   }
 7756   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7757   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7758 }
 7759 
 7760 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7761   assert(VM_Version::supports_evex(), "");
 7762   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7763   InstructionMark im(this);
 7764   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7765   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7766   attributes.set_is_evex_instruction();
 7767   attributes.set_embedded_opmask_register_specifier(mask);
 7768   if (merge) {
 7769     attributes.reset_is_clear_context();
 7770   }
 7771   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7772   emit_int8((unsigned char)0xEB);
 7773   emit_operand(dst, src);
 7774 }
 7775 
 7776 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7779   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7780   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7781 }
 7782 
 7783 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7784   assert(UseAVX > 0, "requires some form of AVX");
 7785   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7786          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7787          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7788   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7789   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7790   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7791 }
 7792 
 7793 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7794   assert(UseAVX > 0, "requires some form of AVX");
 7795   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7796          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7797          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7798   InstructionMark im(this);
 7799   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7800   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7801   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7802   emit_int8((unsigned char)0xEF);
 7803   emit_operand(dst, src);
 7804 }
 7805 
 7806 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7807   assert(UseAVX > 2, "requires some form of EVEX");
 7808   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7809   attributes.set_rex_vex_w_reverted();
 7810   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7811   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7812 }
 7813 
 7814 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7815   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7816   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7817   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7818   attributes.set_is_evex_instruction();
 7819   attributes.set_embedded_opmask_register_specifier(mask);
 7820   if (merge) {
 7821     attributes.reset_is_clear_context();
 7822   }
 7823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7824   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7825 }
 7826 
 7827 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7828   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7829   InstructionMark im(this);
 7830   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7832   attributes.set_is_evex_instruction();
 7833   attributes.set_embedded_opmask_register_specifier(mask);
 7834   if (merge) {
 7835     attributes.reset_is_clear_context();
 7836   }
 7837   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7838   emit_int8((unsigned char)0xEF);
 7839   emit_operand(dst, src);
 7840 }
 7841 
 7842 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7843   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7844   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7845   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7846   attributes.set_is_evex_instruction();
 7847   attributes.set_embedded_opmask_register_specifier(mask);
 7848   if (merge) {
 7849     attributes.reset_is_clear_context();
 7850   }
 7851   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7852   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7853 }
 7854 
 7855 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7856   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7857   InstructionMark im(this);
 7858   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7859   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7860   attributes.set_is_evex_instruction();
 7861   attributes.set_embedded_opmask_register_specifier(mask);
 7862   if (merge) {
 7863     attributes.reset_is_clear_context();
 7864   }
 7865   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7866   emit_int8((unsigned char)0xEF);
 7867   emit_operand(dst, src);
 7868 }
 7869 
 7870 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7871   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7872   InstructionMark im(this);
 7873   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7874   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7875   attributes.set_is_evex_instruction();
 7876   attributes.set_embedded_opmask_register_specifier(mask);
 7877   if (merge) {
 7878     attributes.reset_is_clear_context();
 7879   }
 7880   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7881   emit_int8((unsigned char)0xDB);
 7882   emit_operand(dst, src);
 7883 }
 7884 
 7885 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7886   assert(VM_Version::supports_evex(), "");
 7887   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7888   attributes.set_is_evex_instruction();
 7889   attributes.set_embedded_opmask_register_specifier(mask);
 7890   if (merge) {
 7891     attributes.reset_is_clear_context();
 7892   }
 7893   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7894   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7895 }
 7896 
 7897 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7898   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7899   InstructionMark im(this);
 7900   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7901   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7902   attributes.set_is_evex_instruction();
 7903   attributes.set_embedded_opmask_register_specifier(mask);
 7904   if (merge) {
 7905     attributes.reset_is_clear_context();
 7906   }
 7907   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7908   emit_int8((unsigned char)0xDB);
 7909   emit_operand(dst, src);
 7910 }
 7911 
 7912 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7913   assert(VM_Version::supports_evex(), "");
 7914   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7915   attributes.set_is_evex_instruction();
 7916   attributes.set_embedded_opmask_register_specifier(mask);
 7917   if (merge) {
 7918     attributes.reset_is_clear_context();
 7919   }
 7920   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7921   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7922 }
 7923 
 7924 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7925   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7926   InstructionMark im(this);
 7927   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7928   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7929   attributes.set_is_evex_instruction();
 7930   attributes.set_embedded_opmask_register_specifier(mask);
 7931   if (merge) {
 7932     attributes.reset_is_clear_context();
 7933   }
 7934   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7935   emit_int8((unsigned char)0xEB);
 7936   emit_operand(dst, src);
 7937 }
 7938 
 7939 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7940   assert(VM_Version::supports_evex(), "requires EVEX support");
 7941   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7942   attributes.set_is_evex_instruction();
 7943   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7944   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7945 }
 7946 
 7947 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7948   assert(VM_Version::supports_evex(), "requires EVEX support");
 7949   assert(dst != xnoreg, "sanity");
 7950   InstructionMark im(this);
 7951   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7952   attributes.set_is_evex_instruction();
 7953   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7954   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7955   emit_int8((unsigned char)0xEF);
 7956   emit_operand(dst, src);
 7957 }
 7958 
 7959 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7960   assert(VM_Version::supports_evex(), "requires EVEX support");
 7961   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7962   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7963   attributes.set_is_evex_instruction();
 7964   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7965   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7966 }
 7967 
 7968 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7969   assert(VM_Version::supports_evex(), "requires EVEX support");
 7970   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7971   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7972   attributes.set_is_evex_instruction();
 7973   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7974   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7975 }
 7976 
 7977 // Register is a class, but it would be assigned numerical value.
 7978 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7979 PRAGMA_DIAG_PUSH
 7980 PRAGMA_NONNULL_IGNORED
 7981 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7982   assert(VM_Version::supports_evex(), "requires EVEX support");
 7983   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7984   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7985   attributes.set_is_evex_instruction();
 7986   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7987   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7988 }
 7989 
 7990 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7991   assert(VM_Version::supports_evex(), "requires EVEX support");
 7992   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7993   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7994   attributes.set_is_evex_instruction();
 7995   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7996   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7997 }
 7998 PRAGMA_DIAG_POP
 7999 
 8000 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8001   assert(VM_Version::supports_evex(), "requires EVEX support");
 8002   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8003   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8004   attributes.set_is_evex_instruction();
 8005   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8006   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8007 }
 8008 
 8009 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8010   assert(VM_Version::supports_evex(), "requires EVEX support");
 8011   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8013   attributes.set_is_evex_instruction();
 8014   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8015   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8016 }
 8017 
 8018 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8019   assert(VM_Version::supports_evex(), "requires EVEX support");
 8020   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8021   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8022   attributes.set_is_evex_instruction();
 8023   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8024   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8025 }
 8026 
 8027 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8028   assert(VM_Version::supports_evex(), "requires EVEX support");
 8029   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8030   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8031   attributes.set_is_evex_instruction();
 8032   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8033   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8034 }
 8035 
 8036 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8037   assert(VM_Version::supports_avx512cd(), "");
 8038   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8039   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8040   attributes.set_is_evex_instruction();
 8041   attributes.set_embedded_opmask_register_specifier(mask);
 8042   if (merge) {
 8043     attributes.reset_is_clear_context();
 8044   }
 8045   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8046   emit_int16(0x44, (0xC0 | encode));
 8047 }
 8048 
 8049 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8050   assert(VM_Version::supports_avx512cd(), "");
 8051   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8052   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8053   attributes.set_is_evex_instruction();
 8054   attributes.set_embedded_opmask_register_specifier(mask);
 8055   if (merge) {
 8056     attributes.reset_is_clear_context();
 8057   }
 8058   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8059   emit_int16(0x44, (0xC0 | encode));
 8060 }
 8061 
 8062 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8063   assert(VM_Version::supports_evex(), "requires EVEX support");
 8064   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8065   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8066   attributes.set_is_evex_instruction();
 8067   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8068   emit_int8(0x25);
 8069   emit_int8((unsigned char)(0xC0 | encode));
 8070   emit_int8(imm8);
 8071 }
 8072 
 8073 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8074   assert(VM_Version::supports_evex(), "requires EVEX support");
 8075   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8076   assert(dst != xnoreg, "sanity");
 8077   InstructionMark im(this);
 8078   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8079   attributes.set_is_evex_instruction();
 8080   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8081   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8082   emit_int8(0x25);
 8083   emit_operand(dst, src3);
 8084   emit_int8(imm8);
 8085 }
 8086 
 8087 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8088   assert(VM_Version::supports_evex(), "requires EVEX support");
 8089   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8090   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8091   attributes.set_is_evex_instruction();
 8092   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8093   emit_int8(0x25);
 8094   emit_int8((unsigned char)(0xC0 | encode));
 8095   emit_int8(imm8);
 8096 }
 8097 
 8098 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8099   assert(VM_Version::supports_evex(), "");
 8100   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8102   attributes.set_is_evex_instruction();
 8103   attributes.set_embedded_opmask_register_specifier(mask);
 8104   if (merge) {
 8105     attributes.reset_is_clear_context();
 8106   }
 8107   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8108   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8109 }
 8110 
 8111 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8112   assert(VM_Version::supports_evex(), "");
 8113   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8114   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8115   attributes.set_is_evex_instruction();
 8116   attributes.set_embedded_opmask_register_specifier(mask);
 8117   if (merge) {
 8118     attributes.reset_is_clear_context();
 8119   }
 8120   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8121   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8122 }
 8123 
 8124 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8125   assert(VM_Version::supports_avx512_vbmi2(), "");
 8126   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8127   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8128   attributes.set_is_evex_instruction();
 8129   attributes.set_embedded_opmask_register_specifier(mask);
 8130   if (merge) {
 8131     attributes.reset_is_clear_context();
 8132   }
 8133   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8134   emit_int16(0x62, (0xC0 | encode));
 8135 }
 8136 
 8137 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8138   assert(VM_Version::supports_avx512_vbmi2(), "");
 8139   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8140   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8141   attributes.set_is_evex_instruction();
 8142   attributes.set_embedded_opmask_register_specifier(mask);
 8143   if (merge) {
 8144     attributes.reset_is_clear_context();
 8145   }
 8146   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8147   emit_int16(0x62, (0xC0 | encode));
 8148 }
 8149 
 8150 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8151   assert(VM_Version::supports_evex(), "");
 8152   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8153   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8154   attributes.set_is_evex_instruction();
 8155   attributes.set_embedded_opmask_register_specifier(mask);
 8156   if (merge) {
 8157     attributes.reset_is_clear_context();
 8158   }
 8159   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8160   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8161 }
 8162 
 8163 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8164   assert(VM_Version::supports_evex(), "");
 8165   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8166   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8167   attributes.set_is_evex_instruction();
 8168   attributes.set_embedded_opmask_register_specifier(mask);
 8169   if (merge) {
 8170     attributes.reset_is_clear_context();
 8171   }
 8172   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8173   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8174 }
 8175 
 8176 // vinserti forms
 8177 
 8178 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8179   assert(VM_Version::supports_avx2(), "");
 8180   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8181   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8182   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8183   // last byte:
 8184   // 0x00 - insert into lower 128 bits
 8185   // 0x01 - insert into upper 128 bits
 8186   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8187 }
 8188 
 8189 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8190   assert(VM_Version::supports_avx2(), "");
 8191   assert(dst != xnoreg, "sanity");
 8192   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8193   InstructionMark im(this);
 8194   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8195   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8196   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8197   emit_int8(0x38);
 8198   emit_operand(dst, src);
 8199   // 0x00 - insert into lower 128 bits
 8200   // 0x01 - insert into upper 128 bits
 8201   emit_int8(imm8 & 0x01);
 8202 }
 8203 
 8204 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8205   assert(VM_Version::supports_evex(), "");
 8206   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8207   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8208   attributes.set_is_evex_instruction();
 8209   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8210   // imm8:
 8211   // 0x00 - insert into q0 128 bits (0..127)
 8212   // 0x01 - insert into q1 128 bits (128..255)
 8213   // 0x02 - insert into q2 128 bits (256..383)
 8214   // 0x03 - insert into q3 128 bits (384..511)
 8215   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8216 }
 8217 
 8218 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8219   assert(VM_Version::supports_evex(), "");
 8220   assert(dst != xnoreg, "sanity");
 8221   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8222   InstructionMark im(this);
 8223   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8224   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8225   attributes.set_is_evex_instruction();
 8226   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8227   emit_int8(0x18);
 8228   emit_operand(dst, src);
 8229   // 0x00 - insert into q0 128 bits (0..127)
 8230   // 0x01 - insert into q1 128 bits (128..255)
 8231   // 0x02 - insert into q2 128 bits (256..383)
 8232   // 0x03 - insert into q3 128 bits (384..511)
 8233   emit_int8(imm8 & 0x03);
 8234 }
 8235 
 8236 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8237   assert(VM_Version::supports_evex(), "");
 8238   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8239   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8240   attributes.set_is_evex_instruction();
 8241   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8242   //imm8:
 8243   // 0x00 - insert into lower 256 bits
 8244   // 0x01 - insert into upper 256 bits
 8245   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8246 }
 8247 
 8248 
 8249 // vinsertf forms
 8250 
 8251 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8252   assert(VM_Version::supports_avx(), "");
 8253   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8254   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8255   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8256   // imm8:
 8257   // 0x00 - insert into lower 128 bits
 8258   // 0x01 - insert into upper 128 bits
 8259   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8260 }
 8261 
 8262 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8263   assert(VM_Version::supports_avx(), "");
 8264   assert(dst != xnoreg, "sanity");
 8265   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8266   InstructionMark im(this);
 8267   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8268   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8269   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8270   emit_int8(0x18);
 8271   emit_operand(dst, src);
 8272   // 0x00 - insert into lower 128 bits
 8273   // 0x01 - insert into upper 128 bits
 8274   emit_int8(imm8 & 0x01);
 8275 }
 8276 
 8277 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8278   assert(VM_Version::supports_evex(), "");
 8279   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8280   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8281   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8282   // imm8:
 8283   // 0x00 - insert into q0 128 bits (0..127)
 8284   // 0x01 - insert into q1 128 bits (128..255)
 8285   // 0x02 - insert into q0 128 bits (256..383)
 8286   // 0x03 - insert into q1 128 bits (384..512)
 8287   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8288 }
 8289 
 8290 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8291   assert(VM_Version::supports_evex(), "");
 8292   assert(dst != xnoreg, "sanity");
 8293   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8294   InstructionMark im(this);
 8295   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8296   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8297   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8298   emit_int8(0x18);
 8299   emit_operand(dst, src);
 8300   // 0x00 - insert into q0 128 bits (0..127)
 8301   // 0x01 - insert into q1 128 bits (128..255)
 8302   // 0x02 - insert into q0 128 bits (256..383)
 8303   // 0x03 - insert into q1 128 bits (384..512)
 8304   emit_int8(imm8 & 0x03);
 8305 }
 8306 
 8307 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8308   assert(VM_Version::supports_evex(), "");
 8309   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8310   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8311   attributes.set_is_evex_instruction();
 8312   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8313   // imm8:
 8314   // 0x00 - insert into lower 256 bits
 8315   // 0x01 - insert into upper 256 bits
 8316   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8317 }
 8318 
 8319 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8320   assert(VM_Version::supports_evex(), "");
 8321   assert(dst != xnoreg, "sanity");
 8322   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8323   InstructionMark im(this);
 8324   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8325   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8326   attributes.set_is_evex_instruction();
 8327   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8328   emit_int8(0x1A);
 8329   emit_operand(dst, src);
 8330   // 0x00 - insert into lower 256 bits
 8331   // 0x01 - insert into upper 256 bits
 8332   emit_int8(imm8 & 0x01);
 8333 }
 8334 
 8335 
 8336 // vextracti forms
 8337 
 8338 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8339   assert(VM_Version::supports_avx2(), "");
 8340   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8341   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8342   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8343   // imm8:
 8344   // 0x00 - extract from lower 128 bits
 8345   // 0x01 - extract from upper 128 bits
 8346   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8347 }
 8348 
 8349 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8350   assert(VM_Version::supports_avx2(), "");
 8351   assert(src != xnoreg, "sanity");
 8352   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8353   InstructionMark im(this);
 8354   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8355   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8356   attributes.reset_is_clear_context();
 8357   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8358   emit_int8(0x39);
 8359   emit_operand(src, dst);
 8360   // 0x00 - extract from lower 128 bits
 8361   // 0x01 - extract from upper 128 bits
 8362   emit_int8(imm8 & 0x01);
 8363 }
 8364 
 8365 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8366   assert(VM_Version::supports_evex(), "");
 8367   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8368   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8369   attributes.set_is_evex_instruction();
 8370   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8371   // imm8:
 8372   // 0x00 - extract from bits 127:0
 8373   // 0x01 - extract from bits 255:128
 8374   // 0x02 - extract from bits 383:256
 8375   // 0x03 - extract from bits 511:384
 8376   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8377 }
 8378 
 8379 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8380   assert(VM_Version::supports_evex(), "");
 8381   assert(src != xnoreg, "sanity");
 8382   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8383   InstructionMark im(this);
 8384   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8385   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8386   attributes.reset_is_clear_context();
 8387   attributes.set_is_evex_instruction();
 8388   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8389   emit_int8(0x39);
 8390   emit_operand(src, dst);
 8391   // 0x00 - extract from bits 127:0
 8392   // 0x01 - extract from bits 255:128
 8393   // 0x02 - extract from bits 383:256
 8394   // 0x03 - extract from bits 511:384
 8395   emit_int8(imm8 & 0x03);
 8396 }
 8397 
 8398 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8399   assert(VM_Version::supports_avx512dq(), "");
 8400   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8401   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8402   attributes.set_is_evex_instruction();
 8403   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8404   // imm8:
 8405   // 0x00 - extract from bits 127:0
 8406   // 0x01 - extract from bits 255:128
 8407   // 0x02 - extract from bits 383:256
 8408   // 0x03 - extract from bits 511:384
 8409   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8410 }
 8411 
 8412 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8413   assert(VM_Version::supports_evex(), "");
 8414   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8415   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8416   attributes.set_is_evex_instruction();
 8417   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8418   // imm8:
 8419   // 0x00 - extract from lower 256 bits
 8420   // 0x01 - extract from upper 256 bits
 8421   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8422 }
 8423 
 8424 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8425   assert(VM_Version::supports_evex(), "");
 8426   assert(src != xnoreg, "sanity");
 8427   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8428   InstructionMark im(this);
 8429   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8430   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8431   attributes.reset_is_clear_context();
 8432   attributes.set_is_evex_instruction();
 8433   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8434   emit_int8(0x38);
 8435   emit_operand(src, dst);
 8436   // 0x00 - extract from lower 256 bits
 8437   // 0x01 - extract from upper 256 bits
 8438   emit_int8(imm8 & 0x01);
 8439 }
 8440 // vextractf forms
 8441 
 8442 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8443   assert(VM_Version::supports_avx(), "");
 8444   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8445   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8446   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8447   // imm8:
 8448   // 0x00 - extract from lower 128 bits
 8449   // 0x01 - extract from upper 128 bits
 8450   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8451 }
 8452 
 8453 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8454   assert(VM_Version::supports_avx(), "");
 8455   assert(src != xnoreg, "sanity");
 8456   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8457   InstructionMark im(this);
 8458   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8459   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8460   attributes.reset_is_clear_context();
 8461   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8462   emit_int8(0x19);
 8463   emit_operand(src, dst);
 8464   // 0x00 - extract from lower 128 bits
 8465   // 0x01 - extract from upper 128 bits
 8466   emit_int8(imm8 & 0x01);
 8467 }
 8468 
 8469 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8470   assert(VM_Version::supports_evex(), "");
 8471   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8472   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8473   attributes.set_is_evex_instruction();
 8474   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8475   // imm8:
 8476   // 0x00 - extract from bits 127:0
 8477   // 0x01 - extract from bits 255:128
 8478   // 0x02 - extract from bits 383:256
 8479   // 0x03 - extract from bits 511:384
 8480   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8481 }
 8482 
 8483 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8484   assert(VM_Version::supports_evex(), "");
 8485   assert(src != xnoreg, "sanity");
 8486   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8487   InstructionMark im(this);
 8488   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8489   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8490   attributes.reset_is_clear_context();
 8491   attributes.set_is_evex_instruction();
 8492   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8493   emit_int8(0x19);
 8494   emit_operand(src, dst);
 8495   // 0x00 - extract from bits 127:0
 8496   // 0x01 - extract from bits 255:128
 8497   // 0x02 - extract from bits 383:256
 8498   // 0x03 - extract from bits 511:384
 8499   emit_int8(imm8 & 0x03);
 8500 }
 8501 
 8502 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8503   assert(VM_Version::supports_avx512dq(), "");
 8504   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8505   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8506   attributes.set_is_evex_instruction();
 8507   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8508   // imm8:
 8509   // 0x00 - extract from bits 127:0
 8510   // 0x01 - extract from bits 255:128
 8511   // 0x02 - extract from bits 383:256
 8512   // 0x03 - extract from bits 511:384
 8513   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8514 }
 8515 
 8516 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8517   assert(VM_Version::supports_evex(), "");
 8518   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8519   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8520   attributes.set_is_evex_instruction();
 8521   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8522   // imm8:
 8523   // 0x00 - extract from lower 256 bits
 8524   // 0x01 - extract from upper 256 bits
 8525   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8526 }
 8527 
 8528 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8529   assert(VM_Version::supports_evex(), "");
 8530   assert(src != xnoreg, "sanity");
 8531   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8532   InstructionMark im(this);
 8533   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8534   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8535   attributes.reset_is_clear_context();
 8536   attributes.set_is_evex_instruction();
 8537   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8538   emit_int8(0x1B);
 8539   emit_operand(src, dst);
 8540   // 0x00 - extract from lower 256 bits
 8541   // 0x01 - extract from upper 256 bits
 8542   emit_int8(imm8 & 0x01);
 8543 }
 8544 
 8545 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8546 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8547   assert(VM_Version::supports_avx2(), "");
 8548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8549   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8550   emit_int16(0x78, (0xC0 | encode));
 8551 }
 8552 
 8553 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8554   assert(VM_Version::supports_avx2(), "");
 8555   assert(dst != xnoreg, "sanity");
 8556   InstructionMark im(this);
 8557   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8558   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8559   // swap src<->dst for encoding
 8560   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8561   emit_int8(0x78);
 8562   emit_operand(dst, src);
 8563 }
 8564 
 8565 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8566 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8567   assert(VM_Version::supports_avx2(), "");
 8568   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8569   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8570   emit_int16(0x79, (0xC0 | encode));
 8571 }
 8572 
 8573 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8574   assert(VM_Version::supports_avx2(), "");
 8575   assert(dst != xnoreg, "sanity");
 8576   InstructionMark im(this);
 8577   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8578   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8579   // swap src<->dst for encoding
 8580   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8581   emit_int8(0x79);
 8582   emit_operand(dst, src);
 8583 }
 8584 
 8585 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8586   assert(UseAVX > 0, "requires some form of AVX");
 8587   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8588   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8589   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8590 }
 8591 
 8592 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8593   assert(UseAVX > 0, "requires some form of AVX");
 8594   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8595   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8596   emit_int16(0x69, (0xC0 | encode));
 8597 }
 8598 
 8599 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8600   assert(UseAVX > 0, "requires some form of AVX");
 8601   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8602   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8603   emit_int16(0x61, (0xC0 | encode));
 8604 }
 8605 
 8606 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8607   assert(UseAVX > 0, "requires some form of AVX");
 8608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8609   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8610   emit_int16(0x6A, (0xC0 | encode));
 8611 }
 8612 
 8613 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8614   assert(UseAVX > 0, "requires some form of AVX");
 8615   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8616   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8617   emit_int16(0x62, (0xC0 | encode));
 8618 }
 8619 
 8620 // xmm/mem sourced byte/word/dword/qword replicate
 8621 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8622   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8623   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8624   attributes.set_is_evex_instruction();
 8625   attributes.set_embedded_opmask_register_specifier(mask);
 8626   if (merge) {
 8627     attributes.reset_is_clear_context();
 8628   }
 8629   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8630   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8631 }
 8632 
 8633 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8634   InstructionMark im(this);
 8635   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8636   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8637   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8638   attributes.set_is_evex_instruction();
 8639   attributes.set_embedded_opmask_register_specifier(mask);
 8640   if (merge) {
 8641     attributes.reset_is_clear_context();
 8642   }
 8643   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8644   emit_int8((unsigned char)0xFC);
 8645   emit_operand(dst, src);
 8646 }
 8647 
 8648 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8649   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8651   attributes.set_is_evex_instruction();
 8652   attributes.set_embedded_opmask_register_specifier(mask);
 8653   if (merge) {
 8654     attributes.reset_is_clear_context();
 8655   }
 8656   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8657   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8658 }
 8659 
 8660 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8661   InstructionMark im(this);
 8662   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8663   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8664   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8665   attributes.set_is_evex_instruction();
 8666   attributes.set_embedded_opmask_register_specifier(mask);
 8667   if (merge) {
 8668     attributes.reset_is_clear_context();
 8669   }
 8670   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8671   emit_int8((unsigned char)0xFD);
 8672   emit_operand(dst, src);
 8673 }
 8674 
 8675 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8676   assert(VM_Version::supports_evex(), "");
 8677   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8678   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8679   attributes.set_is_evex_instruction();
 8680   attributes.set_embedded_opmask_register_specifier(mask);
 8681   if (merge) {
 8682     attributes.reset_is_clear_context();
 8683   }
 8684   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8685   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8686 }
 8687 
 8688 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8689   InstructionMark im(this);
 8690   assert(VM_Version::supports_evex(), "");
 8691   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8692   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8693   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8694   attributes.set_is_evex_instruction();
 8695   attributes.set_embedded_opmask_register_specifier(mask);
 8696   if (merge) {
 8697     attributes.reset_is_clear_context();
 8698   }
 8699   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8700   emit_int8((unsigned char)0xFE);
 8701   emit_operand(dst, src);
 8702 }
 8703 
 8704 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8705   assert(VM_Version::supports_evex(), "");
 8706   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8707   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8708   attributes.set_is_evex_instruction();
 8709   attributes.set_embedded_opmask_register_specifier(mask);
 8710   if (merge) {
 8711     attributes.reset_is_clear_context();
 8712   }
 8713   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8714   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8715 }
 8716 
 8717 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8718   InstructionMark im(this);
 8719   assert(VM_Version::supports_evex(), "");
 8720   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8721   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8722   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8723   attributes.set_is_evex_instruction();
 8724   attributes.set_embedded_opmask_register_specifier(mask);
 8725   if (merge) {
 8726     attributes.reset_is_clear_context();
 8727   }
 8728   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8729   emit_int8((unsigned char)0xD4);
 8730   emit_operand(dst, src);
 8731 }
 8732 
 8733 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8734   assert(VM_Version::supports_evex(), "");
 8735   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8736   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8737   attributes.set_is_evex_instruction();
 8738   attributes.set_embedded_opmask_register_specifier(mask);
 8739   if (merge) {
 8740     attributes.reset_is_clear_context();
 8741   }
 8742   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8743   emit_int16(0x58, (0xC0 | encode));
 8744 }
 8745 
 8746 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8747   InstructionMark im(this);
 8748   assert(VM_Version::supports_evex(), "");
 8749   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8750   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8751   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8752   attributes.set_is_evex_instruction();
 8753   attributes.set_embedded_opmask_register_specifier(mask);
 8754   if (merge) {
 8755     attributes.reset_is_clear_context();
 8756   }
 8757   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8758   emit_int8(0x58);
 8759   emit_operand(dst, src);
 8760 }
 8761 
 8762 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8763   assert(VM_Version::supports_evex(), "");
 8764   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8765   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8766   attributes.set_is_evex_instruction();
 8767   attributes.set_embedded_opmask_register_specifier(mask);
 8768   if (merge) {
 8769     attributes.reset_is_clear_context();
 8770   }
 8771   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8772   emit_int16(0x58, (0xC0 | encode));
 8773 }
 8774 
 8775 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8776   InstructionMark im(this);
 8777   assert(VM_Version::supports_evex(), "");
 8778   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8779   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8780   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8781   attributes.set_is_evex_instruction();
 8782   attributes.set_embedded_opmask_register_specifier(mask);
 8783   if (merge) {
 8784     attributes.reset_is_clear_context();
 8785   }
 8786   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8787   emit_int8(0x58);
 8788   emit_operand(dst, src);
 8789 }
 8790 
 8791 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8792   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8793   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8794   attributes.set_is_evex_instruction();
 8795   attributes.set_embedded_opmask_register_specifier(mask);
 8796   if (merge) {
 8797     attributes.reset_is_clear_context();
 8798   }
 8799   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8800   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8801 }
 8802 
 8803 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8804   InstructionMark im(this);
 8805   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8806   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8807   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8808   attributes.set_is_evex_instruction();
 8809   attributes.set_embedded_opmask_register_specifier(mask);
 8810   if (merge) {
 8811     attributes.reset_is_clear_context();
 8812   }
 8813   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8814   emit_int8((unsigned char)0xF8);
 8815   emit_operand(dst, src);
 8816 }
 8817 
 8818 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8819   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8820   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8821   attributes.set_is_evex_instruction();
 8822   attributes.set_embedded_opmask_register_specifier(mask);
 8823   if (merge) {
 8824     attributes.reset_is_clear_context();
 8825   }
 8826   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8827   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8828 }
 8829 
 8830 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8831   InstructionMark im(this);
 8832   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8833   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8834   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8835   attributes.set_is_evex_instruction();
 8836   attributes.set_embedded_opmask_register_specifier(mask);
 8837   if (merge) {
 8838     attributes.reset_is_clear_context();
 8839   }
 8840   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8841   emit_int8((unsigned char)0xF9);
 8842   emit_operand(dst, src);
 8843 }
 8844 
 8845 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8846   assert(VM_Version::supports_evex(), "");
 8847   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8848   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8849   attributes.set_is_evex_instruction();
 8850   attributes.set_embedded_opmask_register_specifier(mask);
 8851   if (merge) {
 8852     attributes.reset_is_clear_context();
 8853   }
 8854   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8855   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8856 }
 8857 
 8858 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8859   InstructionMark im(this);
 8860   assert(VM_Version::supports_evex(), "");
 8861   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8862   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8863   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8864   attributes.set_is_evex_instruction();
 8865   attributes.set_embedded_opmask_register_specifier(mask);
 8866   if (merge) {
 8867     attributes.reset_is_clear_context();
 8868   }
 8869   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8870   emit_int8((unsigned char)0xFA);
 8871   emit_operand(dst, src);
 8872 }
 8873 
 8874 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8875   assert(VM_Version::supports_evex(), "");
 8876   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8877   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8878   attributes.set_is_evex_instruction();
 8879   attributes.set_embedded_opmask_register_specifier(mask);
 8880   if (merge) {
 8881     attributes.reset_is_clear_context();
 8882   }
 8883   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8884   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 8885 }
 8886 
 8887 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8888   InstructionMark im(this);
 8889   assert(VM_Version::supports_evex(), "");
 8890   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8891   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8892   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8893   attributes.set_is_evex_instruction();
 8894   attributes.set_embedded_opmask_register_specifier(mask);
 8895   if (merge) {
 8896     attributes.reset_is_clear_context();
 8897   }
 8898   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8899   emit_int8((unsigned char)0xFB);
 8900   emit_operand(dst, src);
 8901 }
 8902 
 8903 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8904   assert(VM_Version::supports_evex(), "");
 8905   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8906   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8907   attributes.set_is_evex_instruction();
 8908   attributes.set_embedded_opmask_register_specifier(mask);
 8909   if (merge) {
 8910     attributes.reset_is_clear_context();
 8911   }
 8912   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8913   emit_int16(0x5C, (0xC0 | encode));
 8914 }
 8915 
 8916 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8917   InstructionMark im(this);
 8918   assert(VM_Version::supports_evex(), "");
 8919   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8920   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8921   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8922   attributes.set_is_evex_instruction();
 8923   attributes.set_embedded_opmask_register_specifier(mask);
 8924   if (merge) {
 8925     attributes.reset_is_clear_context();
 8926   }
 8927   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8928   emit_int8(0x5C);
 8929   emit_operand(dst, src);
 8930 }
 8931 
 8932 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8933   assert(VM_Version::supports_evex(), "");
 8934   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8935   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8936   attributes.set_is_evex_instruction();
 8937   attributes.set_embedded_opmask_register_specifier(mask);
 8938   if (merge) {
 8939     attributes.reset_is_clear_context();
 8940   }
 8941   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8942   emit_int16(0x5C, (0xC0 | encode));
 8943 }
 8944 
 8945 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8946   InstructionMark im(this);
 8947   assert(VM_Version::supports_evex(), "");
 8948   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8949   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8950   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8951   attributes.set_is_evex_instruction();
 8952   attributes.set_embedded_opmask_register_specifier(mask);
 8953   if (merge) {
 8954     attributes.reset_is_clear_context();
 8955   }
 8956   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8957   emit_int8(0x5C);
 8958   emit_operand(dst, src);
 8959 }
 8960 
 8961 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8962   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8963   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8964   attributes.set_is_evex_instruction();
 8965   attributes.set_embedded_opmask_register_specifier(mask);
 8966   if (merge) {
 8967     attributes.reset_is_clear_context();
 8968   }
 8969   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8970   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 8971 }
 8972 
 8973 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8974   InstructionMark im(this);
 8975   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8976   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8977   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8978   attributes.set_is_evex_instruction();
 8979   attributes.set_embedded_opmask_register_specifier(mask);
 8980   if (merge) {
 8981     attributes.reset_is_clear_context();
 8982   }
 8983   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8984   emit_int8((unsigned char)0xD5);
 8985   emit_operand(dst, src);
 8986 }
 8987 
 8988 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8989   assert(VM_Version::supports_evex(), "");
 8990   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8991   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8992   attributes.set_is_evex_instruction();
 8993   attributes.set_embedded_opmask_register_specifier(mask);
 8994   if (merge) {
 8995     attributes.reset_is_clear_context();
 8996   }
 8997   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8998   emit_int16(0x40, (0xC0 | encode));
 8999 }
 9000 
 9001 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9002   InstructionMark im(this);
 9003   assert(VM_Version::supports_evex(), "");
 9004   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9005   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9006   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9007   attributes.set_is_evex_instruction();
 9008   attributes.set_embedded_opmask_register_specifier(mask);
 9009   if (merge) {
 9010     attributes.reset_is_clear_context();
 9011   }
 9012   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9013   emit_int8(0x40);
 9014   emit_operand(dst, src);
 9015 }
 9016 
 9017 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9018   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9019   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9020   attributes.set_is_evex_instruction();
 9021   attributes.set_embedded_opmask_register_specifier(mask);
 9022   if (merge) {
 9023     attributes.reset_is_clear_context();
 9024   }
 9025   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9026   emit_int16(0x40, (0xC0 | encode));
 9027 }
 9028 
 9029 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9030   InstructionMark im(this);
 9031   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9032   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9033   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9034   attributes.set_is_evex_instruction();
 9035   attributes.set_embedded_opmask_register_specifier(mask);
 9036   if (merge) {
 9037     attributes.reset_is_clear_context();
 9038   }
 9039   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9040   emit_int8(0x40);
 9041   emit_operand(dst, src);
 9042 }
 9043 
 9044 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9045   assert(VM_Version::supports_evex(), "");
 9046   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9047   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9048   attributes.set_is_evex_instruction();
 9049   attributes.set_embedded_opmask_register_specifier(mask);
 9050   if (merge) {
 9051     attributes.reset_is_clear_context();
 9052   }
 9053   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9054   emit_int16(0x59, (0xC0 | encode));
 9055 }
 9056 
 9057 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9058   InstructionMark im(this);
 9059   assert(VM_Version::supports_evex(), "");
 9060   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9061   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9062   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9063   attributes.set_is_evex_instruction();
 9064   attributes.set_embedded_opmask_register_specifier(mask);
 9065   if (merge) {
 9066     attributes.reset_is_clear_context();
 9067   }
 9068   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9069   emit_int8(0x59);
 9070   emit_operand(dst, src);
 9071 }
 9072 
 9073 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9074   assert(VM_Version::supports_evex(), "");
 9075   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9076   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9077   attributes.set_is_evex_instruction();
 9078   attributes.set_embedded_opmask_register_specifier(mask);
 9079   if (merge) {
 9080     attributes.reset_is_clear_context();
 9081   }
 9082   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9083   emit_int16(0x59, (0xC0 | encode));
 9084 }
 9085 
 9086 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9087   InstructionMark im(this);
 9088   assert(VM_Version::supports_evex(), "");
 9089   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9090   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9091   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9092   attributes.set_is_evex_instruction();
 9093   attributes.set_embedded_opmask_register_specifier(mask);
 9094   if (merge) {
 9095     attributes.reset_is_clear_context();
 9096   }
 9097   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9098   emit_int8(0x59);
 9099   emit_operand(dst, src);
 9100 }
 9101 
 9102 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9103   assert(VM_Version::supports_evex(), "");
 9104   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9105   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9106   attributes.set_is_evex_instruction();
 9107   attributes.set_embedded_opmask_register_specifier(mask);
 9108   if (merge) {
 9109     attributes.reset_is_clear_context();
 9110   }
 9111   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9112   emit_int16(0x51, (0xC0 | encode));
 9113 }
 9114 
 9115 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9116   InstructionMark im(this);
 9117   assert(VM_Version::supports_evex(), "");
 9118   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9119   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9120   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9121   attributes.set_is_evex_instruction();
 9122   attributes.set_embedded_opmask_register_specifier(mask);
 9123   if (merge) {
 9124     attributes.reset_is_clear_context();
 9125   }
 9126   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9127   emit_int8(0x51);
 9128   emit_operand(dst, src);
 9129 }
 9130 
 9131 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9132   assert(VM_Version::supports_evex(), "");
 9133   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9134   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9135   attributes.set_is_evex_instruction();
 9136   attributes.set_embedded_opmask_register_specifier(mask);
 9137   if (merge) {
 9138     attributes.reset_is_clear_context();
 9139   }
 9140   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9141   emit_int16(0x51, (0xC0 | encode));
 9142 }
 9143 
 9144 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9145   InstructionMark im(this);
 9146   assert(VM_Version::supports_evex(), "");
 9147   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9148   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9149   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9150   attributes.set_is_evex_instruction();
 9151   attributes.set_embedded_opmask_register_specifier(mask);
 9152   if (merge) {
 9153     attributes.reset_is_clear_context();
 9154   }
 9155   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9156   emit_int8(0x51);
 9157   emit_operand(dst, src);
 9158 }
 9159 
 9160 
 9161 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9162   assert(VM_Version::supports_evex(), "");
 9163   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9164   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9165   attributes.set_is_evex_instruction();
 9166   attributes.set_embedded_opmask_register_specifier(mask);
 9167   if (merge) {
 9168     attributes.reset_is_clear_context();
 9169   }
 9170   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9171   emit_int16(0x5E, (0xC0 | encode));
 9172 }
 9173 
 9174 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9175   InstructionMark im(this);
 9176   assert(VM_Version::supports_evex(), "");
 9177   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9178   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9179   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9180   attributes.set_is_evex_instruction();
 9181   attributes.set_embedded_opmask_register_specifier(mask);
 9182   if (merge) {
 9183     attributes.reset_is_clear_context();
 9184   }
 9185   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9186   emit_int8(0x5E);
 9187   emit_operand(dst, src);
 9188 }
 9189 
 9190 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9191   assert(VM_Version::supports_evex(), "");
 9192   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9193   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9194   attributes.set_is_evex_instruction();
 9195   attributes.set_embedded_opmask_register_specifier(mask);
 9196   if (merge) {
 9197     attributes.reset_is_clear_context();
 9198   }
 9199   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9200   emit_int16(0x5E, (0xC0 | encode));
 9201 }
 9202 
 9203 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9204   InstructionMark im(this);
 9205   assert(VM_Version::supports_evex(), "");
 9206   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9207   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9208   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9209   attributes.set_is_evex_instruction();
 9210   attributes.set_embedded_opmask_register_specifier(mask);
 9211   if (merge) {
 9212     attributes.reset_is_clear_context();
 9213   }
 9214   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9215   emit_int8(0x5E);
 9216   emit_operand(dst, src);
 9217 }
 9218 
 9219 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9220   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9221   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9222   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9223   attributes.set_is_evex_instruction();
 9224   attributes.set_embedded_opmask_register_specifier(mask);
 9225   if (merge) {
 9226     attributes.reset_is_clear_context();
 9227   }
 9228   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9229   emit_int16(0x1C, (0xC0 | encode));
 9230 }
 9231 
 9232 
 9233 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9234   InstructionMark im(this);
 9235   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9236   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9237   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9238   attributes.set_is_evex_instruction();
 9239   attributes.set_embedded_opmask_register_specifier(mask);
 9240   if (merge) {
 9241     attributes.reset_is_clear_context();
 9242   }
 9243   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9244   emit_int8(0x1C);
 9245   emit_operand(dst, src);
 9246 }
 9247 
 9248 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9249   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9250   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9251   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9252   attributes.set_is_evex_instruction();
 9253   attributes.set_embedded_opmask_register_specifier(mask);
 9254   if (merge) {
 9255     attributes.reset_is_clear_context();
 9256   }
 9257   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9258   emit_int16(0x1D, (0xC0 | encode));
 9259 }
 9260 
 9261 
 9262 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9263   InstructionMark im(this);
 9264   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9265   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9266   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9267   attributes.set_is_evex_instruction();
 9268   attributes.set_embedded_opmask_register_specifier(mask);
 9269   if (merge) {
 9270     attributes.reset_is_clear_context();
 9271   }
 9272   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9273   emit_int8(0x1D);
 9274   emit_operand(dst, src);
 9275 }
 9276 
 9277 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9278   assert(VM_Version::supports_evex(), "");
 9279   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9280   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9281   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9282   attributes.set_is_evex_instruction();
 9283   attributes.set_embedded_opmask_register_specifier(mask);
 9284   if (merge) {
 9285     attributes.reset_is_clear_context();
 9286   }
 9287   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9288   emit_int16(0x1E, (0xC0 | encode));
 9289 }
 9290 
 9291 
 9292 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9293   InstructionMark im(this);
 9294   assert(VM_Version::supports_evex(), "");
 9295   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9296   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9297   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9298   attributes.set_is_evex_instruction();
 9299   attributes.set_embedded_opmask_register_specifier(mask);
 9300   if (merge) {
 9301     attributes.reset_is_clear_context();
 9302   }
 9303   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9304   emit_int8(0x1E);
 9305   emit_operand(dst, src);
 9306 }
 9307 
 9308 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9309   assert(VM_Version::supports_evex(), "");
 9310   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9311   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9312   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9313   attributes.set_is_evex_instruction();
 9314   attributes.set_embedded_opmask_register_specifier(mask);
 9315   if (merge) {
 9316     attributes.reset_is_clear_context();
 9317   }
 9318   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9319   emit_int16(0x1F, (0xC0 | encode));
 9320 }
 9321 
 9322 
 9323 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9324   InstructionMark im(this);
 9325   assert(VM_Version::supports_evex(), "");
 9326   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9327   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9328   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9329   attributes.set_is_evex_instruction();
 9330   attributes.set_embedded_opmask_register_specifier(mask);
 9331   if (merge) {
 9332     attributes.reset_is_clear_context();
 9333   }
 9334   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9335   emit_int8(0x1F);
 9336   emit_operand(dst, src);
 9337 }
 9338 
 9339 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9340   assert(VM_Version::supports_evex(), "");
 9341   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9342   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9343   attributes.set_is_evex_instruction();
 9344   attributes.set_embedded_opmask_register_specifier(mask);
 9345   if (merge) {
 9346     attributes.reset_is_clear_context();
 9347   }
 9348   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9349   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9350 }
 9351 
 9352 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9353   InstructionMark im(this);
 9354   assert(VM_Version::supports_evex(), "");
 9355   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9356   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9357   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9358   attributes.set_is_evex_instruction();
 9359   attributes.set_embedded_opmask_register_specifier(mask);
 9360   if (merge) {
 9361     attributes.reset_is_clear_context();
 9362   }
 9363   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9364   emit_int8((unsigned char)0xA8);
 9365   emit_operand(dst, src);
 9366 }
 9367 
 9368 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9369   assert(VM_Version::supports_evex(), "");
 9370   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9371   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9372   attributes.set_is_evex_instruction();
 9373   attributes.set_embedded_opmask_register_specifier(mask);
 9374   if (merge) {
 9375     attributes.reset_is_clear_context();
 9376   }
 9377   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9378   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9379 }
 9380 
 9381 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9382   InstructionMark im(this);
 9383   assert(VM_Version::supports_evex(), "");
 9384   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9385   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9386   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9387   attributes.set_is_evex_instruction();
 9388   attributes.set_embedded_opmask_register_specifier(mask);
 9389   if (merge) {
 9390     attributes.reset_is_clear_context();
 9391   }
 9392   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9393   emit_int8((unsigned char)0xA8);
 9394   emit_operand(dst, src);
 9395 }
 9396 
 9397 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9398   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9399   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9400   attributes.set_is_evex_instruction();
 9401   attributes.set_embedded_opmask_register_specifier(mask);
 9402   if (merge) {
 9403     attributes.reset_is_clear_context();
 9404   }
 9405   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9406   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9407 }
 9408 
 9409 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9410   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9411   InstructionMark im(this);
 9412   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9413   attributes.set_is_evex_instruction();
 9414   attributes.set_embedded_opmask_register_specifier(mask);
 9415   if (merge) {
 9416     attributes.reset_is_clear_context();
 9417   }
 9418   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9419   emit_int8((unsigned char)0x8D);
 9420   emit_operand(dst, src);
 9421 }
 9422 
 9423 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9424   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9425   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9426   attributes.set_is_evex_instruction();
 9427   attributes.set_embedded_opmask_register_specifier(mask);
 9428   if (merge) {
 9429     attributes.reset_is_clear_context();
 9430   }
 9431   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9432   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9433 }
 9434 
 9435 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9436   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9437   InstructionMark im(this);
 9438   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9439   attributes.set_is_evex_instruction();
 9440   attributes.set_embedded_opmask_register_specifier(mask);
 9441   if (merge) {
 9442     attributes.reset_is_clear_context();
 9443   }
 9444   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9445   emit_int8((unsigned char)0x8D);
 9446   emit_operand(dst, src);
 9447 }
 9448 
 9449 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9450   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9451   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9452   attributes.set_is_evex_instruction();
 9453   attributes.set_embedded_opmask_register_specifier(mask);
 9454   if (merge) {
 9455     attributes.reset_is_clear_context();
 9456   }
 9457   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9458   emit_int16(0x36, (0xC0 | encode));
 9459 }
 9460 
 9461 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9462   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9463   InstructionMark im(this);
 9464   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9465   attributes.set_is_evex_instruction();
 9466   attributes.set_embedded_opmask_register_specifier(mask);
 9467   if (merge) {
 9468     attributes.reset_is_clear_context();
 9469   }
 9470   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9471   emit_int8(0x36);
 9472   emit_operand(dst, src);
 9473 }
 9474 
 9475 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9476   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9477   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9478   attributes.set_is_evex_instruction();
 9479   attributes.set_embedded_opmask_register_specifier(mask);
 9480   if (merge) {
 9481     attributes.reset_is_clear_context();
 9482   }
 9483   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9484   emit_int16(0x36, (0xC0 | encode));
 9485 }
 9486 
 9487 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9488   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9489   InstructionMark im(this);
 9490   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9491   attributes.set_is_evex_instruction();
 9492   attributes.set_embedded_opmask_register_specifier(mask);
 9493   if (merge) {
 9494     attributes.reset_is_clear_context();
 9495   }
 9496   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9497   emit_int8(0x36);
 9498   emit_operand(dst, src);
 9499 }
 9500 
 9501 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9502   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9504   attributes.set_is_evex_instruction();
 9505   attributes.set_embedded_opmask_register_specifier(mask);
 9506   if (merge) {
 9507     attributes.reset_is_clear_context();
 9508   }
 9509   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9510   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9511 }
 9512 
 9513 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9514   assert(VM_Version::supports_evex(), "");
 9515   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9516   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9517   attributes.set_is_evex_instruction();
 9518   attributes.set_embedded_opmask_register_specifier(mask);
 9519   if (merge) {
 9520     attributes.reset_is_clear_context();
 9521   }
 9522   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9523   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9524 }
 9525 
 9526 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9527   assert(VM_Version::supports_evex(), "");
 9528   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9529   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9530   attributes.set_is_evex_instruction();
 9531   attributes.set_embedded_opmask_register_specifier(mask);
 9532   if (merge) {
 9533     attributes.reset_is_clear_context();
 9534   }
 9535   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9536   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9537 }
 9538 
 9539 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9540   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9541   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9542   attributes.set_is_evex_instruction();
 9543   attributes.set_embedded_opmask_register_specifier(mask);
 9544   if (merge) {
 9545     attributes.reset_is_clear_context();
 9546   }
 9547   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9548   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9549 }
 9550 
 9551 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9552   assert(VM_Version::supports_evex(), "");
 9553   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9554   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9555   attributes.set_is_evex_instruction();
 9556   attributes.set_embedded_opmask_register_specifier(mask);
 9557   if (merge) {
 9558     attributes.reset_is_clear_context();
 9559   }
 9560   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9561   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9562 }
 9563 
 9564 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9565   assert(VM_Version::supports_evex(), "");
 9566   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9567   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9568   attributes.set_is_evex_instruction();
 9569   attributes.set_embedded_opmask_register_specifier(mask);
 9570   if (merge) {
 9571     attributes.reset_is_clear_context();
 9572   }
 9573   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9574   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9575 }
 9576 
 9577 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9578   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9579   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9580   attributes.set_is_evex_instruction();
 9581   attributes.set_embedded_opmask_register_specifier(mask);
 9582   if (merge) {
 9583     attributes.reset_is_clear_context();
 9584   }
 9585   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9586   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9587 }
 9588 
 9589 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9590   assert(VM_Version::supports_evex(), "");
 9591   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9592   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9593   attributes.set_is_evex_instruction();
 9594   attributes.set_embedded_opmask_register_specifier(mask);
 9595   if (merge) {
 9596     attributes.reset_is_clear_context();
 9597   }
 9598   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9599   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9600 }
 9601 
 9602 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9603   assert(VM_Version::supports_evex(), "");
 9604   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9605   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9606   attributes.set_is_evex_instruction();
 9607   attributes.set_embedded_opmask_register_specifier(mask);
 9608   if (merge) {
 9609     attributes.reset_is_clear_context();
 9610   }
 9611   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9612   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9613 }
 9614 
 9615 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9616   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9617   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9618   attributes.set_is_evex_instruction();
 9619   attributes.set_embedded_opmask_register_specifier(mask);
 9620   if (merge) {
 9621     attributes.reset_is_clear_context();
 9622   }
 9623   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9624   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9625 }
 9626 
 9627 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9628   assert(VM_Version::supports_evex(), "");
 9629   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9630   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9631   attributes.set_is_evex_instruction();
 9632   attributes.set_embedded_opmask_register_specifier(mask);
 9633   if (merge) {
 9634     attributes.reset_is_clear_context();
 9635   }
 9636   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9637   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9638 }
 9639 
 9640 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9641   assert(VM_Version::supports_evex(), "");
 9642   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9643   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9644   attributes.set_is_evex_instruction();
 9645   attributes.set_embedded_opmask_register_specifier(mask);
 9646   if (merge) {
 9647     attributes.reset_is_clear_context();
 9648   }
 9649   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9650   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9651 }
 9652 
 9653 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9654   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9655   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9656   attributes.set_is_evex_instruction();
 9657   attributes.set_embedded_opmask_register_specifier(mask);
 9658   if (merge) {
 9659     attributes.reset_is_clear_context();
 9660   }
 9661   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9662   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9663 }
 9664 
 9665 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9666   assert(VM_Version::supports_evex(), "");
 9667   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9669   attributes.set_is_evex_instruction();
 9670   attributes.set_embedded_opmask_register_specifier(mask);
 9671   if (merge) {
 9672     attributes.reset_is_clear_context();
 9673   }
 9674   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9675   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9676 }
 9677 
 9678 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9679   assert(VM_Version::supports_evex(), "");
 9680   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9681   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9682   attributes.set_is_evex_instruction();
 9683   attributes.set_embedded_opmask_register_specifier(mask);
 9684   if (merge) {
 9685     attributes.reset_is_clear_context();
 9686   }
 9687   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9688   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9689 }
 9690 
 9691 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9692   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9693   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9694   attributes.set_is_evex_instruction();
 9695   attributes.set_embedded_opmask_register_specifier(mask);
 9696   if (merge) {
 9697     attributes.reset_is_clear_context();
 9698   }
 9699   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9700   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9701 }
 9702 
 9703 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9704   assert(VM_Version::supports_evex(), "");
 9705   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9706   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9707   attributes.set_is_evex_instruction();
 9708   attributes.set_embedded_opmask_register_specifier(mask);
 9709   if (merge) {
 9710     attributes.reset_is_clear_context();
 9711   }
 9712   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9713   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9714 }
 9715 
 9716 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9717   assert(VM_Version::supports_evex(), "");
 9718   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9719   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9720   attributes.set_is_evex_instruction();
 9721   attributes.set_embedded_opmask_register_specifier(mask);
 9722   if (merge) {
 9723     attributes.reset_is_clear_context();
 9724   }
 9725   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9726   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9727 }
 9728 
 9729 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9730   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9731   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9732   attributes.set_is_evex_instruction();
 9733   attributes.set_embedded_opmask_register_specifier(mask);
 9734   if (merge) {
 9735     attributes.reset_is_clear_context();
 9736   }
 9737   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9738   emit_int16(0x12, (0xC0 | encode));
 9739 }
 9740 
 9741 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9742   assert(VM_Version::supports_evex(), "");
 9743   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9744   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9745   attributes.set_is_evex_instruction();
 9746   attributes.set_embedded_opmask_register_specifier(mask);
 9747   if (merge) {
 9748     attributes.reset_is_clear_context();
 9749   }
 9750   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9751   emit_int16(0x47, (0xC0 | encode));
 9752 }
 9753 
 9754 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9755   assert(VM_Version::supports_evex(), "");
 9756   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9757   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9758   attributes.set_is_evex_instruction();
 9759   attributes.set_embedded_opmask_register_specifier(mask);
 9760   if (merge) {
 9761     attributes.reset_is_clear_context();
 9762   }
 9763   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9764   emit_int16(0x47, (0xC0 | encode));
 9765 }
 9766 
 9767 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9768   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9769   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9770   attributes.set_is_evex_instruction();
 9771   attributes.set_embedded_opmask_register_specifier(mask);
 9772   if (merge) {
 9773     attributes.reset_is_clear_context();
 9774   }
 9775   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9776   emit_int16(0x10, (0xC0 | encode));
 9777 }
 9778 
 9779 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9780   assert(VM_Version::supports_evex(), "");
 9781   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9782   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9783   attributes.set_is_evex_instruction();
 9784   attributes.set_embedded_opmask_register_specifier(mask);
 9785   if (merge) {
 9786     attributes.reset_is_clear_context();
 9787   }
 9788   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9789   emit_int16(0x45, (0xC0 | encode));
 9790 }
 9791 
 9792 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9793   assert(VM_Version::supports_evex(), "");
 9794   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9795   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9796   attributes.set_is_evex_instruction();
 9797   attributes.set_embedded_opmask_register_specifier(mask);
 9798   if (merge) {
 9799     attributes.reset_is_clear_context();
 9800   }
 9801   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9802   emit_int16(0x45, (0xC0 | encode));
 9803 }
 9804 
 9805 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9806   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9807   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9808   attributes.set_is_evex_instruction();
 9809   attributes.set_embedded_opmask_register_specifier(mask);
 9810   if (merge) {
 9811     attributes.reset_is_clear_context();
 9812   }
 9813   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9814   emit_int16(0x11, (0xC0 | encode));
 9815 }
 9816 
 9817 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9818   assert(VM_Version::supports_evex(), "");
 9819   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9820   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9821   attributes.set_is_evex_instruction();
 9822   attributes.set_embedded_opmask_register_specifier(mask);
 9823   if (merge) {
 9824     attributes.reset_is_clear_context();
 9825   }
 9826   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9827   emit_int16(0x46, (0xC0 | encode));
 9828 }
 9829 
 9830 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9831   assert(VM_Version::supports_evex(), "");
 9832   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9833   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9834   attributes.set_is_evex_instruction();
 9835   attributes.set_embedded_opmask_register_specifier(mask);
 9836   if (merge) {
 9837     attributes.reset_is_clear_context();
 9838   }
 9839   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9840   emit_int16(0x46, (0xC0 | encode));
 9841 }
 9842 
 9843 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9844   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9845   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9846   attributes.set_is_evex_instruction();
 9847   attributes.set_embedded_opmask_register_specifier(mask);
 9848   if (merge) {
 9849     attributes.reset_is_clear_context();
 9850   }
 9851   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9852   emit_int16(0x38, (0xC0 | encode));
 9853 }
 9854 
 9855 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9856   assert(VM_Version::supports_avx512bw(), "");
 9857   InstructionMark im(this);
 9858   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9859   attributes.set_is_evex_instruction();
 9860   attributes.set_embedded_opmask_register_specifier(mask);
 9861   if (merge) {
 9862     attributes.reset_is_clear_context();
 9863   }
 9864   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9865   emit_int8(0x38);
 9866   emit_operand(dst, src);
 9867 }
 9868 
 9869 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9870   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9871   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9872   attributes.set_is_evex_instruction();
 9873   attributes.set_embedded_opmask_register_specifier(mask);
 9874   if (merge) {
 9875     attributes.reset_is_clear_context();
 9876   }
 9877   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9878   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9879 }
 9880 
 9881 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9882   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9883   InstructionMark im(this);
 9884   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9885   attributes.set_is_evex_instruction();
 9886   attributes.set_embedded_opmask_register_specifier(mask);
 9887   if (merge) {
 9888     attributes.reset_is_clear_context();
 9889   }
 9890   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9891   emit_int8((unsigned char)0xEA);
 9892   emit_operand(dst, src);
 9893 }
 9894 
 9895 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9896   assert(VM_Version::supports_evex(), "");
 9897   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9898   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9899   attributes.set_is_evex_instruction();
 9900   attributes.set_embedded_opmask_register_specifier(mask);
 9901   if (merge) {
 9902     attributes.reset_is_clear_context();
 9903   }
 9904   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9905   emit_int16(0x39, (0xC0 | encode));
 9906 }
 9907 
 9908 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9909   assert(VM_Version::supports_evex(), "");
 9910   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9911   InstructionMark im(this);
 9912   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9913   attributes.set_is_evex_instruction();
 9914   attributes.set_embedded_opmask_register_specifier(mask);
 9915   if (merge) {
 9916     attributes.reset_is_clear_context();
 9917   }
 9918   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9919   emit_int8(0x39);
 9920   emit_operand(dst, src);
 9921 }
 9922 
 9923 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9924   assert(VM_Version::supports_evex(), "");
 9925   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9926   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9927   attributes.set_is_evex_instruction();
 9928   attributes.set_embedded_opmask_register_specifier(mask);
 9929   if (merge) {
 9930     attributes.reset_is_clear_context();
 9931   }
 9932   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9933   emit_int16(0x39, (0xC0 | encode));
 9934 }
 9935 
 9936 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9937   assert(VM_Version::supports_evex(), "");
 9938   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9939   InstructionMark im(this);
 9940   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9941   attributes.set_is_evex_instruction();
 9942   attributes.set_embedded_opmask_register_specifier(mask);
 9943   if (merge) {
 9944     attributes.reset_is_clear_context();
 9945   }
 9946   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9947   emit_int8(0x39);
 9948   emit_operand(dst, src);
 9949 }
 9950 
 9951 
 9952 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9953   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9954   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9955   attributes.set_is_evex_instruction();
 9956   attributes.set_embedded_opmask_register_specifier(mask);
 9957   if (merge) {
 9958     attributes.reset_is_clear_context();
 9959   }
 9960   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9961   emit_int16(0x3C, (0xC0 | encode));
 9962 }
 9963 
 9964 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9965   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9966   InstructionMark im(this);
 9967   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9968   attributes.set_is_evex_instruction();
 9969   attributes.set_embedded_opmask_register_specifier(mask);
 9970   if (merge) {
 9971     attributes.reset_is_clear_context();
 9972   }
 9973   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9974   emit_int8(0x3C);
 9975   emit_operand(dst, src);
 9976 }
 9977 
 9978 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9979   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9980   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9981   attributes.set_is_evex_instruction();
 9982   attributes.set_embedded_opmask_register_specifier(mask);
 9983   if (merge) {
 9984     attributes.reset_is_clear_context();
 9985   }
 9986   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9987   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 9988 }
 9989 
 9990 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9991   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9992   InstructionMark im(this);
 9993   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9994   attributes.set_is_evex_instruction();
 9995   attributes.set_embedded_opmask_register_specifier(mask);
 9996   if (merge) {
 9997     attributes.reset_is_clear_context();
 9998   }
 9999   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10000   emit_int8((unsigned char)0xEE);
10001   emit_operand(dst, src);
10002 }
10003 
10004 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10005   assert(VM_Version::supports_evex(), "");
10006   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10008   attributes.set_is_evex_instruction();
10009   attributes.set_embedded_opmask_register_specifier(mask);
10010   if (merge) {
10011     attributes.reset_is_clear_context();
10012   }
10013   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10014   emit_int16(0x3D, (0xC0 | encode));
10015 }
10016 
10017 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10018   assert(VM_Version::supports_evex(), "");
10019   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10020   InstructionMark im(this);
10021   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10022   attributes.set_is_evex_instruction();
10023   attributes.set_embedded_opmask_register_specifier(mask);
10024   if (merge) {
10025     attributes.reset_is_clear_context();
10026   }
10027   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10028   emit_int8(0x3D);
10029   emit_operand(dst, src);
10030 }
10031 
10032 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10033   assert(VM_Version::supports_evex(), "");
10034   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10035   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10036   attributes.set_is_evex_instruction();
10037   attributes.set_embedded_opmask_register_specifier(mask);
10038   if (merge) {
10039     attributes.reset_is_clear_context();
10040   }
10041   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10042   emit_int16(0x3D, (0xC0 | encode));
10043 }
10044 
10045 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10046   assert(VM_Version::supports_evex(), "");
10047   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10048   InstructionMark im(this);
10049   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10050   attributes.set_is_evex_instruction();
10051   attributes.set_embedded_opmask_register_specifier(mask);
10052   if (merge) {
10053     attributes.reset_is_clear_context();
10054   }
10055   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10056   emit_int8(0x3D);
10057   emit_operand(dst, src);
10058 }
10059 
10060 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10061   assert(VM_Version::supports_evex(), "requires EVEX support");
10062   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10063   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10064   attributes.set_is_evex_instruction();
10065   attributes.set_embedded_opmask_register_specifier(mask);
10066   if (merge) {
10067     attributes.reset_is_clear_context();
10068   }
10069   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10070   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10071 }
10072 
10073 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10074   assert(VM_Version::supports_evex(), "requires EVEX support");
10075   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10076   assert(dst != xnoreg, "sanity");
10077   InstructionMark im(this);
10078   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10079   attributes.set_is_evex_instruction();
10080   attributes.set_embedded_opmask_register_specifier(mask);
10081   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10082   if (merge) {
10083     attributes.reset_is_clear_context();
10084   }
10085   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10086   emit_int8(0x25);
10087   emit_operand(dst, src3);
10088   emit_int8(imm8);
10089 }
10090 
10091 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10092   assert(VM_Version::supports_evex(), "requires EVEX support");
10093   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10094   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10095   attributes.set_is_evex_instruction();
10096   attributes.set_embedded_opmask_register_specifier(mask);
10097   if (merge) {
10098     attributes.reset_is_clear_context();
10099   }
10100   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10101   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10102 }
10103 
10104 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10105   assert(VM_Version::supports_evex(), "requires EVEX support");
10106   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10107   assert(dst != xnoreg, "sanity");
10108   InstructionMark im(this);
10109   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10110   attributes.set_is_evex_instruction();
10111   attributes.set_embedded_opmask_register_specifier(mask);
10112   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10113   if (merge) {
10114     attributes.reset_is_clear_context();
10115   }
10116   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10117   emit_int8(0x25);
10118   emit_operand(dst, src3);
10119   emit_int8(imm8);
10120 }
10121 
10122 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10123   assert(VM_Version::supports_gfni(), "");
10124   assert(VM_Version::supports_sse(), "");
10125   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10126   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10127   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10128 }
10129 
10130 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10131   assert(VM_Version::supports_gfni(), "requires GFNI support");
10132   assert(VM_Version::supports_sse(), "");
10133   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10134   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10135   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10136 }
10137 
10138 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10139 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10140   assert(UseAVX >= 2, "");
10141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10142   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10143   emit_int16(0x58, (0xC0 | encode));
10144 }
10145 
10146 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10147   assert(VM_Version::supports_avx2(), "");
10148   assert(dst != xnoreg, "sanity");
10149   InstructionMark im(this);
10150   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10151   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10152   // swap src<->dst for encoding
10153   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10154   emit_int8(0x58);
10155   emit_operand(dst, src);
10156 }
10157 
10158 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10159 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10160   assert(VM_Version::supports_avx2(), "");
10161   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10162   attributes.set_rex_vex_w_reverted();
10163   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10164   emit_int16(0x59, (0xC0 | encode));
10165 }
10166 
10167 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10168   assert(VM_Version::supports_avx2(), "");
10169   assert(dst != xnoreg, "sanity");
10170   InstructionMark im(this);
10171   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10172   attributes.set_rex_vex_w_reverted();
10173   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10174   // swap src<->dst for encoding
10175   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10176   emit_int8(0x59);
10177   emit_operand(dst, src);
10178 }
10179 
10180 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10181   assert(vector_len != Assembler::AVX_128bit, "");
10182   assert(VM_Version::supports_evex(), "");
10183   assert(dst != xnoreg, "sanity");
10184   InstructionMark im(this);
10185   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10186   attributes.set_rex_vex_w_reverted();
10187   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10188   // swap src<->dst for encoding
10189   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10190   emit_int8(0x5A);
10191   emit_operand(dst, src);
10192 }
10193 
10194 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10195   assert(vector_len != Assembler::AVX_128bit, "");
10196   assert(VM_Version::supports_avx512dq(), "");
10197   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10198   attributes.set_rex_vex_w_reverted();
10199   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10200   emit_int16(0x5A, (0xC0 | encode));
10201 }
10202 
10203 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10204   assert(vector_len != Assembler::AVX_128bit, "");
10205   assert(VM_Version::supports_avx512dq(), "");
10206   assert(dst != xnoreg, "sanity");
10207   InstructionMark im(this);
10208   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10209   attributes.set_rex_vex_w_reverted();
10210   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10211   // swap src<->dst for encoding
10212   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10213   emit_int8(0x5A);
10214   emit_operand(dst, src);
10215 }
10216 
10217 // scalar single/double precision replicate
10218 
10219 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10220 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10221   assert(VM_Version::supports_avx2(), "");
10222   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10223   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10224   emit_int16(0x18, (0xC0 | encode));
10225 }
10226 
10227 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10228   assert(VM_Version::supports_avx(), "");
10229   assert(dst != xnoreg, "sanity");
10230   InstructionMark im(this);
10231   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10232   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10233   // swap src<->dst for encoding
10234   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10235   emit_int8(0x18);
10236   emit_operand(dst, src);
10237 }
10238 
10239 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10240 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10241   assert(VM_Version::supports_avx2(), "");
10242   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10243   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10244   attributes.set_rex_vex_w_reverted();
10245   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10246   emit_int16(0x19, (0xC0 | encode));
10247 }
10248 
10249 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10250   assert(VM_Version::supports_avx(), "");
10251   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10252   assert(dst != xnoreg, "sanity");
10253   InstructionMark im(this);
10254   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10255   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10256   attributes.set_rex_vex_w_reverted();
10257   // swap src<->dst for encoding
10258   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10259   emit_int8(0x19);
10260   emit_operand(dst, src);
10261 }
10262 
10263 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10264   assert(VM_Version::supports_avx(), "");
10265   assert(vector_len == AVX_256bit, "");
10266   assert(dst != xnoreg, "sanity");
10267   InstructionMark im(this);
10268   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10269   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10270   // swap src<->dst for encoding
10271   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10272   emit_int8(0x1A);
10273   emit_operand(dst, src);
10274 }
10275 
10276 // gpr source broadcast forms
10277 
10278 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10279 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10280   assert(VM_Version::supports_avx512bw(), "");
10281   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10282   attributes.set_is_evex_instruction();
10283   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10284   emit_int16(0x7A, (0xC0 | encode));
10285 }
10286 
10287 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10288 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10289   assert(VM_Version::supports_avx512bw(), "");
10290   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10291   attributes.set_is_evex_instruction();
10292   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10293   emit_int16(0x7B, (0xC0 | encode));
10294 }
10295 
10296 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10297 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10298   assert(VM_Version::supports_evex(), "");
10299   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10300   attributes.set_is_evex_instruction();
10301   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10302   emit_int16(0x7C, (0xC0 | encode));
10303 }
10304 
10305 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10306 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10307   assert(VM_Version::supports_evex(), "");
10308   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10309   attributes.set_is_evex_instruction();
10310   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10311   emit_int16(0x7C, (0xC0 | encode));
10312 }
10313 
10314 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10315   assert(VM_Version::supports_avx2(), "");
10316   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10317   assert(dst != xnoreg, "sanity");
10318   assert(src.isxmmindex(),"expected to be xmm index");
10319   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10320   InstructionMark im(this);
10321   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10322   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10323   emit_int8((unsigned char)0x90);
10324   emit_operand(dst, src);
10325 }
10326 
10327 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10328   assert(VM_Version::supports_avx2(), "");
10329   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10330   assert(dst != xnoreg, "sanity");
10331   assert(src.isxmmindex(),"expected to be xmm index");
10332   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10333   InstructionMark im(this);
10334   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10335   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10336   emit_int8((unsigned char)0x90);
10337   emit_operand(dst, src);
10338 }
10339 
10340 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10341   assert(VM_Version::supports_avx2(), "");
10342   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10343   assert(dst != xnoreg, "sanity");
10344   assert(src.isxmmindex(),"expected to be xmm index");
10345   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10346   InstructionMark im(this);
10347   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10348   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10349   emit_int8((unsigned char)0x92);
10350   emit_operand(dst, src);
10351 }
10352 
10353 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10354   assert(VM_Version::supports_avx2(), "");
10355   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10356   assert(dst != xnoreg, "sanity");
10357   assert(src.isxmmindex(),"expected to be xmm index");
10358   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10359   InstructionMark im(this);
10360   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10361   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10362   emit_int8((unsigned char)0x92);
10363   emit_operand(dst, src);
10364 }
10365 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10366   assert(VM_Version::supports_evex(), "");
10367   assert(dst != xnoreg, "sanity");
10368   assert(src.isxmmindex(),"expected to be xmm index");
10369   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10370   assert(mask != k0, "instruction will #UD if mask is in k0");
10371   InstructionMark im(this);
10372   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10373   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10374   attributes.reset_is_clear_context();
10375   attributes.set_embedded_opmask_register_specifier(mask);
10376   attributes.set_is_evex_instruction();
10377   // swap src<->dst for encoding
10378   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10379   emit_int8((unsigned char)0x90);
10380   emit_operand(dst, src);
10381 }
10382 
10383 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10384   assert(VM_Version::supports_evex(), "");
10385   assert(dst != xnoreg, "sanity");
10386   assert(src.isxmmindex(),"expected to be xmm index");
10387   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10388   assert(mask != k0, "instruction will #UD if mask is in k0");
10389   InstructionMark im(this);
10390   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10391   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10392   attributes.reset_is_clear_context();
10393   attributes.set_embedded_opmask_register_specifier(mask);
10394   attributes.set_is_evex_instruction();
10395   // swap src<->dst for encoding
10396   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10397   emit_int8((unsigned char)0x90);
10398   emit_operand(dst, src);
10399 }
10400 
10401 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10402   assert(VM_Version::supports_evex(), "");
10403   assert(dst != xnoreg, "sanity");
10404   assert(src.isxmmindex(),"expected to be xmm index");
10405   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10406   assert(mask != k0, "instruction will #UD if mask is in k0");
10407   InstructionMark im(this);
10408   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10409   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10410   attributes.reset_is_clear_context();
10411   attributes.set_embedded_opmask_register_specifier(mask);
10412   attributes.set_is_evex_instruction();
10413   // swap src<->dst for encoding
10414   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10415   emit_int8((unsigned char)0x92);
10416   emit_operand(dst, src);
10417 }
10418 
10419 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10420   assert(VM_Version::supports_evex(), "");
10421   assert(dst != xnoreg, "sanity");
10422   assert(src.isxmmindex(),"expected to be xmm index");
10423   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10424   assert(mask != k0, "instruction will #UD if mask is in k0");
10425   InstructionMark im(this);
10426   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10427   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10428   attributes.reset_is_clear_context();
10429   attributes.set_embedded_opmask_register_specifier(mask);
10430   attributes.set_is_evex_instruction();
10431   // swap src<->dst for encoding
10432   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10433   emit_int8((unsigned char)0x92);
10434   emit_operand(dst, src);
10435 }
10436 
10437 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10438   assert(VM_Version::supports_evex(), "");
10439   assert(mask != k0, "instruction will #UD if mask is in k0");
10440   InstructionMark im(this);
10441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10442   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10443   attributes.reset_is_clear_context();
10444   attributes.set_embedded_opmask_register_specifier(mask);
10445   attributes.set_is_evex_instruction();
10446   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10447   emit_int8((unsigned char)0xA0);
10448   emit_operand(src, dst);
10449 }
10450 
10451 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10452   assert(VM_Version::supports_evex(), "");
10453   assert(mask != k0, "instruction will #UD if mask is in k0");
10454   InstructionMark im(this);
10455   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10456   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10457   attributes.reset_is_clear_context();
10458   attributes.set_embedded_opmask_register_specifier(mask);
10459   attributes.set_is_evex_instruction();
10460   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10461   emit_int8((unsigned char)0xA0);
10462   emit_operand(src, dst);
10463 }
10464 
10465 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10466   assert(VM_Version::supports_evex(), "");
10467   assert(mask != k0, "instruction will #UD if mask is in k0");
10468   InstructionMark im(this);
10469   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10470   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10471   attributes.reset_is_clear_context();
10472   attributes.set_embedded_opmask_register_specifier(mask);
10473   attributes.set_is_evex_instruction();
10474   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10475   emit_int8((unsigned char)0xA2);
10476   emit_operand(src, dst);
10477 }
10478 
10479 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10480   assert(VM_Version::supports_evex(), "");
10481   assert(mask != k0, "instruction will #UD if mask is in k0");
10482   InstructionMark im(this);
10483   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10484   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10485   attributes.reset_is_clear_context();
10486   attributes.set_embedded_opmask_register_specifier(mask);
10487   attributes.set_is_evex_instruction();
10488   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10489   emit_int8((unsigned char)0xA2);
10490   emit_operand(src, dst);
10491 }
10492 // Carry-Less Multiplication Quadword
10493 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10494   assert(VM_Version::supports_clmul(), "");
10495   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10496   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10497   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10498 }
10499 
10500 // Carry-Less Multiplication Quadword
10501 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10502   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10503   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10504   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10505   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10506 }
10507 
10508 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10509   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10510   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10511   attributes.set_is_evex_instruction();
10512   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10513   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10514 }
10515 
10516 void Assembler::vzeroupper_uncached() {
10517   if (VM_Version::supports_vzeroupper()) {
10518     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10519     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10520     emit_int8(0x77);
10521   }
10522 }
10523 
10524 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10525   // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10526   assert(VM_Version::supports_evex(), "");
10527   assert(VM_Version::supports_avx512dq(), "");
10528   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10529   attributes.set_is_evex_instruction();
10530   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10531   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10532 }
10533 
10534 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10535   // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10536   assert(VM_Version::supports_evex(), "");
10537   assert(VM_Version::supports_avx512dq(), "");
10538   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10539   attributes.set_is_evex_instruction();
10540   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10541   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10542 }
10543 
10544 void Assembler::fld_x(Address adr) {
10545   InstructionMark im(this);
10546   emit_int8((unsigned char)0xDB);
10547   emit_operand32(rbp, adr);
10548 }
10549 
10550 void Assembler::fstp_x(Address adr) {
10551   InstructionMark im(this);
10552   emit_int8((unsigned char)0xDB);
10553   emit_operand32(rdi, adr);
10554 }
10555 
10556 void Assembler::emit_operand32(Register reg, Address adr) {
10557   assert(reg->encoding() < 8, "no extended registers");
10558   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10559   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
10560                adr._rspec);
10561 }
10562 
10563 #ifndef _LP64
10564 // 32bit only pieces of the assembler
10565 
10566 void Assembler::emms() {
10567   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10568   emit_int16(0x0F, 0x77);
10569 }
10570 
10571 void Assembler::vzeroupper() {
10572   vzeroupper_uncached();
10573 }
10574 
10575 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10576   // NO PREFIX AS NEVER 64BIT
10577   InstructionMark im(this);
10578   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10579   emit_data(imm32, rspec, 0);
10580 }
10581 
10582 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10583   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10584   InstructionMark im(this);
10585   emit_int8((unsigned char)0x81);
10586   emit_operand(rdi, src1);
10587   emit_data(imm32, rspec, 0);
10588 }
10589 
10590 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10591 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10592 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10593 void Assembler::cmpxchg8(Address adr) {
10594   InstructionMark im(this);
10595   emit_int16(0x0F, (unsigned char)0xC7);
10596   emit_operand(rcx, adr);
10597 }
10598 
10599 void Assembler::decl(Register dst) {
10600   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10601  emit_int8(0x48 | dst->encoding());
10602 }
10603 
10604 // 64bit doesn't use the x87
10605 
10606 void Assembler::emit_farith(int b1, int b2, int i) {
10607   assert(isByte(b1) && isByte(b2), "wrong opcode");
10608   assert(0 <= i &&  i < 8, "illegal stack offset");
10609   emit_int16(b1, b2 + i);
10610 }
10611 
10612 void Assembler::fabs() {
10613   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10614 }
10615 
10616 void Assembler::fadd(int i) {
10617   emit_farith(0xD8, 0xC0, i);
10618 }
10619 
10620 void Assembler::fadd_d(Address src) {
10621   InstructionMark im(this);
10622   emit_int8((unsigned char)0xDC);
10623   emit_operand32(rax, src);
10624 }
10625 
10626 void Assembler::fadd_s(Address src) {
10627   InstructionMark im(this);
10628   emit_int8((unsigned char)0xD8);
10629   emit_operand32(rax, src);
10630 }
10631 
10632 void Assembler::fadda(int i) {
10633   emit_farith(0xDC, 0xC0, i);
10634 }
10635 
10636 void Assembler::faddp(int i) {
10637   emit_farith(0xDE, 0xC0, i);
10638 }
10639 
10640 void Assembler::fchs() {
10641   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10642 }
10643 
10644 void Assembler::fcom(int i) {
10645   emit_farith(0xD8, 0xD0, i);
10646 }
10647 
10648 void Assembler::fcomp(int i) {
10649   emit_farith(0xD8, 0xD8, i);
10650 }
10651 
10652 void Assembler::fcomp_d(Address src) {
10653   InstructionMark im(this);
10654   emit_int8((unsigned char)0xDC);
10655   emit_operand32(rbx, src);
10656 }
10657 
10658 void Assembler::fcomp_s(Address src) {
10659   InstructionMark im(this);
10660   emit_int8((unsigned char)0xD8);
10661   emit_operand32(rbx, src);
10662 }
10663 
10664 void Assembler::fcompp() {
10665   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10666 }
10667 
10668 void Assembler::fcos() {
10669   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10670 }
10671 
10672 void Assembler::fdecstp() {
10673   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10674 }
10675 
10676 void Assembler::fdiv(int i) {
10677   emit_farith(0xD8, 0xF0, i);
10678 }
10679 
10680 void Assembler::fdiv_d(Address src) {
10681   InstructionMark im(this);
10682   emit_int8((unsigned char)0xDC);
10683   emit_operand32(rsi, src);
10684 }
10685 
10686 void Assembler::fdiv_s(Address src) {
10687   InstructionMark im(this);
10688   emit_int8((unsigned char)0xD8);
10689   emit_operand32(rsi, src);
10690 }
10691 
10692 void Assembler::fdiva(int i) {
10693   emit_farith(0xDC, 0xF8, i);
10694 }
10695 
10696 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10697 //       is erroneous for some of the floating-point instructions below.
10698 
10699 void Assembler::fdivp(int i) {
10700   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10701 }
10702 
10703 void Assembler::fdivr(int i) {
10704   emit_farith(0xD8, 0xF8, i);
10705 }
10706 
10707 void Assembler::fdivr_d(Address src) {
10708   InstructionMark im(this);
10709   emit_int8((unsigned char)0xDC);
10710   emit_operand32(rdi, src);
10711 }
10712 
10713 void Assembler::fdivr_s(Address src) {
10714   InstructionMark im(this);
10715   emit_int8((unsigned char)0xD8);
10716   emit_operand32(rdi, src);
10717 }
10718 
10719 void Assembler::fdivra(int i) {
10720   emit_farith(0xDC, 0xF0, i);
10721 }
10722 
10723 void Assembler::fdivrp(int i) {
10724   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10725 }
10726 
10727 void Assembler::ffree(int i) {
10728   emit_farith(0xDD, 0xC0, i);
10729 }
10730 
10731 void Assembler::fild_d(Address adr) {
10732   InstructionMark im(this);
10733   emit_int8((unsigned char)0xDF);
10734   emit_operand32(rbp, adr);
10735 }
10736 
10737 void Assembler::fild_s(Address adr) {
10738   InstructionMark im(this);
10739   emit_int8((unsigned char)0xDB);
10740   emit_operand32(rax, adr);
10741 }
10742 
10743 void Assembler::fincstp() {
10744   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10745 }
10746 
10747 void Assembler::finit() {
10748   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10749 }
10750 
10751 void Assembler::fist_s(Address adr) {
10752   InstructionMark im(this);
10753   emit_int8((unsigned char)0xDB);
10754   emit_operand32(rdx, adr);
10755 }
10756 
10757 void Assembler::fistp_d(Address adr) {
10758   InstructionMark im(this);
10759   emit_int8((unsigned char)0xDF);
10760   emit_operand32(rdi, adr);
10761 }
10762 
10763 void Assembler::fistp_s(Address adr) {
10764   InstructionMark im(this);
10765   emit_int8((unsigned char)0xDB);
10766   emit_operand32(rbx, adr);
10767 }
10768 
10769 void Assembler::fld1() {
10770   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10771 }
10772 
10773 void Assembler::fld_d(Address adr) {
10774   InstructionMark im(this);
10775   emit_int8((unsigned char)0xDD);
10776   emit_operand32(rax, adr);
10777 }
10778 
10779 void Assembler::fld_s(Address adr) {
10780   InstructionMark im(this);
10781   emit_int8((unsigned char)0xD9);
10782   emit_operand32(rax, adr);
10783 }
10784 
10785 
10786 void Assembler::fld_s(int index) {
10787   emit_farith(0xD9, 0xC0, index);
10788 }
10789 
10790 void Assembler::fldcw(Address src) {
10791   InstructionMark im(this);
10792   emit_int8((unsigned char)0xD9);
10793   emit_operand32(rbp, src);
10794 }
10795 
10796 void Assembler::fldenv(Address src) {
10797   InstructionMark im(this);
10798   emit_int8((unsigned char)0xD9);
10799   emit_operand32(rsp, src);
10800 }
10801 
10802 void Assembler::fldlg2() {
10803   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10804 }
10805 
10806 void Assembler::fldln2() {
10807   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10808 }
10809 
10810 void Assembler::fldz() {
10811   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10812 }
10813 
10814 void Assembler::flog() {
10815   fldln2();
10816   fxch();
10817   fyl2x();
10818 }
10819 
10820 void Assembler::flog10() {
10821   fldlg2();
10822   fxch();
10823   fyl2x();
10824 }
10825 
10826 void Assembler::fmul(int i) {
10827   emit_farith(0xD8, 0xC8, i);
10828 }
10829 
10830 void Assembler::fmul_d(Address src) {
10831   InstructionMark im(this);
10832   emit_int8((unsigned char)0xDC);
10833   emit_operand32(rcx, src);
10834 }
10835 
10836 void Assembler::fmul_s(Address src) {
10837   InstructionMark im(this);
10838   emit_int8((unsigned char)0xD8);
10839   emit_operand32(rcx, src);
10840 }
10841 
10842 void Assembler::fmula(int i) {
10843   emit_farith(0xDC, 0xC8, i);
10844 }
10845 
10846 void Assembler::fmulp(int i) {
10847   emit_farith(0xDE, 0xC8, i);
10848 }
10849 
10850 void Assembler::fnsave(Address dst) {
10851   InstructionMark im(this);
10852   emit_int8((unsigned char)0xDD);
10853   emit_operand32(rsi, dst);
10854 }
10855 
10856 void Assembler::fnstcw(Address src) {
10857   InstructionMark im(this);
10858   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10859   emit_operand32(rdi, src);
10860 }
10861 
10862 void Assembler::fnstsw_ax() {
10863   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10864 }
10865 
10866 void Assembler::fprem() {
10867   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10868 }
10869 
10870 void Assembler::fprem1() {
10871   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10872 }
10873 
10874 void Assembler::frstor(Address src) {
10875   InstructionMark im(this);
10876   emit_int8((unsigned char)0xDD);
10877   emit_operand32(rsp, src);
10878 }
10879 
10880 void Assembler::fsin() {
10881   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10882 }
10883 
10884 void Assembler::fsqrt() {
10885   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
10886 }
10887 
10888 void Assembler::fst_d(Address adr) {
10889   InstructionMark im(this);
10890   emit_int8((unsigned char)0xDD);
10891   emit_operand32(rdx, adr);
10892 }
10893 
10894 void Assembler::fst_s(Address adr) {
10895   InstructionMark im(this);
10896   emit_int8((unsigned char)0xD9);
10897   emit_operand32(rdx, adr);
10898 }
10899 
10900 void Assembler::fstp_d(Address adr) {
10901   InstructionMark im(this);
10902   emit_int8((unsigned char)0xDD);
10903   emit_operand32(rbx, adr);
10904 }
10905 
10906 void Assembler::fstp_d(int index) {
10907   emit_farith(0xDD, 0xD8, index);
10908 }
10909 
10910 void Assembler::fstp_s(Address adr) {
10911   InstructionMark im(this);
10912   emit_int8((unsigned char)0xD9);
10913   emit_operand32(rbx, adr);
10914 }
10915 
10916 void Assembler::fsub(int i) {
10917   emit_farith(0xD8, 0xE0, i);
10918 }
10919 
10920 void Assembler::fsub_d(Address src) {
10921   InstructionMark im(this);
10922   emit_int8((unsigned char)0xDC);
10923   emit_operand32(rsp, src);
10924 }
10925 
10926 void Assembler::fsub_s(Address src) {
10927   InstructionMark im(this);
10928   emit_int8((unsigned char)0xD8);
10929   emit_operand32(rsp, src);
10930 }
10931 
10932 void Assembler::fsuba(int i) {
10933   emit_farith(0xDC, 0xE8, i);
10934 }
10935 
10936 void Assembler::fsubp(int i) {
10937   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
10938 }
10939 
10940 void Assembler::fsubr(int i) {
10941   emit_farith(0xD8, 0xE8, i);
10942 }
10943 
10944 void Assembler::fsubr_d(Address src) {
10945   InstructionMark im(this);
10946   emit_int8((unsigned char)0xDC);
10947   emit_operand32(rbp, src);
10948 }
10949 
10950 void Assembler::fsubr_s(Address src) {
10951   InstructionMark im(this);
10952   emit_int8((unsigned char)0xD8);
10953   emit_operand32(rbp, src);
10954 }
10955 
10956 void Assembler::fsubra(int i) {
10957   emit_farith(0xDC, 0xE0, i);
10958 }
10959 
10960 void Assembler::fsubrp(int i) {
10961   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
10962 }
10963 
10964 void Assembler::ftan() {
10965   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
10966 }
10967 
10968 void Assembler::ftst() {
10969   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
10970 }
10971 
10972 void Assembler::fucomi(int i) {
10973   // make sure the instruction is supported (introduced for P6, together with cmov)
10974   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10975   emit_farith(0xDB, 0xE8, i);
10976 }
10977 
10978 void Assembler::fucomip(int i) {
10979   // make sure the instruction is supported (introduced for P6, together with cmov)
10980   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10981   emit_farith(0xDF, 0xE8, i);
10982 }
10983 
10984 void Assembler::fwait() {
10985   emit_int8((unsigned char)0x9B);
10986 }
10987 
10988 void Assembler::fxch(int i) {
10989   emit_farith(0xD9, 0xC8, i);
10990 }
10991 
10992 void Assembler::fyl2x() {
10993   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
10994 }
10995 
10996 void Assembler::frndint() {
10997   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
10998 }
10999 
11000 void Assembler::f2xm1() {
11001   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11002 }
11003 
11004 void Assembler::fldl2e() {
11005   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11006 }
11007 #endif // !_LP64
11008 
11009 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11010 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11011 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11012 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
11013 
11014 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11015 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11016   if (pre > 0) {
11017     emit_int8(simd_pre[pre]);
11018   }
11019   if (rex_w) {
11020     prefixq(adr, xreg);
11021   } else {
11022     prefix(adr, xreg);
11023   }
11024   if (opc > 0) {
11025     emit_int8(0x0F);
11026     int opc2 = simd_opc[opc];
11027     if (opc2 > 0) {
11028       emit_int8(opc2);
11029     }
11030   }
11031 }
11032 
11033 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11034   if (pre > 0) {
11035     emit_int8(simd_pre[pre]);
11036   }
11037   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11038   if (opc > 0) {
11039     emit_int8(0x0F);
11040     int opc2 = simd_opc[opc];
11041     if (opc2 > 0) {
11042       emit_int8(opc2);
11043     }
11044   }
11045   return encode;
11046 }
11047 
11048 
11049 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11050   int vector_len = _attributes->get_vector_len();
11051   bool vex_w = _attributes->is_rex_vex_w();
11052   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11053     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11054     byte1 = (~byte1) & 0xE0;
11055     byte1 |= opc;
11056 
11057     int byte2 = ((~nds_enc) & 0xf) << 3;
11058     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11059 
11060     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11061   } else {
11062     int byte1 = vex_r ? VEX_R : 0;
11063     byte1 = (~byte1) & 0x80;
11064     byte1 |= ((~nds_enc) & 0xf) << 3;
11065     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11066     emit_int16((unsigned char)VEX_2bytes, byte1);
11067   }
11068 }
11069 
11070 // This is a 4 byte encoding
11071 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11072   // EVEX 0x62 prefix
11073   // byte1 = EVEX_4bytes;
11074 
11075   bool vex_w = _attributes->is_rex_vex_w();
11076   int evex_encoding = (vex_w ? VEX_W : 0);
11077   // EVEX.b is not currently used for broadcast of single element or data rounding modes
11078   _attributes->set_evex_encoding(evex_encoding);
11079 
11080   // P0: byte 2, initialized to RXBR`00mm
11081   // instead of not'd
11082   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11083   byte2 = (~byte2) & 0xF0;
11084   // confine opc opcode extensions in mm bits to lower two bits
11085   // of form {0F, 0F_38, 0F_3A}
11086   byte2 |= opc;
11087 
11088   // P1: byte 3 as Wvvvv1pp
11089   int byte3 = ((~nds_enc) & 0xf) << 3;
11090   // p[10] is always 1
11091   byte3 |= EVEX_F;
11092   byte3 |= (vex_w & 1) << 7;
11093   // confine pre opcode extensions in pp bits to lower two bits
11094   // of form {66, F3, F2}
11095   byte3 |= pre;
11096 
11097   // P2: byte 4 as zL'Lbv'aaa
11098   // kregs are implemented in the low 3 bits as aaa
11099   int byte4 = (_attributes->is_no_reg_mask()) ?
11100               0 :
11101               _attributes->get_embedded_opmask_register_specifier();
11102   // EVEX.v` for extending EVEX.vvvv or VIDX
11103   byte4 |= (evex_v ? 0: EVEX_V);
11104   // third EXEC.b for broadcast actions
11105   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11106   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11107   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11108   // last is EVEX.z for zero/merge actions
11109   if (_attributes->is_no_reg_mask() == false &&
11110       _attributes->get_embedded_opmask_register_specifier() != 0) {
11111     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11112   }
11113 
11114   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11115 }
11116 
11117 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11118   bool vex_r = (xreg_enc & 8) == 8;
11119   bool vex_b = adr.base_needs_rex();
11120   bool vex_x;
11121   if (adr.isxmmindex()) {
11122     vex_x = adr.xmmindex_needs_rex();
11123   } else {
11124     vex_x = adr.index_needs_rex();
11125   }
11126   set_attributes(attributes);
11127   attributes->set_current_assembler(this);
11128 
11129   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11130   // is allowed in legacy mode and has resources which will fit in it.
11131   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11132   if (!attributes->is_legacy_mode()) {
11133     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11134       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11135           attributes->set_is_legacy_mode();
11136       }
11137     }
11138   }
11139 
11140   if (UseAVX > 2) {
11141     assert(((!attributes->uses_vl()) ||
11142             (attributes->get_vector_len() == AVX_512bit) ||
11143             (!_legacy_mode_vl) ||
11144             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11145     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11146   }
11147 
11148   clear_managed();
11149   if (UseAVX > 2 && !attributes->is_legacy_mode())
11150   {
11151     bool evex_r = (xreg_enc >= 16);
11152     bool evex_v;
11153     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11154     if (adr.isxmmindex())  {
11155       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11156     } else {
11157       evex_v = (nds_enc >= 16);
11158     }
11159     attributes->set_is_evex_instruction();
11160     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11161   } else {
11162     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11163       attributes->set_rex_vex_w(false);
11164     }
11165     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11166   }
11167 }
11168 
11169 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11170   bool vex_r = (dst_enc & 8) == 8;
11171   bool vex_b = (src_enc & 8) == 8;
11172   bool vex_x = false;
11173   set_attributes(attributes);
11174   attributes->set_current_assembler(this);
11175 
11176   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11177   // is allowed in legacy mode and has resources which will fit in it.
11178   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11179   if (!attributes->is_legacy_mode()) {
11180     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11181       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11182           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11183           attributes->set_is_legacy_mode();
11184       }
11185     }
11186   }
11187 
11188   if (UseAVX > 2) {
11189     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11190     // Instruction with uses_vl true are vector instructions
11191     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11192     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11193     // Rest all should have legacy_mode set as true
11194     assert(((!attributes->uses_vl()) ||
11195             (attributes->get_vector_len() == AVX_512bit) ||
11196             (!_legacy_mode_vl) ||
11197             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11198     // Instruction with legacy_mode true should have dst, nds and src < 15
11199     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11200   }
11201 
11202   clear_managed();
11203   if (UseAVX > 2 && !attributes->is_legacy_mode())
11204   {
11205     bool evex_r = (dst_enc >= 16);
11206     bool evex_v = (nds_enc >= 16);
11207     // can use vex_x as bank extender on rm encoding
11208     vex_x = (src_enc >= 16);
11209     attributes->set_is_evex_instruction();
11210     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11211   } else {
11212     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11213       attributes->set_rex_vex_w(false);
11214     }
11215     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11216   }
11217 
11218   // return modrm byte components for operands
11219   return (((dst_enc & 7) << 3) | (src_enc & 7));
11220 }
11221 
11222 
11223 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11224                             VexOpcode opc, InstructionAttr *attributes) {
11225   if (UseAVX > 0) {
11226     int xreg_enc = xreg->encoding();
11227     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11228     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11229   } else {
11230     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11231     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11232   }
11233 }
11234 
11235 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11236                                       VexOpcode opc, InstructionAttr *attributes) {
11237   int dst_enc = dst->encoding();
11238   int src_enc = src->encoding();
11239   if (UseAVX > 0) {
11240     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11241     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11242   } else {
11243     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11244     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11245   }
11246 }
11247 
11248 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11249   assert(VM_Version::supports_avx(), "");
11250   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11251   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11252   emit_int16(0x5F, (0xC0 | encode));
11253 }
11254 
11255 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11256   assert(VM_Version::supports_avx(), "");
11257   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11258   attributes.set_rex_vex_w_reverted();
11259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11260   emit_int16(0x5F, (0xC0 | encode));
11261 }
11262 
11263 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11264   assert(VM_Version::supports_avx(), "");
11265   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11266   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11267   emit_int16(0x5D, (0xC0 | encode));
11268 }
11269 
11270 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11271   assert(VM_Version::supports_avx(), "");
11272   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11273   attributes.set_rex_vex_w_reverted();
11274   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11275   emit_int16(0x5D, (0xC0 | encode));
11276 }
11277 
11278 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11279   assert(VM_Version::supports_avx(), "");
11280   assert(vector_len <= AVX_256bit, "");
11281   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11282   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11283   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11284 }
11285 
11286 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11287   assert(VM_Version::supports_avx(), "");
11288   assert(vector_len <= AVX_256bit, "");
11289   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11290   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11291   int src2_enc = src2->encoding();
11292   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11293 }
11294 
11295 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11296   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11297   assert(vector_len <= AVX_256bit, "");
11298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11299   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11300   int src2_enc = src2->encoding();
11301   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11302 }
11303 
11304 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11305   assert(VM_Version::supports_avx2(), "");
11306   assert(vector_len <= AVX_256bit, "");
11307   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11308   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11309   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11310 }
11311 
11312 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11313   assert(VM_Version::supports_avx(), "");
11314   assert(vector_len <= AVX_256bit, "");
11315   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11316   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11317   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11318 }
11319 
11320 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11321                         ComparisonPredicateFP comparison, int vector_len) {
11322   assert(VM_Version::supports_evex(), "");
11323   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11324   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11325   attributes.set_is_evex_instruction();
11326   attributes.set_embedded_opmask_register_specifier(mask);
11327   attributes.reset_is_clear_context();
11328   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11329   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11330 }
11331 
11332 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11333                         ComparisonPredicateFP comparison, int vector_len) {
11334   assert(VM_Version::supports_evex(), "");
11335   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11336   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11337   attributes.set_is_evex_instruction();
11338   attributes.set_embedded_opmask_register_specifier(mask);
11339   attributes.reset_is_clear_context();
11340   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11341   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11342 }
11343 
11344 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11345   assert(VM_Version::supports_sse4_1(), "");
11346   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11347   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11348   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11349   emit_int16(0x14, (0xC0 | encode));
11350 }
11351 
11352 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11353   assert(VM_Version::supports_sse4_1(), "");
11354   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11355   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11356   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11357   emit_int16(0x15, (0xC0 | encode));
11358 }
11359 
11360 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11361   assert(VM_Version::supports_sse4_1(), "");
11362   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11363   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11364   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11365   emit_int16(0x10, (0xC0 | encode));
11366 }
11367 
11368 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11369   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11370   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11371   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11372   int src2_enc = src2->encoding();
11373   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11374 }
11375 
11376 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11377   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11378   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11379   emit_int24(0x0C, (0xC0 | encode), imm8);
11380 }
11381 
11382 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11383   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11384   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11385   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11386   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11387   emit_int16(0x64, (0xC0 | encode));
11388 }
11389 
11390 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11391   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11392   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11393   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11395   emit_int16(0x65, (0xC0 | encode));
11396 }
11397 
11398 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11399   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11400   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11401   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11402   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11403   emit_int16(0x66, (0xC0 | encode));
11404 }
11405 
11406 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11407   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11408   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11409   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11410   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11411   emit_int16(0x37, (0xC0 | encode));
11412 }
11413 
11414 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11415                         int comparison, bool is_signed, int vector_len) {
11416   assert(VM_Version::supports_evex(), "");
11417   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11418   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11419   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11420   attributes.set_is_evex_instruction();
11421   attributes.set_embedded_opmask_register_specifier(mask);
11422   attributes.reset_is_clear_context();
11423   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11424   int opcode = is_signed ? 0x1F : 0x1E;
11425   emit_int24(opcode, (0xC0 | encode), comparison);
11426 }
11427 
11428 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11429                         int comparison, bool is_signed, int vector_len) {
11430   assert(VM_Version::supports_evex(), "");
11431   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11432   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11433   InstructionMark im(this);
11434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11435   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11436   attributes.set_is_evex_instruction();
11437   attributes.set_embedded_opmask_register_specifier(mask);
11438   attributes.reset_is_clear_context();
11439   int dst_enc = kdst->encoding();
11440   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11441   int opcode = is_signed ? 0x1F : 0x1E;
11442   emit_int8((unsigned char)opcode);
11443   emit_operand(as_Register(dst_enc), src);
11444   emit_int8((unsigned char)comparison);
11445 }
11446 
11447 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11448                         int comparison, bool is_signed, int vector_len) {
11449   assert(VM_Version::supports_evex(), "");
11450   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11451   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11452   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11453   attributes.set_is_evex_instruction();
11454   attributes.set_embedded_opmask_register_specifier(mask);
11455   attributes.reset_is_clear_context();
11456   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11457   int opcode = is_signed ? 0x1F : 0x1E;
11458   emit_int24(opcode, (0xC0 | encode), comparison);
11459 }
11460 
11461 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11462                         int comparison, bool is_signed, int vector_len) {
11463   assert(VM_Version::supports_evex(), "");
11464   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11465   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11466   InstructionMark im(this);
11467   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11468   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11469   attributes.set_is_evex_instruction();
11470   attributes.set_embedded_opmask_register_specifier(mask);
11471   attributes.reset_is_clear_context();
11472   int dst_enc = kdst->encoding();
11473   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11474   int opcode = is_signed ? 0x1F : 0x1E;
11475   emit_int8((unsigned char)opcode);
11476   emit_operand(as_Register(dst_enc), src);
11477   emit_int8((unsigned char)comparison);
11478 }
11479 
11480 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11481                         int comparison, bool is_signed, int vector_len) {
11482   assert(VM_Version::supports_evex(), "");
11483   assert(VM_Version::supports_avx512bw(), "");
11484   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11485   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11486   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11487   attributes.set_is_evex_instruction();
11488   attributes.set_embedded_opmask_register_specifier(mask);
11489   attributes.reset_is_clear_context();
11490   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11491   int opcode = is_signed ? 0x3F : 0x3E;
11492   emit_int24(opcode, (0xC0 | encode), comparison);
11493 }
11494 
11495 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11496                         int comparison, bool is_signed, int vector_len) {
11497   assert(VM_Version::supports_evex(), "");
11498   assert(VM_Version::supports_avx512bw(), "");
11499   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11500   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11501   InstructionMark im(this);
11502   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11503   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11504   attributes.set_is_evex_instruction();
11505   attributes.set_embedded_opmask_register_specifier(mask);
11506   attributes.reset_is_clear_context();
11507   int dst_enc = kdst->encoding();
11508   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11509   int opcode = is_signed ? 0x3F : 0x3E;
11510   emit_int8((unsigned char)opcode);
11511   emit_operand(as_Register(dst_enc), src);
11512   emit_int8((unsigned char)comparison);
11513 }
11514 
11515 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11516                         int comparison, bool is_signed, int vector_len) {
11517   assert(VM_Version::supports_evex(), "");
11518   assert(VM_Version::supports_avx512bw(), "");
11519   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11520   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11521   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11522   attributes.set_is_evex_instruction();
11523   attributes.set_embedded_opmask_register_specifier(mask);
11524   attributes.reset_is_clear_context();
11525   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11526   int opcode = is_signed ? 0x3F : 0x3E;
11527   emit_int24(opcode, (0xC0 | encode), comparison);
11528 }
11529 
11530 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11531                         int comparison, bool is_signed, int vector_len) {
11532   assert(VM_Version::supports_evex(), "");
11533   assert(VM_Version::supports_avx512bw(), "");
11534   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11535   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11536   InstructionMark im(this);
11537   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11538   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11539   attributes.set_is_evex_instruction();
11540   attributes.set_embedded_opmask_register_specifier(mask);
11541   attributes.reset_is_clear_context();
11542   int dst_enc = kdst->encoding();
11543   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11544   int opcode = is_signed ? 0x3F : 0x3E;
11545   emit_int8((unsigned char)opcode);
11546   emit_operand(as_Register(dst_enc), src);
11547   emit_int8((unsigned char)comparison);
11548 }
11549 
11550 // Register is a class, but it would be assigned numerical value.
11551 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
11552 PRAGMA_DIAG_PUSH
11553 PRAGMA_NONNULL_IGNORED
11554 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11555   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11557   attributes.set_is_evex_instruction();
11558   attributes.set_embedded_opmask_register_specifier(mask);
11559   if (merge) {
11560     attributes.reset_is_clear_context();
11561   }
11562   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11563   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11564 }
11565 
11566 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11567   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11568   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11569   attributes.set_is_evex_instruction();
11570   attributes.set_embedded_opmask_register_specifier(mask);
11571   if (merge) {
11572     attributes.reset_is_clear_context();
11573   }
11574   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11575   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11576 }
11577 PRAGMA_DIAG_POP
11578 
11579 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11580   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11581   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11582   attributes.set_is_evex_instruction();
11583   attributes.set_embedded_opmask_register_specifier(mask);
11584   if (merge) {
11585     attributes.reset_is_clear_context();
11586   }
11587   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11588   emit_int16(0x14, (0xC0 | encode));
11589 }
11590 
11591 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11592   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11593   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11594   attributes.set_is_evex_instruction();
11595   attributes.set_embedded_opmask_register_specifier(mask);
11596   if (merge) {
11597     attributes.reset_is_clear_context();
11598   }
11599   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11600   emit_int16(0x14, (0xC0 | encode));
11601 }
11602 
11603 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11604   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11606   attributes.set_is_evex_instruction();
11607   attributes.set_embedded_opmask_register_specifier(mask);
11608   if (merge) {
11609     attributes.reset_is_clear_context();
11610   }
11611   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11612   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11613 }
11614 
11615 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11616   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11617   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11618   attributes.set_is_evex_instruction();
11619   attributes.set_embedded_opmask_register_specifier(mask);
11620   if (merge) {
11621     attributes.reset_is_clear_context();
11622   }
11623   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11624   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11625 }
11626 
11627 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11628   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11629   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11630   attributes.set_is_evex_instruction();
11631   attributes.set_embedded_opmask_register_specifier(mask);
11632   if (merge) {
11633     attributes.reset_is_clear_context();
11634   }
11635   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11636   emit_int16(0x15, (0xC0 | encode));
11637 }
11638 
11639 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11640   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11641   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11642   attributes.set_is_evex_instruction();
11643   attributes.set_embedded_opmask_register_specifier(mask);
11644   if (merge) {
11645     attributes.reset_is_clear_context();
11646   }
11647   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11648   emit_int16(0x15, (0xC0 | encode));
11649 }
11650 
11651 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11652   assert(VM_Version::supports_avx(), "");
11653   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11654   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11655   int mask_enc = mask->encoding();
11656   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11657 }
11658 
11659 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11660   assert(VM_Version::supports_evex(), "");
11661   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11662   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11663   attributes.set_is_evex_instruction();
11664   attributes.set_embedded_opmask_register_specifier(mask);
11665   if (merge) {
11666     attributes.reset_is_clear_context();
11667   }
11668   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11669   emit_int16(0x65, (0xC0 | encode));
11670 }
11671 
11672 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11673   assert(VM_Version::supports_evex(), "");
11674   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11675   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11676   attributes.set_is_evex_instruction();
11677   attributes.set_embedded_opmask_register_specifier(mask);
11678   if (merge) {
11679     attributes.reset_is_clear_context();
11680   }
11681   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11682   emit_int16(0x65, (0xC0 | encode));
11683 }
11684 
11685 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11686   assert(VM_Version::supports_evex(), "");
11687   assert(VM_Version::supports_avx512bw(), "");
11688   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11689   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11690   attributes.set_is_evex_instruction();
11691   attributes.set_embedded_opmask_register_specifier(mask);
11692   if (merge) {
11693     attributes.reset_is_clear_context();
11694   }
11695   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11696   emit_int16(0x66, (0xC0 | encode));
11697 }
11698 
11699 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11700   assert(VM_Version::supports_evex(), "");
11701   assert(VM_Version::supports_avx512bw(), "");
11702   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11703   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11704   attributes.set_is_evex_instruction();
11705   attributes.set_embedded_opmask_register_specifier(mask);
11706   if (merge) {
11707     attributes.reset_is_clear_context();
11708   }
11709   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11710   emit_int16(0x66, (0xC0 | encode));
11711 }
11712 
11713 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11714   assert(VM_Version::supports_evex(), "");
11715   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11716   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11717   attributes.set_is_evex_instruction();
11718   attributes.set_embedded_opmask_register_specifier(mask);
11719   if (merge) {
11720     attributes.reset_is_clear_context();
11721   }
11722   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11723   emit_int16(0x64, (0xC0 | encode));
11724 }
11725 
11726 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11727   assert(VM_Version::supports_evex(), "");
11728   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11729   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11730   attributes.set_is_evex_instruction();
11731   attributes.set_embedded_opmask_register_specifier(mask);
11732   if (merge) {
11733     attributes.reset_is_clear_context();
11734   }
11735   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11736   emit_int16(0x64, (0xC0 | encode));
11737 }
11738 
11739 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11740   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11741   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11742   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11743   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11744 }
11745 
11746 void Assembler::pextl(Register dst, Register src1, Register src2) {
11747   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11748   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11749   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11750   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11751 }
11752 
11753 void Assembler::pdepl(Register dst, Register src1, Register src2) {
11754   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11755   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11756   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11757   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11758 }
11759 
11760 void Assembler::pextq(Register dst, Register src1, Register src2) {
11761   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11762   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11763   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11764   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11765 }
11766 
11767 void Assembler::pdepq(Register dst, Register src1, Register src2) {
11768   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11769   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11770   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11771   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11772 }
11773 
11774 void Assembler::pextl(Register dst, Register src1, Address src2) {
11775   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11776   InstructionMark im(this);
11777   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11778   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11779   emit_int8((unsigned char)0xF5);
11780   emit_operand(dst, src2);
11781 }
11782 
11783 void Assembler::pdepl(Register dst, Register src1, Address src2) {
11784   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11785   InstructionMark im(this);
11786   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11787   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11788   emit_int8((unsigned char)0xF5);
11789   emit_operand(dst, src2);
11790 }
11791 
11792 void Assembler::pextq(Register dst, Register src1, Address src2) {
11793   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11794   InstructionMark im(this);
11795   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11796   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11797   emit_int8((unsigned char)0xF5);
11798   emit_operand(dst, src2);
11799 }
11800 
11801 void Assembler::pdepq(Register dst, Register src1, Address src2) {
11802   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11803   InstructionMark im(this);
11804   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11805   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11806   emit_int8((unsigned char)0xF5);
11807   emit_operand(dst, src2);
11808 }
11809 
11810 void Assembler::sarxl(Register dst, Register src1, Register src2) {
11811   assert(VM_Version::supports_bmi2(), "");
11812   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11813   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11814   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11815 }
11816 
11817 void Assembler::sarxl(Register dst, Address src1, Register src2) {
11818   assert(VM_Version::supports_bmi2(), "");
11819   InstructionMark im(this);
11820   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11821   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11822   emit_int8((unsigned char)0xF7);
11823   emit_operand(dst, src1);
11824 }
11825 
11826 void Assembler::sarxq(Register dst, Register src1, Register src2) {
11827   assert(VM_Version::supports_bmi2(), "");
11828   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11829   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11830   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11831 }
11832 
11833 void Assembler::sarxq(Register dst, Address src1, Register src2) {
11834   assert(VM_Version::supports_bmi2(), "");
11835   InstructionMark im(this);
11836   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11837   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11838   emit_int8((unsigned char)0xF7);
11839   emit_operand(dst, src1);
11840 }
11841 
11842 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11843   assert(VM_Version::supports_bmi2(), "");
11844   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11845   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11846   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11847 }
11848 
11849 void Assembler::shlxl(Register dst, Address src1, Register src2) {
11850   assert(VM_Version::supports_bmi2(), "");
11851   InstructionMark im(this);
11852   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11853   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11854   emit_int8((unsigned char)0xF7);
11855   emit_operand(dst, src1);
11856 }
11857 
11858 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11859   assert(VM_Version::supports_bmi2(), "");
11860   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11861   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11862   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11863 }
11864 
11865 void Assembler::shlxq(Register dst, Address src1, Register src2) {
11866   assert(VM_Version::supports_bmi2(), "");
11867   InstructionMark im(this);
11868   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11869   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11870   emit_int8((unsigned char)0xF7);
11871   emit_operand(dst, src1);
11872 }
11873 
11874 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11875   assert(VM_Version::supports_bmi2(), "");
11876   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11877   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11878   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11879 }
11880 
11881 void Assembler::shrxl(Register dst, Address src1, Register src2) {
11882   assert(VM_Version::supports_bmi2(), "");
11883   InstructionMark im(this);
11884   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11885   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11886   emit_int8((unsigned char)0xF7);
11887   emit_operand(dst, src1);
11888 }
11889 
11890 void Assembler::shrxq(Register dst, Register src1, Register src2) {
11891   assert(VM_Version::supports_bmi2(), "");
11892   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11893   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11894   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11895 }
11896 
11897 void Assembler::shrxq(Register dst, Address src1, Register src2) {
11898   assert(VM_Version::supports_bmi2(), "");
11899   InstructionMark im(this);
11900   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11901   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11902   emit_int8((unsigned char)0xF7);
11903   emit_operand(dst, src1);
11904 }
11905 
11906 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
11907   assert(VM_Version::supports_avx512vldq(), "");
11908   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11909   attributes.set_is_evex_instruction();
11910   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11911   emit_int16(0x39, (0xC0 | encode));
11912 }
11913 
11914 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
11915   assert(VM_Version::supports_avx512vldq(), "");
11916   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11917   attributes.set_is_evex_instruction();
11918   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11919   emit_int16(0x39, (0xC0 | encode));
11920 }
11921 
11922 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
11923   assert(VM_Version::supports_avx512vlbw(), "");
11924   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11925   attributes.set_is_evex_instruction();
11926   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11927   emit_int16(0x29, (0xC0 | encode));
11928 }
11929 
11930 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
11931   assert(VM_Version::supports_avx512vlbw(), "");
11932   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11933   attributes.set_is_evex_instruction();
11934   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11935   emit_int16(0x29, (0xC0 | encode));
11936 }
11937 
11938 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
11939   assert(VM_Version::supports_avx512vldq(), "");
11940   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11941   attributes.set_is_evex_instruction();
11942   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11943   emit_int16(0x38, (0xC0 | encode));
11944 }
11945 
11946 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
11947   assert(VM_Version::supports_avx512vldq(), "");
11948   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11949   attributes.set_is_evex_instruction();
11950   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11951   emit_int16(0x38, (0xC0 | encode));
11952 }
11953 
11954 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
11955   assert(VM_Version::supports_avx512vlbw(), "");
11956   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11957   attributes.set_is_evex_instruction();
11958   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11959   emit_int16(0x28, (0xC0 | encode));
11960 }
11961 
11962 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
11963   assert(VM_Version::supports_avx512vlbw(), "");
11964   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11965   attributes.set_is_evex_instruction();
11966   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11967   emit_int16(0x28, (0xC0 | encode));
11968 }
11969 
11970 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11971   assert(VM_Version::supports_avx512_vbmi2(), "");
11972   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11973   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11974   attributes.set_embedded_opmask_register_specifier(mask);
11975   attributes.set_is_evex_instruction();
11976   if (merge) {
11977     attributes.reset_is_clear_context();
11978   }
11979   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11980   emit_int16((unsigned char)0x63, (0xC0 | encode));
11981 }
11982 
11983 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11984   assert(VM_Version::supports_avx512_vbmi2(), "");
11985   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11986   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11987   attributes.set_embedded_opmask_register_specifier(mask);
11988   attributes.set_is_evex_instruction();
11989   if (merge) {
11990     attributes.reset_is_clear_context();
11991   }
11992   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11993   emit_int16((unsigned char)0x63, (0xC0 | encode));
11994 }
11995 
11996 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11997   assert(VM_Version::supports_evex(), "");
11998   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11999   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12000   attributes.set_embedded_opmask_register_specifier(mask);
12001   attributes.set_is_evex_instruction();
12002   if (merge) {
12003     attributes.reset_is_clear_context();
12004   }
12005   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12006   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12007 }
12008 
12009 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12010   assert(VM_Version::supports_evex(), "");
12011   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12013   attributes.set_embedded_opmask_register_specifier(mask);
12014   attributes.set_is_evex_instruction();
12015   if (merge) {
12016     attributes.reset_is_clear_context();
12017   }
12018   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12019   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12020 }
12021 
12022 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12023   assert(VM_Version::supports_evex(), "");
12024   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12026   attributes.set_embedded_opmask_register_specifier(mask);
12027   attributes.set_is_evex_instruction();
12028   if (merge) {
12029     attributes.reset_is_clear_context();
12030   }
12031   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12032   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12033 }
12034 
12035 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12036   assert(VM_Version::supports_evex(), "");
12037   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12038   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12039   attributes.set_embedded_opmask_register_specifier(mask);
12040   attributes.set_is_evex_instruction();
12041   if (merge) {
12042     attributes.reset_is_clear_context();
12043   }
12044   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12045   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12046 }
12047 
12048 #ifndef _LP64
12049 
12050 void Assembler::incl(Register dst) {
12051   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12052   emit_int8(0x40 | dst->encoding());
12053 }
12054 
12055 void Assembler::lea(Register dst, Address src) {
12056   leal(dst, src);
12057 }
12058 
12059 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12060   InstructionMark im(this);
12061   emit_int8((unsigned char)0xC7);
12062   emit_operand(rax, dst);
12063   emit_data((int)imm32, rspec, 0);
12064 }
12065 
12066 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12067   InstructionMark im(this);
12068   int encode = prefix_and_encode(dst->encoding());
12069   emit_int8((0xB8 | encode));
12070   emit_data((int)imm32, rspec, 0);
12071 }
12072 
12073 void Assembler::popa() { // 32bit
12074   emit_int8(0x61);
12075 }
12076 
12077 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12078   InstructionMark im(this);
12079   emit_int8(0x68);
12080   emit_data(imm32, rspec, 0);
12081 }
12082 
12083 void Assembler::pusha() { // 32bit
12084   emit_int8(0x60);
12085 }
12086 
12087 void Assembler::set_byte_if_not_zero(Register dst) {
12088   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12089 }
12090 
12091 #else // LP64
12092 
12093 void Assembler::set_byte_if_not_zero(Register dst) {
12094   int enc = prefix_and_encode(dst->encoding(), true);
12095   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12096 }
12097 
12098 // 64bit only pieces of the assembler
12099 // This should only be used by 64bit instructions that can use rip-relative
12100 // it cannot be used by instructions that want an immediate value.
12101 
12102 bool Assembler::reachable(AddressLiteral adr) {
12103   int64_t disp;
12104   relocInfo::relocType relocType = adr.reloc();
12105 
12106   // None will force a 64bit literal to the code stream. Likely a placeholder
12107   // for something that will be patched later and we need to certain it will
12108   // always be reachable.
12109   if (relocType == relocInfo::none) {
12110     return false;
12111   }
12112   if (relocType == relocInfo::internal_word_type) {
12113     // This should be rip relative and easily reachable.
12114     return true;
12115   }
12116   if (relocType == relocInfo::virtual_call_type ||
12117       relocType == relocInfo::opt_virtual_call_type ||
12118       relocType == relocInfo::static_call_type ||
12119       relocType == relocInfo::static_stub_type ) {
12120     // This should be rip relative within the code cache and easily
12121     // reachable until we get huge code caches. (At which point
12122     // ic code is going to have issues).
12123     return true;
12124   }
12125   if (relocType != relocInfo::external_word_type &&
12126       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
12127       relocType != relocInfo::poll_type &&         // relocs to identify them
12128       relocType != relocInfo::runtime_call_type ) {
12129     return false;
12130   }
12131 
12132   // Stress the correction code
12133   if (ForceUnreachable) {
12134     // Must be runtimecall reloc, see if it is in the codecache
12135     // Flipping stuff in the codecache to be unreachable causes issues
12136     // with things like inline caches where the additional instructions
12137     // are not handled.
12138     if (CodeCache::find_blob(adr._target) == NULL) {
12139       return false;
12140     }
12141   }
12142   // For external_word_type/runtime_call_type if it is reachable from where we
12143   // are now (possibly a temp buffer) and where we might end up
12144   // anywhere in the codeCache then we are always reachable.
12145   // This would have to change if we ever save/restore shared code
12146   // to be more pessimistic.
12147   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
12148   if (!is_simm32(disp)) return false;
12149   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
12150   if (!is_simm32(disp)) return false;
12151 
12152   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
12153 
12154   // Because rip relative is a disp + address_of_next_instruction and we
12155   // don't know the value of address_of_next_instruction we apply a fudge factor
12156   // to make sure we will be ok no matter the size of the instruction we get placed into.
12157   // We don't have to fudge the checks above here because they are already worst case.
12158 
12159   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
12160   // + 4 because better safe than sorry.
12161   const int fudge = 12 + 4;
12162   if (disp < 0) {
12163     disp -= fudge;
12164   } else {
12165     disp += fudge;
12166   }
12167   return is_simm32(disp);
12168 }
12169 
12170 void Assembler::emit_data64(jlong data,
12171                             relocInfo::relocType rtype,
12172                             int format) {
12173   if (rtype == relocInfo::none) {
12174     emit_int64(data);
12175   } else {
12176     emit_data64(data, Relocation::spec_simple(rtype), format);
12177   }
12178 }
12179 
12180 void Assembler::emit_data64(jlong data,
12181                             RelocationHolder const& rspec,
12182                             int format) {
12183   assert(imm_operand == 0, "default format must be immediate in this file");
12184   assert(imm_operand == format, "must be immediate");
12185   assert(inst_mark() != NULL, "must be inside InstructionMark");
12186   // Do not use AbstractAssembler::relocate, which is not intended for
12187   // embedded words.  Instead, relocate to the enclosing instruction.
12188   code_section()->relocate(inst_mark(), rspec, format);
12189 #ifdef ASSERT
12190   check_relocation(rspec, format);
12191 #endif
12192   emit_int64(data);
12193 }
12194 
12195 void Assembler::prefix(Register reg) {
12196   if (reg->encoding() >= 8) {
12197     prefix(REX_B);
12198   }
12199 }
12200 
12201 void Assembler::prefix(Register dst, Register src, Prefix p) {
12202   if (src->encoding() >= 8) {
12203     p = (Prefix)(p | REX_B);
12204   }
12205   if (dst->encoding() >= 8) {
12206     p = (Prefix)(p | REX_R);
12207   }
12208   if (p != Prefix_EMPTY) {
12209     // do not generate an empty prefix
12210     prefix(p);
12211   }
12212 }
12213 
12214 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12215   if (adr.base_needs_rex()) {
12216     if (adr.index_needs_rex()) {
12217       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12218     } else {
12219       prefix(REX_B);
12220     }
12221   } else {
12222     if (adr.index_needs_rex()) {
12223       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12224     }
12225   }
12226   if (dst->encoding() >= 8) {
12227     p = (Prefix)(p | REX_R);
12228   }
12229   if (p != Prefix_EMPTY) {
12230     // do not generate an empty prefix
12231     prefix(p);
12232   }
12233 }
12234 
12235 void Assembler::prefix(Address adr) {
12236   if (adr.base_needs_rex()) {
12237     if (adr.index_needs_rex()) {
12238       prefix(REX_XB);
12239     } else {
12240       prefix(REX_B);
12241     }
12242   } else {
12243     if (adr.index_needs_rex()) {
12244       prefix(REX_X);
12245     }
12246   }
12247 }
12248 
12249 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12250   if (reg->encoding() < 8) {
12251     if (adr.base_needs_rex()) {
12252       if (adr.index_needs_rex()) {
12253         prefix(REX_XB);
12254       } else {
12255         prefix(REX_B);
12256       }
12257     } else {
12258       if (adr.index_needs_rex()) {
12259         prefix(REX_X);
12260       } else if (byteinst && reg->encoding() >= 4) {
12261         prefix(REX);
12262       }
12263     }
12264   } else {
12265     if (adr.base_needs_rex()) {
12266       if (adr.index_needs_rex()) {
12267         prefix(REX_RXB);
12268       } else {
12269         prefix(REX_RB);
12270       }
12271     } else {
12272       if (adr.index_needs_rex()) {
12273         prefix(REX_RX);
12274       } else {
12275         prefix(REX_R);
12276       }
12277     }
12278   }
12279 }
12280 
12281 void Assembler::prefix(Address adr, XMMRegister reg) {
12282   if (reg->encoding() < 8) {
12283     if (adr.base_needs_rex()) {
12284       if (adr.index_needs_rex()) {
12285         prefix(REX_XB);
12286       } else {
12287         prefix(REX_B);
12288       }
12289     } else {
12290       if (adr.index_needs_rex()) {
12291         prefix(REX_X);
12292       }
12293     }
12294   } else {
12295     if (adr.base_needs_rex()) {
12296       if (adr.index_needs_rex()) {
12297         prefix(REX_RXB);
12298       } else {
12299         prefix(REX_RB);
12300       }
12301     } else {
12302       if (adr.index_needs_rex()) {
12303         prefix(REX_RX);
12304       } else {
12305         prefix(REX_R);
12306       }
12307     }
12308   }
12309 }
12310 
12311 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12312   if (reg_enc >= 8) {
12313     prefix(REX_B);
12314     reg_enc -= 8;
12315   } else if (byteinst && reg_enc >= 4) {
12316     prefix(REX);
12317   }
12318   return reg_enc;
12319 }
12320 
12321 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12322   if (dst_enc < 8) {
12323     if (src_enc >= 8) {
12324       prefix(REX_B);
12325       src_enc -= 8;
12326     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12327       prefix(REX);
12328     }
12329   } else {
12330     if (src_enc < 8) {
12331       prefix(REX_R);
12332     } else {
12333       prefix(REX_RB);
12334       src_enc -= 8;
12335     }
12336     dst_enc -= 8;
12337   }
12338   return dst_enc << 3 | src_enc;
12339 }
12340 
12341 int8_t Assembler::get_prefixq(Address adr) {
12342   int8_t prfx = get_prefixq(adr, rax);
12343   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12344   return prfx;
12345 }
12346 
12347 int8_t Assembler::get_prefixq(Address adr, Register src) {
12348   int8_t prfx = (int8_t)(REX_W +
12349                          ((int)adr.base_needs_rex()) +
12350                          ((int)adr.index_needs_rex() << 1) +
12351                          ((int)(src->encoding() >= 8) << 2));
12352 #ifdef ASSERT
12353   if (src->encoding() < 8) {
12354     if (adr.base_needs_rex()) {
12355       if (adr.index_needs_rex()) {
12356         assert(prfx == REX_WXB, "must be");
12357       } else {
12358         assert(prfx == REX_WB, "must be");
12359       }
12360     } else {
12361       if (adr.index_needs_rex()) {
12362         assert(prfx == REX_WX, "must be");
12363       } else {
12364         assert(prfx == REX_W, "must be");
12365       }
12366     }
12367   } else {
12368     if (adr.base_needs_rex()) {
12369       if (adr.index_needs_rex()) {
12370         assert(prfx == REX_WRXB, "must be");
12371       } else {
12372         assert(prfx == REX_WRB, "must be");
12373       }
12374     } else {
12375       if (adr.index_needs_rex()) {
12376         assert(prfx == REX_WRX, "must be");
12377       } else {
12378         assert(prfx == REX_WR, "must be");
12379       }
12380     }
12381   }
12382 #endif
12383   return prfx;
12384 }
12385 
12386 void Assembler::prefixq(Address adr) {
12387   emit_int8(get_prefixq(adr));
12388 }
12389 
12390 void Assembler::prefixq(Address adr, Register src) {
12391   emit_int8(get_prefixq(adr, src));
12392 }
12393 
12394 void Assembler::prefixq(Address adr, XMMRegister src) {
12395   if (src->encoding() < 8) {
12396     if (adr.base_needs_rex()) {
12397       if (adr.index_needs_rex()) {
12398         prefix(REX_WXB);
12399       } else {
12400         prefix(REX_WB);
12401       }
12402     } else {
12403       if (adr.index_needs_rex()) {
12404         prefix(REX_WX);
12405       } else {
12406         prefix(REX_W);
12407       }
12408     }
12409   } else {
12410     if (adr.base_needs_rex()) {
12411       if (adr.index_needs_rex()) {
12412         prefix(REX_WRXB);
12413       } else {
12414         prefix(REX_WRB);
12415       }
12416     } else {
12417       if (adr.index_needs_rex()) {
12418         prefix(REX_WRX);
12419       } else {
12420         prefix(REX_WR);
12421       }
12422     }
12423   }
12424 }
12425 
12426 int Assembler::prefixq_and_encode(int reg_enc) {
12427   if (reg_enc < 8) {
12428     prefix(REX_W);
12429   } else {
12430     prefix(REX_WB);
12431     reg_enc -= 8;
12432   }
12433   return reg_enc;
12434 }
12435 
12436 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12437   if (dst_enc < 8) {
12438     if (src_enc < 8) {
12439       prefix(REX_W);
12440     } else {
12441       prefix(REX_WB);
12442       src_enc -= 8;
12443     }
12444   } else {
12445     if (src_enc < 8) {
12446       prefix(REX_WR);
12447     } else {
12448       prefix(REX_WRB);
12449       src_enc -= 8;
12450     }
12451     dst_enc -= 8;
12452   }
12453   return dst_enc << 3 | src_enc;
12454 }
12455 
12456 void Assembler::adcq(Register dst, int32_t imm32) {
12457   (void) prefixq_and_encode(dst->encoding());
12458   emit_arith(0x81, 0xD0, dst, imm32);
12459 }
12460 
12461 void Assembler::adcq(Register dst, Address src) {
12462   InstructionMark im(this);
12463   emit_int16(get_prefixq(src, dst), 0x13);
12464   emit_operand(dst, src);
12465 }
12466 
12467 void Assembler::adcq(Register dst, Register src) {
12468   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12469   emit_arith(0x13, 0xC0, dst, src);
12470 }
12471 
12472 void Assembler::addq(Address dst, int32_t imm32) {
12473   InstructionMark im(this);
12474   prefixq(dst);
12475   emit_arith_operand(0x81, rax, dst, imm32);
12476 }
12477 
12478 void Assembler::addq(Address dst, Register src) {
12479   InstructionMark im(this);
12480   emit_int16(get_prefixq(dst, src), 0x01);
12481   emit_operand(src, dst);
12482 }
12483 
12484 void Assembler::addq(Register dst, int32_t imm32) {
12485   (void) prefixq_and_encode(dst->encoding());
12486   emit_arith(0x81, 0xC0, dst, imm32);
12487 }
12488 
12489 void Assembler::addq(Register dst, Address src) {
12490   InstructionMark im(this);
12491   emit_int16(get_prefixq(src, dst), 0x03);
12492   emit_operand(dst, src);
12493 }
12494 
12495 void Assembler::addq(Register dst, Register src) {
12496   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12497   emit_arith(0x03, 0xC0, dst, src);
12498 }
12499 
12500 void Assembler::adcxq(Register dst, Register src) {
12501   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12502   emit_int8(0x66);
12503   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12504   emit_int32(0x0F,
12505              0x38,
12506              (unsigned char)0xF6,
12507              (0xC0 | encode));
12508 }
12509 
12510 void Assembler::adoxq(Register dst, Register src) {
12511   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12512   emit_int8((unsigned char)0xF3);
12513   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12514   emit_int32(0x0F,
12515              0x38,
12516              (unsigned char)0xF6,
12517              (0xC0 | encode));
12518 }
12519 
12520 void Assembler::andq(Address dst, int32_t imm32) {
12521   InstructionMark im(this);
12522   prefixq(dst);
12523   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12524 }
12525 
12526 void Assembler::andq(Register dst, int32_t imm32) {
12527   (void) prefixq_and_encode(dst->encoding());
12528   emit_arith(0x81, 0xE0, dst, imm32);
12529 }
12530 
12531 void Assembler::andq(Register dst, Address src) {
12532   InstructionMark im(this);
12533   emit_int16(get_prefixq(src, dst), 0x23);
12534   emit_operand(dst, src);
12535 }
12536 
12537 void Assembler::andq(Register dst, Register src) {
12538   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12539   emit_arith(0x23, 0xC0, dst, src);
12540 }
12541 
12542 void Assembler::andq(Address dst, Register src) {
12543   InstructionMark im(this);
12544   emit_int16(get_prefixq(dst, src), 0x21);
12545   emit_operand(src, dst);
12546 }
12547 
12548 void Assembler::andnq(Register dst, Register src1, Register src2) {
12549   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12550   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12551   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12552   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12553 }
12554 
12555 void Assembler::andnq(Register dst, Register src1, Address src2) {
12556   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12557   InstructionMark im(this);
12558   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12559   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12560   emit_int8((unsigned char)0xF2);
12561   emit_operand(dst, src2);
12562 }
12563 
12564 void Assembler::bsfq(Register dst, Register src) {
12565   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12566   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12567 }
12568 
12569 void Assembler::bsrq(Register dst, Register src) {
12570   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12571   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12572 }
12573 
12574 void Assembler::bswapq(Register reg) {
12575   int encode = prefixq_and_encode(reg->encoding());
12576   emit_int16(0x0F, (0xC8 | encode));
12577 }
12578 
12579 void Assembler::blsiq(Register dst, Register src) {
12580   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12581   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12582   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12583   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12584 }
12585 
12586 void Assembler::blsiq(Register dst, Address src) {
12587   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12588   InstructionMark im(this);
12589   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12590   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12591   emit_int8((unsigned char)0xF3);
12592   emit_operand(rbx, src);
12593 }
12594 
12595 void Assembler::blsmskq(Register dst, Register src) {
12596   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12597   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12598   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12599   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12600 }
12601 
12602 void Assembler::blsmskq(Register dst, Address src) {
12603   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12604   InstructionMark im(this);
12605   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12606   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12607   emit_int8((unsigned char)0xF3);
12608   emit_operand(rdx, src);
12609 }
12610 
12611 void Assembler::blsrq(Register dst, Register src) {
12612   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12613   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12614   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12615   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12616 }
12617 
12618 void Assembler::blsrq(Register dst, Address src) {
12619   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12620   InstructionMark im(this);
12621   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12622   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12623   emit_int8((unsigned char)0xF3);
12624   emit_operand(rcx, src);
12625 }
12626 
12627 void Assembler::cdqq() {
12628   emit_int16(REX_W, (unsigned char)0x99);
12629 }
12630 
12631 void Assembler::clflush(Address adr) {
12632   assert(VM_Version::supports_clflush(), "should do");
12633   prefix(adr);
12634   emit_int16(0x0F, (unsigned char)0xAE);
12635   emit_operand(rdi, adr);
12636 }
12637 
12638 void Assembler::clflushopt(Address adr) {
12639   assert(VM_Version::supports_clflushopt(), "should do!");
12640   // adr should be base reg only with no index or offset
12641   assert(adr.index() == noreg, "index should be noreg");
12642   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12643   assert(adr.disp() == 0, "displacement should be 0");
12644   // instruction prefix is 0x66
12645   emit_int8(0x66);
12646   prefix(adr);
12647   // opcode family is 0x0F 0xAE
12648   emit_int16(0x0F, (unsigned char)0xAE);
12649   // extended opcode byte is 7 == rdi
12650   emit_operand(rdi, adr);
12651 }
12652 
12653 void Assembler::clwb(Address adr) {
12654   assert(VM_Version::supports_clwb(), "should do!");
12655   // adr should be base reg only with no index or offset
12656   assert(adr.index() == noreg, "index should be noreg");
12657   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12658   assert(adr.disp() == 0, "displacement should be 0");
12659   // instruction prefix is 0x66
12660   emit_int8(0x66);
12661   prefix(adr);
12662   // opcode family is 0x0f 0xAE
12663   emit_int16(0x0F, (unsigned char)0xAE);
12664   // extended opcode byte is 6 == rsi
12665   emit_operand(rsi, adr);
12666 }
12667 
12668 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12669   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12670   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12671 }
12672 
12673 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12674   InstructionMark im(this);
12675   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12676   emit_operand(dst, src);
12677 }
12678 
12679 void Assembler::cmpq(Address dst, int32_t imm32) {
12680   InstructionMark im(this);
12681   emit_int16(get_prefixq(dst), (unsigned char)0x81);
12682   emit_operand(rdi, dst, 4);
12683   emit_int32(imm32);
12684 }
12685 
12686 void Assembler::cmpq(Register dst, int32_t imm32) {
12687   (void) prefixq_and_encode(dst->encoding());
12688   emit_arith(0x81, 0xF8, dst, imm32);
12689 }
12690 
12691 void Assembler::cmpq(Address dst, Register src) {
12692   InstructionMark im(this);
12693   emit_int16(get_prefixq(dst, src), 0x39);
12694   emit_operand(src, dst);
12695 }
12696 
12697 void Assembler::cmpq(Register dst, Register src) {
12698   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12699   emit_arith(0x3B, 0xC0, dst, src);
12700 }
12701 
12702 void Assembler::cmpq(Register dst, Address src) {
12703   InstructionMark im(this);
12704   emit_int16(get_prefixq(src, dst), 0x3B);
12705   emit_operand(dst, src);
12706 }
12707 
12708 void Assembler::cmpxchgq(Register reg, Address adr) {
12709   InstructionMark im(this);
12710   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12711   emit_operand(reg, adr);
12712 }
12713 
12714 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12716   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12717   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12718   emit_int16(0x2A, (0xC0 | encode));
12719 }
12720 
12721 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12723   InstructionMark im(this);
12724   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12725   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12726   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12727   emit_int8(0x2A);
12728   emit_operand(dst, src);
12729 }
12730 
12731 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12732   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12733   InstructionMark im(this);
12734   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12735   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12736   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12737   emit_int8(0x2A);
12738   emit_operand(dst, src);
12739 }
12740 
12741 void Assembler::cvttsd2siq(Register dst, Address src) {
12742   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12743   // F2 REX.W 0F 2C /r
12744   // CVTTSD2SI r64, xmm1/m64
12745   InstructionMark im(this);
12746   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12747   emit_operand(dst, src);
12748 }
12749 
12750 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12751   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12752   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12753   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12754   emit_int16(0x2C, (0xC0 | encode));
12755 }
12756 
12757 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12758   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12759   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12760   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12761   emit_int16(0x2D, (0xC0 | encode));
12762 }
12763 
12764 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12765   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12766   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12767   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12768   emit_int16(0x2C, (0xC0 | encode));
12769 }
12770 
12771 void Assembler::decl(Register dst) {
12772   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12773   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12774   int encode = prefix_and_encode(dst->encoding());
12775   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12776 }
12777 
12778 void Assembler::decq(Register dst) {
12779   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12780   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12781   int encode = prefixq_and_encode(dst->encoding());
12782   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12783 }
12784 
12785 void Assembler::decq(Address dst) {
12786   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12787   InstructionMark im(this);
12788   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12789   emit_operand(rcx, dst);
12790 }
12791 
12792 void Assembler::fxrstor(Address src) {
12793   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12794   emit_operand(as_Register(1), src);
12795 }
12796 
12797 void Assembler::xrstor(Address src) {
12798   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12799   emit_operand(as_Register(5), src);
12800 }
12801 
12802 void Assembler::fxsave(Address dst) {
12803   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12804   emit_operand(as_Register(0), dst);
12805 }
12806 
12807 void Assembler::xsave(Address dst) {
12808   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12809   emit_operand(as_Register(4), dst);
12810 }
12811 
12812 void Assembler::idivq(Register src) {
12813   int encode = prefixq_and_encode(src->encoding());
12814   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12815 }
12816 
12817 void Assembler::divq(Register src) {
12818   int encode = prefixq_and_encode(src->encoding());
12819   emit_int16((unsigned char)0xF7, (0xF0 | encode));
12820 }
12821 
12822 void Assembler::imulq(Register dst, Register src) {
12823   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12824   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12825 }
12826 
12827 void Assembler::imulq(Register src) {
12828   int encode = prefixq_and_encode(src->encoding());
12829   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12830 }
12831 
12832 void Assembler::imulq(Register dst, Address src, int32_t value) {
12833   InstructionMark im(this);
12834   prefixq(src, dst);
12835   if (is8bit(value)) {
12836     emit_int8((unsigned char)0x6B);
12837     emit_operand(dst, src);
12838     emit_int8(value);
12839   } else {
12840     emit_int8((unsigned char)0x69);
12841     emit_operand(dst, src);
12842     emit_int32(value);
12843   }
12844 }
12845 
12846 void Assembler::imulq(Register dst, Register src, int value) {
12847   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12848   if (is8bit(value)) {
12849     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12850   } else {
12851     emit_int16(0x69, (0xC0 | encode));
12852     emit_int32(value);
12853   }
12854 }
12855 
12856 void Assembler::imulq(Register dst, Address src) {
12857   InstructionMark im(this);
12858   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12859   emit_operand(dst, src);
12860 }
12861 
12862 void Assembler::incl(Register dst) {
12863   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12864   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12865   int encode = prefix_and_encode(dst->encoding());
12866   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12867 }
12868 
12869 void Assembler::incq(Register dst) {
12870   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12871   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12872   int encode = prefixq_and_encode(dst->encoding());
12873   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12874 }
12875 
12876 void Assembler::incq(Address dst) {
12877   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12878   InstructionMark im(this);
12879   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12880   emit_operand(rax, dst);
12881 }
12882 
12883 void Assembler::lea(Register dst, Address src) {
12884   leaq(dst, src);
12885 }
12886 
12887 void Assembler::leaq(Register dst, Address src) {
12888   InstructionMark im(this);
12889   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
12890   emit_operand(dst, src);
12891 }
12892 
12893 void Assembler::mov64(Register dst, int64_t imm64) {
12894   InstructionMark im(this);
12895   int encode = prefixq_and_encode(dst->encoding());
12896   emit_int8(0xB8 | encode);
12897   emit_int64(imm64);
12898 }
12899 
12900 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
12901   InstructionMark im(this);
12902   int encode = prefixq_and_encode(dst->encoding());
12903   emit_int8(0xB8 | encode);
12904   emit_data64(imm64, rtype, format);
12905 }
12906 
12907 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
12908   InstructionMark im(this);
12909   int encode = prefixq_and_encode(dst->encoding());
12910   emit_int8(0xB8 | encode);
12911   emit_data64(imm64, rspec);
12912 }
12913 
12914 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12915   InstructionMark im(this);
12916   int encode = prefix_and_encode(dst->encoding());
12917   emit_int8(0xB8 | encode);
12918   emit_data((int)imm32, rspec, narrow_oop_operand);
12919 }
12920 
12921 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
12922   InstructionMark im(this);
12923   prefix(dst);
12924   emit_int8((unsigned char)0xC7);
12925   emit_operand(rax, dst, 4);
12926   emit_data((int)imm32, rspec, narrow_oop_operand);
12927 }
12928 
12929 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
12930   InstructionMark im(this);
12931   int encode = prefix_and_encode(src1->encoding());
12932   emit_int16((unsigned char)0x81, (0xF8 | encode));
12933   emit_data((int)imm32, rspec, narrow_oop_operand);
12934 }
12935 
12936 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
12937   InstructionMark im(this);
12938   prefix(src1);
12939   emit_int8((unsigned char)0x81);
12940   emit_operand(rax, src1, 4);
12941   emit_data((int)imm32, rspec, narrow_oop_operand);
12942 }
12943 
12944 void Assembler::lzcntq(Register dst, Register src) {
12945   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12946   emit_int8((unsigned char)0xF3);
12947   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12948   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12949 }
12950 
12951 void Assembler::lzcntq(Register dst, Address src) {
12952   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12953   InstructionMark im(this);
12954   emit_int8((unsigned char)0xF3);
12955   prefixq(src, dst);
12956   emit_int16(0x0F, (unsigned char)0xBD);
12957   emit_operand(dst, src);
12958 }
12959 
12960 void Assembler::movdq(XMMRegister dst, Register src) {
12961   // table D-1 says MMX/SSE2
12962   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12963   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12964   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12965   emit_int16(0x6E, (0xC0 | encode));
12966 }
12967 
12968 void Assembler::movdq(Register dst, XMMRegister src) {
12969   // table D-1 says MMX/SSE2
12970   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12971   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12972   // swap src/dst to get correct prefix
12973   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12974   emit_int16(0x7E,
12975              (0xC0 | encode));
12976 }
12977 
12978 void Assembler::movq(Register dst, Register src) {
12979   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12980   emit_int16((unsigned char)0x8B,
12981              (0xC0 | encode));
12982 }
12983 
12984 void Assembler::movq(Register dst, Address src) {
12985   InstructionMark im(this);
12986   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
12987   emit_operand(dst, src);
12988 }
12989 
12990 void Assembler::movq(Address dst, Register src) {
12991   InstructionMark im(this);
12992   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
12993   emit_operand(src, dst);
12994 }
12995 
12996 void Assembler::movq(Address dst, int32_t imm32) {
12997   InstructionMark im(this);
12998   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12999   emit_operand(as_Register(0), dst);
13000   emit_int32(imm32);
13001 }
13002 
13003 void Assembler::movq(Register dst, int32_t imm32) {
13004   int encode = prefixq_and_encode(dst->encoding());
13005   emit_int16((unsigned char)0xC7, (0xC0 | encode));
13006   emit_int32(imm32);
13007 }
13008 
13009 void Assembler::movsbq(Register dst, Address src) {
13010   InstructionMark im(this);
13011   emit_int24(get_prefixq(src, dst),
13012              0x0F,
13013              (unsigned char)0xBE);
13014   emit_operand(dst, src);
13015 }
13016 
13017 void Assembler::movsbq(Register dst, Register src) {
13018   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13019   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13020 }
13021 
13022 void Assembler::movslq(Register dst, int32_t imm32) {
13023   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
13024   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
13025   // as a result we shouldn't use until tested at runtime...
13026   ShouldNotReachHere();
13027   InstructionMark im(this);
13028   int encode = prefixq_and_encode(dst->encoding());
13029   emit_int8(0xC7 | encode);
13030   emit_int32(imm32);
13031 }
13032 
13033 void Assembler::movslq(Address dst, int32_t imm32) {
13034   assert(is_simm32(imm32), "lost bits");
13035   InstructionMark im(this);
13036   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13037   emit_operand(rax, dst, 4);
13038   emit_int32(imm32);
13039 }
13040 
13041 void Assembler::movslq(Register dst, Address src) {
13042   InstructionMark im(this);
13043   emit_int16(get_prefixq(src, dst), 0x63);
13044   emit_operand(dst, src);
13045 }
13046 
13047 void Assembler::movslq(Register dst, Register src) {
13048   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13049   emit_int16(0x63, (0xC0 | encode));
13050 }
13051 
13052 void Assembler::movswq(Register dst, Address src) {
13053   InstructionMark im(this);
13054   emit_int24(get_prefixq(src, dst),
13055              0x0F,
13056              (unsigned char)0xBF);
13057   emit_operand(dst, src);
13058 }
13059 
13060 void Assembler::movswq(Register dst, Register src) {
13061   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13062   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13063 }
13064 
13065 void Assembler::movzbq(Register dst, Address src) {
13066   InstructionMark im(this);
13067   emit_int24(get_prefixq(src, dst),
13068              0x0F,
13069              (unsigned char)0xB6);
13070   emit_operand(dst, src);
13071 }
13072 
13073 void Assembler::movzbq(Register dst, Register src) {
13074   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13075   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13076 }
13077 
13078 void Assembler::movzwq(Register dst, Address src) {
13079   InstructionMark im(this);
13080   emit_int24(get_prefixq(src, dst),
13081              0x0F,
13082              (unsigned char)0xB7);
13083   emit_operand(dst, src);
13084 }
13085 
13086 void Assembler::movzwq(Register dst, Register src) {
13087   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13088   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13089 }
13090 
13091 void Assembler::mulq(Address src) {
13092   InstructionMark im(this);
13093   emit_int16(get_prefixq(src), (unsigned char)0xF7);
13094   emit_operand(rsp, src);
13095 }
13096 
13097 void Assembler::mulq(Register src) {
13098   int encode = prefixq_and_encode(src->encoding());
13099   emit_int16((unsigned char)0xF7, (0xE0 | encode));
13100 }
13101 
13102 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13103   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13104   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13105   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13106   emit_int16((unsigned char)0xF6, (0xC0 | encode));
13107 }
13108 
13109 void Assembler::negq(Register dst) {
13110   int encode = prefixq_and_encode(dst->encoding());
13111   emit_int16((unsigned char)0xF7, (0xD8 | encode));
13112 }
13113 
13114 void Assembler::negq(Address dst) {
13115   InstructionMark im(this);
13116   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13117   emit_operand(as_Register(3), dst);
13118 }
13119 
13120 void Assembler::notq(Register dst) {
13121   int encode = prefixq_and_encode(dst->encoding());
13122   emit_int16((unsigned char)0xF7, (0xD0 | encode));
13123 }
13124 
13125 void Assembler::btsq(Address dst, int imm8) {
13126   assert(isByte(imm8), "not a byte");
13127   InstructionMark im(this);
13128   emit_int24(get_prefixq(dst),
13129              0x0F,
13130              (unsigned char)0xBA);
13131   emit_operand(rbp /* 5 */, dst, 1);
13132   emit_int8(imm8);
13133 }
13134 
13135 void Assembler::btrq(Address dst, int imm8) {
13136   assert(isByte(imm8), "not a byte");
13137   InstructionMark im(this);
13138   emit_int24(get_prefixq(dst),
13139              0x0F,
13140              (unsigned char)0xBA);
13141   emit_operand(rsi /* 6 */, dst, 1);
13142   emit_int8(imm8);
13143 }
13144 
13145 void Assembler::orq(Address dst, int32_t imm32) {
13146   InstructionMark im(this);
13147   prefixq(dst);
13148   emit_arith_operand(0x81, as_Register(1), dst, imm32);
13149 }
13150 
13151 void Assembler::orq(Address dst, Register src) {
13152   InstructionMark im(this);
13153   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13154   emit_operand(src, dst);
13155 }
13156 
13157 void Assembler::orq(Register dst, int32_t imm32) {
13158   (void) prefixq_and_encode(dst->encoding());
13159   emit_arith(0x81, 0xC8, dst, imm32);
13160 }
13161 
13162 void Assembler::orq(Register dst, Address src) {
13163   InstructionMark im(this);
13164   emit_int16(get_prefixq(src, dst), 0x0B);
13165   emit_operand(dst, src);
13166 }
13167 
13168 void Assembler::orq(Register dst, Register src) {
13169   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13170   emit_arith(0x0B, 0xC0, dst, src);
13171 }
13172 
13173 void Assembler::popcntq(Register dst, Address src) {
13174   assert(VM_Version::supports_popcnt(), "must support");
13175   InstructionMark im(this);
13176   emit_int32((unsigned char)0xF3,
13177              get_prefixq(src, dst),
13178              0x0F,
13179              (unsigned char)0xB8);
13180   emit_operand(dst, src);
13181 }
13182 
13183 void Assembler::popcntq(Register dst, Register src) {
13184   assert(VM_Version::supports_popcnt(), "must support");
13185   emit_int8((unsigned char)0xF3);
13186   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13187   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13188 }
13189 
13190 void Assembler::popq(Address dst) {
13191   InstructionMark im(this);
13192   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13193   emit_operand(rax, dst);
13194 }
13195 
13196 void Assembler::popq(Register dst) {
13197   emit_int8((unsigned char)0x58 | dst->encoding());
13198 }
13199 
13200 // Precomputable: popa, pusha, vzeroupper
13201 
13202 // The result of these routines are invariant from one invocation to another
13203 // invocation for the duration of a run. Caching the result on bootstrap
13204 // and copying it out on subsequent invocations can thus be beneficial
13205 static bool     precomputed = false;
13206 
13207 static u_char* popa_code  = NULL;
13208 static int     popa_len   = 0;
13209 
13210 static u_char* pusha_code = NULL;
13211 static int     pusha_len  = 0;
13212 
13213 static u_char* vzup_code  = NULL;
13214 static int     vzup_len   = 0;
13215 
13216 void Assembler::precompute_instructions() {
13217   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13218   guarantee(!precomputed, "only once");
13219   precomputed = true;
13220   ResourceMark rm;
13221 
13222   // Make a temporary buffer big enough for the routines we're capturing
13223   int size = 256;
13224   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13225   CodeBuffer buffer((address)tmp_code, size);
13226   MacroAssembler masm(&buffer);
13227 
13228   address begin_popa  = masm.code_section()->end();
13229   masm.popa_uncached();
13230   address end_popa    = masm.code_section()->end();
13231   masm.pusha_uncached();
13232   address end_pusha   = masm.code_section()->end();
13233   masm.vzeroupper_uncached();
13234   address end_vzup    = masm.code_section()->end();
13235 
13236   // Save the instructions to permanent buffers.
13237   popa_len = (int)(end_popa - begin_popa);
13238   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13239   memcpy(popa_code, begin_popa, popa_len);
13240 
13241   pusha_len = (int)(end_pusha - end_popa);
13242   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13243   memcpy(pusha_code, end_popa, pusha_len);
13244 
13245   vzup_len = (int)(end_vzup - end_pusha);
13246   if (vzup_len > 0) {
13247     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13248     memcpy(vzup_code, end_pusha, vzup_len);
13249   } else {
13250     vzup_code = pusha_code; // dummy
13251   }
13252 
13253   assert(masm.code()->total_oop_size() == 0 &&
13254          masm.code()->total_metadata_size() == 0 &&
13255          masm.code()->total_relocation_size() == 0,
13256          "pre-computed code can't reference oops, metadata or contain relocations");
13257 }
13258 
13259 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13260   assert(src != NULL, "code to copy must have been pre-computed");
13261   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13262   address end = code_section->end();
13263   memcpy(end, src, src_len);
13264   code_section->set_end(end + src_len);
13265 }
13266 
13267 void Assembler::popa() { // 64bit
13268   emit_copy(code_section(), popa_code, popa_len);
13269 }
13270 
13271 void Assembler::popa_uncached() { // 64bit
13272   movq(r15, Address(rsp, 0));
13273   movq(r14, Address(rsp, wordSize));
13274   movq(r13, Address(rsp, 2 * wordSize));
13275   movq(r12, Address(rsp, 3 * wordSize));
13276   movq(r11, Address(rsp, 4 * wordSize));
13277   movq(r10, Address(rsp, 5 * wordSize));
13278   movq(r9,  Address(rsp, 6 * wordSize));
13279   movq(r8,  Address(rsp, 7 * wordSize));
13280   movq(rdi, Address(rsp, 8 * wordSize));
13281   movq(rsi, Address(rsp, 9 * wordSize));
13282   movq(rbp, Address(rsp, 10 * wordSize));
13283   // Skip rsp as it is restored automatically to the value
13284   // before the corresponding pusha when popa is done.
13285   movq(rbx, Address(rsp, 12 * wordSize));
13286   movq(rdx, Address(rsp, 13 * wordSize));
13287   movq(rcx, Address(rsp, 14 * wordSize));
13288   movq(rax, Address(rsp, 15 * wordSize));
13289 
13290   addq(rsp, 16 * wordSize);
13291 }
13292 
13293 // Does not actually store the value of rsp on the stack.
13294 // The slot for rsp just contains an arbitrary value.
13295 void Assembler::pusha() { // 64bit
13296   emit_copy(code_section(), pusha_code, pusha_len);
13297 }
13298 
13299 // Does not actually store the value of rsp on the stack.
13300 // The slot for rsp just contains an arbitrary value.
13301 void Assembler::pusha_uncached() { // 64bit
13302   subq(rsp, 16 * wordSize);
13303 
13304   movq(Address(rsp, 15 * wordSize), rax);
13305   movq(Address(rsp, 14 * wordSize), rcx);
13306   movq(Address(rsp, 13 * wordSize), rdx);
13307   movq(Address(rsp, 12 * wordSize), rbx);
13308   // Skip rsp as the value is normally not used. There are a few places where
13309   // the original value of rsp needs to be known but that can be computed
13310   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13311   movq(Address(rsp, 10 * wordSize), rbp);
13312   movq(Address(rsp, 9 * wordSize), rsi);
13313   movq(Address(rsp, 8 * wordSize), rdi);
13314   movq(Address(rsp, 7 * wordSize), r8);
13315   movq(Address(rsp, 6 * wordSize), r9);
13316   movq(Address(rsp, 5 * wordSize), r10);
13317   movq(Address(rsp, 4 * wordSize), r11);
13318   movq(Address(rsp, 3 * wordSize), r12);
13319   movq(Address(rsp, 2 * wordSize), r13);
13320   movq(Address(rsp, wordSize), r14);
13321   movq(Address(rsp, 0), r15);
13322 }
13323 
13324 void Assembler::vzeroupper() {
13325   emit_copy(code_section(), vzup_code, vzup_len);
13326 }
13327 
13328 void Assembler::pushq(Address src) {
13329   InstructionMark im(this);
13330   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13331   emit_operand(rsi, src);
13332 }
13333 
13334 void Assembler::rclq(Register dst, int imm8) {
13335   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13336   int encode = prefixq_and_encode(dst->encoding());
13337   if (imm8 == 1) {
13338     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13339   } else {
13340     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13341   }
13342 }
13343 
13344 void Assembler::rcrq(Register dst, int imm8) {
13345   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13346   int encode = prefixq_and_encode(dst->encoding());
13347   if (imm8 == 1) {
13348     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13349   } else {
13350     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13351   }
13352 }
13353 
13354 void Assembler::rorxl(Register dst, Register src, int imm8) {
13355   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13356   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13357   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13358   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13359 }
13360 
13361 void Assembler::rorxl(Register dst, Address src, int imm8) {
13362   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13363   InstructionMark im(this);
13364   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13365   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13366   emit_int8((unsigned char)0xF0);
13367   emit_operand(dst, src);
13368   emit_int8(imm8);
13369 }
13370 
13371 void Assembler::rorxq(Register dst, Register src, int imm8) {
13372   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13373   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13374   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13375   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13376 }
13377 
13378 void Assembler::rorxq(Register dst, Address src, int imm8) {
13379   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13380   InstructionMark im(this);
13381   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13382   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13383   emit_int8((unsigned char)0xF0);
13384   emit_operand(dst, src);
13385   emit_int8(imm8);
13386 }
13387 
13388 #ifdef _LP64
13389 void Assembler::salq(Address dst, int imm8) {
13390   InstructionMark im(this);
13391   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13392   if (imm8 == 1) {
13393     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13394     emit_operand(as_Register(4), dst);
13395   }
13396   else {
13397     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13398     emit_operand(as_Register(4), dst);
13399     emit_int8(imm8);
13400   }
13401 }
13402 
13403 void Assembler::salq(Address dst) {
13404   InstructionMark im(this);
13405   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13406   emit_operand(as_Register(4), dst);
13407 }
13408 
13409 void Assembler::salq(Register dst, int imm8) {
13410   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13411   int encode = prefixq_and_encode(dst->encoding());
13412   if (imm8 == 1) {
13413     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13414   } else {
13415     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13416   }
13417 }
13418 
13419 void Assembler::salq(Register dst) {
13420   int encode = prefixq_and_encode(dst->encoding());
13421   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13422 }
13423 
13424 void Assembler::sarq(Address dst, int imm8) {
13425   InstructionMark im(this);
13426   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13427   if (imm8 == 1) {
13428     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13429     emit_operand(as_Register(7), dst);
13430   }
13431   else {
13432     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13433     emit_operand(as_Register(7), dst);
13434     emit_int8(imm8);
13435   }
13436 }
13437 
13438 void Assembler::sarq(Address dst) {
13439   InstructionMark im(this);
13440   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13441   emit_operand(as_Register(7), dst);
13442 }
13443 
13444 void Assembler::sarq(Register dst, int imm8) {
13445   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13446   int encode = prefixq_and_encode(dst->encoding());
13447   if (imm8 == 1) {
13448     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13449   } else {
13450     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13451   }
13452 }
13453 
13454 void Assembler::sarq(Register dst) {
13455   int encode = prefixq_and_encode(dst->encoding());
13456   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13457 }
13458 #endif
13459 
13460 void Assembler::sbbq(Address dst, int32_t imm32) {
13461   InstructionMark im(this);
13462   prefixq(dst);
13463   emit_arith_operand(0x81, rbx, dst, imm32);
13464 }
13465 
13466 void Assembler::sbbq(Register dst, int32_t imm32) {
13467   (void) prefixq_and_encode(dst->encoding());
13468   emit_arith(0x81, 0xD8, dst, imm32);
13469 }
13470 
13471 void Assembler::sbbq(Register dst, Address src) {
13472   InstructionMark im(this);
13473   emit_int16(get_prefixq(src, dst), 0x1B);
13474   emit_operand(dst, src);
13475 }
13476 
13477 void Assembler::sbbq(Register dst, Register src) {
13478   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13479   emit_arith(0x1B, 0xC0, dst, src);
13480 }
13481 
13482 void Assembler::shlq(Register dst, int imm8) {
13483   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13484   int encode = prefixq_and_encode(dst->encoding());
13485   if (imm8 == 1) {
13486     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13487   } else {
13488     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13489   }
13490 }
13491 
13492 void Assembler::shlq(Register dst) {
13493   int encode = prefixq_and_encode(dst->encoding());
13494   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13495 }
13496 
13497 void Assembler::shrq(Register dst, int imm8) {
13498   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13499   int encode = prefixq_and_encode(dst->encoding());
13500   if (imm8 == 1) {
13501     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13502   }
13503   else {
13504     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13505   }
13506 }
13507 
13508 void Assembler::shrq(Register dst) {
13509   int encode = prefixq_and_encode(dst->encoding());
13510   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13511 }
13512 
13513 void Assembler::shrq(Address dst) {
13514   InstructionMark im(this);
13515   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13516   emit_operand(as_Register(5), dst);
13517 }
13518 
13519 void Assembler::shrq(Address dst, int imm8) {
13520   InstructionMark im(this);
13521   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13522   if (imm8 == 1) {
13523     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13524     emit_operand(as_Register(5), dst);
13525   }
13526   else {
13527     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13528     emit_operand(as_Register(5), dst);
13529     emit_int8(imm8);
13530   }
13531 }
13532 
13533 void Assembler::subq(Address dst, int32_t imm32) {
13534   InstructionMark im(this);
13535   prefixq(dst);
13536   emit_arith_operand(0x81, rbp, dst, imm32);
13537 }
13538 
13539 void Assembler::subq(Address dst, Register src) {
13540   InstructionMark im(this);
13541   emit_int16(get_prefixq(dst, src), 0x29);
13542   emit_operand(src, dst);
13543 }
13544 
13545 void Assembler::subq(Register dst, int32_t imm32) {
13546   (void) prefixq_and_encode(dst->encoding());
13547   emit_arith(0x81, 0xE8, dst, imm32);
13548 }
13549 
13550 // Force generation of a 4 byte immediate value even if it fits into 8bit
13551 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13552   (void) prefixq_and_encode(dst->encoding());
13553   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13554 }
13555 
13556 void Assembler::subq(Register dst, Address src) {
13557   InstructionMark im(this);
13558   emit_int16(get_prefixq(src, dst), 0x2B);
13559   emit_operand(dst, src);
13560 }
13561 
13562 void Assembler::subq(Register dst, Register src) {
13563   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13564   emit_arith(0x2B, 0xC0, dst, src);
13565 }
13566 
13567 void Assembler::testq(Address dst, int32_t imm32) {
13568   if (imm32 >= 0) {
13569     testl(dst, imm32);
13570     return;
13571   }
13572   InstructionMark im(this);
13573   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13574   emit_operand(as_Register(0), dst);
13575   emit_int32(imm32);
13576 }
13577 
13578 void Assembler::testq(Register dst, int32_t imm32) {
13579   if (imm32 >= 0) {
13580     testl(dst, imm32);
13581     return;
13582   }
13583   // not using emit_arith because test
13584   // doesn't support sign-extension of
13585   // 8bit operands
13586   if (dst == rax) {
13587     prefix(REX_W);
13588     emit_int8((unsigned char)0xA9);
13589     emit_int32(imm32);
13590   } else {
13591     int encode = dst->encoding();
13592     encode = prefixq_and_encode(encode);
13593     emit_int16((unsigned char)0xF7, (0xC0 | encode));
13594     emit_int32(imm32);
13595   }
13596 }
13597 
13598 void Assembler::testq(Register dst, Register src) {
13599   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13600   emit_arith(0x85, 0xC0, dst, src);
13601 }
13602 
13603 void Assembler::testq(Register dst, Address src) {
13604   InstructionMark im(this);
13605   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13606   emit_operand(dst, src);
13607 }
13608 
13609 void Assembler::xaddq(Address dst, Register src) {
13610   InstructionMark im(this);
13611   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13612   emit_operand(src, dst);
13613 }
13614 
13615 void Assembler::xchgq(Register dst, Address src) {
13616   InstructionMark im(this);
13617   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13618   emit_operand(dst, src);
13619 }
13620 
13621 void Assembler::xchgq(Register dst, Register src) {
13622   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13623   emit_int16((unsigned char)0x87, (0xc0 | encode));
13624 }
13625 
13626 void Assembler::xorq(Register dst, Register src) {
13627   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13628   emit_arith(0x33, 0xC0, dst, src);
13629 }
13630 
13631 void Assembler::xorq(Register dst, Address src) {
13632   InstructionMark im(this);
13633   emit_int16(get_prefixq(src, dst), 0x33);
13634   emit_operand(dst, src);
13635 }
13636 
13637 void Assembler::xorq(Register dst, int32_t imm32) {
13638   (void) prefixq_and_encode(dst->encoding());
13639   emit_arith(0x81, 0xF0, dst, imm32);
13640 }
13641 
13642 void Assembler::xorq(Address dst, int32_t imm32) {
13643   InstructionMark im(this);
13644   prefixq(dst);
13645   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13646 }
13647 
13648 void Assembler::xorq(Address dst, Register src) {
13649   InstructionMark im(this);
13650   emit_int16(get_prefixq(dst, src), 0x31);
13651   emit_operand(src, dst);
13652 }
13653 
13654 #endif // !LP64
13655 
13656 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13657   if (VM_Version::supports_evex()) {
13658     _tuple_type = tuple_type;
13659     _input_size_in_bits = input_size_in_bits;
13660   }
13661 }