1 /*
    2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec = RelocationHolder::none;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert(op1 == 0x81, "Unexpected opcode");
  304   if (is8bit(imm32)) {
  305     emit_int24(op1 | 0x02,        // set sign bit
  306                op2 | encode(dst),
  307                imm32 & 0xFF);
  308   } else if (dst == rax) {
  309     switch (op2) {
  310       case 0xD0: emit_int8(0x15); break; // adc
  311       case 0xC0: emit_int8(0x05); break; // add
  312       case 0xE0: emit_int8(0x25); break; // and
  313       case 0xF8: emit_int8(0x3D); break; // cmp
  314       case 0xC8: emit_int8(0x0D); break; // or
  315       case 0xD8: emit_int8(0x1D); break; // sbb
  316       case 0xE8: emit_int8(0x2D); break; // sub
  317       case 0xF0: emit_int8(0x35); break; // xor
  318       default: ShouldNotReachHere();
  319     }
  320     emit_int32(imm32);
  321   } else {
  322     emit_int16(op1, (op2 | encode(dst)));
  323     emit_int32(imm32);
  324   }
  325 }
  326 
  327 // Force generation of a 4 byte immediate value even if it fits into 8bit
  328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  329   assert(isByte(op1) && isByte(op2), "wrong opcode");
  330   assert((op1 & 0x01) == 1, "should be 32bit operation");
  331   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  332   emit_int16(op1, (op2 | encode(dst)));
  333   emit_int32(imm32);
  334 }
  335 
  336 // immediate-to-memory forms
  337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  338   assert((op1 & 0x01) == 1, "should be 32bit operation");
  339   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  340   if (is8bit(imm32)) {
  341     emit_int8(op1 | 0x02); // set sign bit
  342     emit_operand(rm, adr, 1);
  343     emit_int8(imm32 & 0xFF);
  344   } else {
  345     emit_int8(op1);
  346     emit_operand(rm, adr, 4);
  347     emit_int32(imm32);
  348   }
  349 }
  350 
  351 
  352 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  353   assert(isByte(op1) && isByte(op2), "wrong opcode");
  354   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  355 }
  356 
  357 
  358 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  359                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  360   int mod_idx = 0;
  361   // We will test if the displacement fits the compressed format and if so
  362   // apply the compression to the displacement iff the result is8bit.
  363   if (VM_Version::supports_evex() && is_evex_inst) {
  364     switch (cur_tuple_type) {
  365     case EVEX_FV:
  366       if ((cur_encoding & VEX_W) == VEX_W) {
  367         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  368       } else {
  369         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  370       }
  371       break;
  372 
  373     case EVEX_HV:
  374       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  375       break;
  376 
  377     case EVEX_FVM:
  378       break;
  379 
  380     case EVEX_T1S:
  381       switch (in_size_in_bits) {
  382       case EVEX_8bit:
  383         break;
  384 
  385       case EVEX_16bit:
  386         mod_idx = 1;
  387         break;
  388 
  389       case EVEX_32bit:
  390         mod_idx = 2;
  391         break;
  392 
  393       case EVEX_64bit:
  394         mod_idx = 3;
  395         break;
  396       }
  397       break;
  398 
  399     case EVEX_T1F:
  400     case EVEX_T2:
  401     case EVEX_T4:
  402       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  403       break;
  404 
  405     case EVEX_T8:
  406       break;
  407 
  408     case EVEX_HVM:
  409       break;
  410 
  411     case EVEX_QVM:
  412       break;
  413 
  414     case EVEX_OVM:
  415       break;
  416 
  417     case EVEX_M128:
  418       break;
  419 
  420     case EVEX_DUP:
  421       break;
  422 
  423     default:
  424       assert(0, "no valid evex tuple_table entry");
  425       break;
  426     }
  427 
  428     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  429       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  430       if ((disp % disp_factor) == 0) {
  431         int new_disp = disp / disp_factor;
  432         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  433           disp = new_disp;
  434         }
  435       } else {
  436         return false;
  437       }
  438     }
  439   }
  440   return (-0x80 <= disp && disp < 0x80);
  441 }
  442 
  443 
  444 bool Assembler::emit_compressed_disp_byte(int &disp) {
  445   int mod_idx = 0;
  446   // We will test if the displacement fits the compressed format and if so
  447   // apply the compression to the displacement iff the result is8bit.
  448   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  449     int evex_encoding = _attributes->get_evex_encoding();
  450     int tuple_type = _attributes->get_tuple_type();
  451     switch (tuple_type) {
  452     case EVEX_FV:
  453       if ((evex_encoding & VEX_W) == VEX_W) {
  454         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  455       } else {
  456         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  457       }
  458       break;
  459 
  460     case EVEX_HV:
  461       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  462       break;
  463 
  464     case EVEX_FVM:
  465       break;
  466 
  467     case EVEX_T1S:
  468       switch (_attributes->get_input_size()) {
  469       case EVEX_8bit:
  470         break;
  471 
  472       case EVEX_16bit:
  473         mod_idx = 1;
  474         break;
  475 
  476       case EVEX_32bit:
  477         mod_idx = 2;
  478         break;
  479 
  480       case EVEX_64bit:
  481         mod_idx = 3;
  482         break;
  483       }
  484       break;
  485 
  486     case EVEX_T1F:
  487     case EVEX_T2:
  488     case EVEX_T4:
  489       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  490       break;
  491 
  492     case EVEX_T8:
  493       break;
  494 
  495     case EVEX_HVM:
  496       break;
  497 
  498     case EVEX_QVM:
  499       break;
  500 
  501     case EVEX_OVM:
  502       break;
  503 
  504     case EVEX_M128:
  505       break;
  506 
  507     case EVEX_DUP:
  508       break;
  509 
  510     default:
  511       assert(0, "no valid evex tuple_table entry");
  512       break;
  513     }
  514 
  515     int vector_len = _attributes->get_vector_len();
  516     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  517       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  518       if ((disp % disp_factor) == 0) {
  519         int new_disp = disp / disp_factor;
  520         if (is8bit(new_disp)) {
  521           disp = new_disp;
  522         }
  523       } else {
  524         return false;
  525       }
  526     }
  527   }
  528   return is8bit(disp);
  529 }
  530 
  531 static bool is_valid_encoding(int reg_enc) {
  532   return reg_enc >= 0;
  533 }
  534 
  535 static int raw_encode(Register reg) {
  536   assert(reg == noreg || reg->is_valid(), "sanity");
  537   int reg_enc = (intptr_t)reg;
  538   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  539   return reg_enc;
  540 }
  541 
  542 static int raw_encode(XMMRegister xmmreg) {
  543   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  544   int xmmreg_enc = (intptr_t)xmmreg;
  545   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  546   return xmmreg_enc;
  547 }
  548 
  549 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  550   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  551 }
  552 
  553 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  554   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  555 }
  556 
  557 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  558   assert((mod & 3) != 0b11, "forbidden");
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   emit_int8(modrm);
  561 }
  562 
  563 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  564                                         int disp) {
  565   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  566   emit_int16(modrm, disp & 0xFF);
  567 }
  568 
  569 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  570                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  571   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  572   int sib = sib_encoding(scale, index_enc, base_enc);
  573   emit_int16(modrm, sib);
  574 }
  575 
  576 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  577                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  578                                             int disp) {
  579   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  580   int sib = sib_encoding(scale, index_enc, base_enc);
  581   emit_int24(modrm, sib, disp & 0xFF);
  582 }
  583 
  584 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  585                                     Address::ScaleFactor scale, int disp,
  586                                     RelocationHolder const& rspec,
  587                                     int rip_relative_correction) {
  588   bool no_relocation = (rspec.type() == relocInfo::none);
  589 
  590   if (is_valid_encoding(base_enc)) {
  591     if (is_valid_encoding(index_enc)) {
  592       assert(scale != Address::no_scale, "inconsistent address");
  593       // [base + index*scale + disp]
  594       if (disp == 0 && no_relocation &&
  595           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  596         // [base + index*scale]
  597         // [00 reg 100][ss index base]
  598         emit_modrm_sib(0b00, reg_enc, 0b100,
  599                        scale, index_enc, base_enc);
  600       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  601         // [base + index*scale + imm8]
  602         // [01 reg 100][ss index base] imm8
  603         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  604                              scale, index_enc, base_enc,
  605                              disp);
  606       } else {
  607         // [base + index*scale + disp32]
  608         // [10 reg 100][ss index base] disp32
  609         emit_modrm_sib(0b10, reg_enc, 0b100,
  610                        scale, index_enc, base_enc);
  611         emit_data(disp, rspec, disp32_operand);
  612       }
  613     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  614       // [rsp + disp]
  615       if (disp == 0 && no_relocation) {
  616         // [rsp]
  617         // [00 reg 100][00 100 100]
  618         emit_modrm_sib(0b00, reg_enc, 0b100,
  619                        Address::times_1, 0b100, 0b100);
  620       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  621         // [rsp + imm8]
  622         // [01 reg 100][00 100 100] disp8
  623         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  624                              Address::times_1, 0b100, 0b100,
  625                              disp);
  626       } else {
  627         // [rsp + imm32]
  628         // [10 reg 100][00 100 100] disp32
  629         emit_modrm_sib(0b10, reg_enc, 0b100,
  630                        Address::times_1, 0b100, 0b100);
  631         emit_data(disp, rspec, disp32_operand);
  632       }
  633     } else {
  634       // [base + disp]
  635       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  636       if (disp == 0 && no_relocation &&
  637           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  638         // [base]
  639         // [00 reg base]
  640         emit_modrm(0, reg_enc, base_enc);
  641       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  642         // [base + disp8]
  643         // [01 reg base] disp8
  644         emit_modrm_disp8(0b01, reg_enc, base_enc,
  645                          disp);
  646       } else {
  647         // [base + disp32]
  648         // [10 reg base] disp32
  649         emit_modrm(0b10, reg_enc, base_enc);
  650         emit_data(disp, rspec, disp32_operand);
  651       }
  652     }
  653   } else {
  654     if (is_valid_encoding(index_enc)) {
  655       assert(scale != Address::no_scale, "inconsistent address");
  656       // base == noreg
  657       // [index*scale + disp]
  658       // [00 reg 100][ss index 101] disp32
  659       emit_modrm_sib(0b00, reg_enc, 0b100,
  660                      scale, index_enc, 0b101 /* no base */);
  661       emit_data(disp, rspec, disp32_operand);
  662     } else if (!no_relocation) {
  663       // base == noreg, index == noreg
  664       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  665       // [00 reg 101] disp32
  666 
  667       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  668       // Note that the RIP-rel. correction applies to the generated
  669       // disp field, but _not_ to the target address in the rspec.
  670 
  671       // disp was created by converting the target address minus the pc
  672       // at the start of the instruction. That needs more correction here.
  673       // intptr_t disp = target - next_ip;
  674       assert(inst_mark() != NULL, "must be inside InstructionMark");
  675       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  676       int64_t adjusted = disp;
  677       // Do rip-rel adjustment for 64bit
  678       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  679       assert(is_simm32(adjusted),
  680              "must be 32bit offset (RIP relative address)");
  681       emit_data((int32_t) adjusted, rspec, disp32_operand);
  682 
  683     } else {
  684       // base == noreg, index == noreg, no_relocation == true
  685       // 32bit never did this, did everything as the rip-rel/disp code above
  686       // [disp] ABSOLUTE
  687       // [00 reg 100][00 100 101] disp32
  688       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  689                      Address::times_1, 0b100, 0b101);
  690       emit_data(disp, rspec, disp32_operand);
  691     }
  692   }
  693 }
  694 
  695 void Assembler::emit_operand(Register reg, Register base, Register index,
  696                              Address::ScaleFactor scale, int disp,
  697                              RelocationHolder const& rspec,
  698                              int rip_relative_correction) {
  699   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  700   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  701                       scale, disp, rspec, rip_relative_correction);
  702 
  703 }
  704 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  705                              Address::ScaleFactor scale, int disp,
  706                              RelocationHolder const& rspec) {
  707   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  708   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  709   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  710                       scale, disp, rspec);
  711 }
  712 
  713 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  714                              Address::ScaleFactor scale, int disp,
  715                              RelocationHolder const& rspec) {
  716   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  717   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  718   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  719                       scale, disp, rspec, /* rip_relative_correction */ 0);
  720 }
  721 
  722 // Secret local extension to Assembler::WhichOperand:
  723 #define end_pc_operand (_WhichOperand_limit)
  724 
  725 address Assembler::locate_operand(address inst, WhichOperand which) {
  726   // Decode the given instruction, and return the address of
  727   // an embedded 32-bit operand word.
  728 
  729   // If "which" is disp32_operand, selects the displacement portion
  730   // of an effective address specifier.
  731   // If "which" is imm64_operand, selects the trailing immediate constant.
  732   // If "which" is call32_operand, selects the displacement of a call or jump.
  733   // Caller is responsible for ensuring that there is such an operand,
  734   // and that it is 32/64 bits wide.
  735 
  736   // If "which" is end_pc_operand, find the end of the instruction.
  737 
  738   address ip = inst;
  739   bool is_64bit = false;
  740 
  741   debug_only(bool has_disp32 = false);
  742   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  743 
  744   again_after_prefix:
  745   switch (0xFF & *ip++) {
  746 
  747   // These convenience macros generate groups of "case" labels for the switch.
  748 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  749 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  750              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  751 #define REP16(x) REP8((x)+0): \
  752               case REP8((x)+8)
  753 
  754   case CS_segment:
  755   case SS_segment:
  756   case DS_segment:
  757   case ES_segment:
  758   case FS_segment:
  759   case GS_segment:
  760     // Seems dubious
  761     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  762     assert(ip == inst+1, "only one prefix allowed");
  763     goto again_after_prefix;
  764 
  765   case 0x67:
  766   case REX:
  767   case REX_B:
  768   case REX_X:
  769   case REX_XB:
  770   case REX_R:
  771   case REX_RB:
  772   case REX_RX:
  773   case REX_RXB:
  774     NOT_LP64(assert(false, "64bit prefixes"));
  775     goto again_after_prefix;
  776 
  777   case REX_W:
  778   case REX_WB:
  779   case REX_WX:
  780   case REX_WXB:
  781   case REX_WR:
  782   case REX_WRB:
  783   case REX_WRX:
  784   case REX_WRXB:
  785     NOT_LP64(assert(false, "64bit prefixes"));
  786     is_64bit = true;
  787     goto again_after_prefix;
  788 
  789   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  790   case 0x88: // movb a, r
  791   case 0x89: // movl a, r
  792   case 0x8A: // movb r, a
  793   case 0x8B: // movl r, a
  794   case 0x8F: // popl a
  795     debug_only(has_disp32 = true);
  796     break;
  797 
  798   case 0x68: // pushq #32
  799     if (which == end_pc_operand) {
  800       return ip + 4;
  801     }
  802     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  803     return ip;                  // not produced by emit_operand
  804 
  805   case 0x66: // movw ... (size prefix)
  806     again_after_size_prefix2:
  807     switch (0xFF & *ip++) {
  808     case REX:
  809     case REX_B:
  810     case REX_X:
  811     case REX_XB:
  812     case REX_R:
  813     case REX_RB:
  814     case REX_RX:
  815     case REX_RXB:
  816     case REX_W:
  817     case REX_WB:
  818     case REX_WX:
  819     case REX_WXB:
  820     case REX_WR:
  821     case REX_WRB:
  822     case REX_WRX:
  823     case REX_WRXB:
  824       NOT_LP64(assert(false, "64bit prefix found"));
  825       goto again_after_size_prefix2;
  826     case 0x8B: // movw r, a
  827     case 0x89: // movw a, r
  828       debug_only(has_disp32 = true);
  829       break;
  830     case 0xC7: // movw a, #16
  831       debug_only(has_disp32 = true);
  832       tail_size = 2;  // the imm16
  833       break;
  834     case 0x0F: // several SSE/SSE2 variants
  835       ip--;    // reparse the 0x0F
  836       goto again_after_prefix;
  837     default:
  838       ShouldNotReachHere();
  839     }
  840     break;
  841 
  842   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  843     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  844     // these asserts are somewhat nonsensical
  845 #ifndef _LP64
  846     assert(which == imm_operand || which == disp32_operand,
  847            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  848 #else
  849     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  850            which == narrow_oop_operand && !is_64bit,
  851            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  852 #endif // _LP64
  853     return ip;
  854 
  855   case 0x69: // imul r, a, #32
  856   case 0xC7: // movl a, #32(oop?)
  857     tail_size = 4;
  858     debug_only(has_disp32 = true); // has both kinds of operands!
  859     break;
  860 
  861   case 0x0F: // movx..., etc.
  862     switch (0xFF & *ip++) {
  863     case 0x3A: // pcmpestri
  864       tail_size = 1;
  865     case 0x38: // ptest, pmovzxbw
  866       ip++; // skip opcode
  867       debug_only(has_disp32 = true); // has both kinds of operands!
  868       break;
  869 
  870     case 0x70: // pshufd r, r/a, #8
  871       debug_only(has_disp32 = true); // has both kinds of operands!
  872     case 0x73: // psrldq r, #8
  873       tail_size = 1;
  874       break;
  875 
  876     case 0x12: // movlps
  877     case 0x28: // movaps
  878     case 0x2E: // ucomiss
  879     case 0x2F: // comiss
  880     case 0x54: // andps
  881     case 0x55: // andnps
  882     case 0x56: // orps
  883     case 0x57: // xorps
  884     case 0x58: // addpd
  885     case 0x59: // mulpd
  886     case 0x6E: // movd
  887     case 0x7E: // movd
  888     case 0x6F: // movdq
  889     case 0x7F: // movdq
  890     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  891     case 0xFE: // paddd
  892       debug_only(has_disp32 = true);
  893       break;
  894 
  895     case 0xAD: // shrd r, a, %cl
  896     case 0xAF: // imul r, a
  897     case 0xBE: // movsbl r, a (movsxb)
  898     case 0xBF: // movswl r, a (movsxw)
  899     case 0xB6: // movzbl r, a (movzxb)
  900     case 0xB7: // movzwl r, a (movzxw)
  901     case REP16(0x40): // cmovl cc, r, a
  902     case 0xB0: // cmpxchgb
  903     case 0xB1: // cmpxchg
  904     case 0xC1: // xaddl
  905     case 0xC7: // cmpxchg8
  906     case REP16(0x90): // setcc a
  907       debug_only(has_disp32 = true);
  908       // fall out of the switch to decode the address
  909       break;
  910 
  911     case 0xC4: // pinsrw r, a, #8
  912       debug_only(has_disp32 = true);
  913     case 0xC5: // pextrw r, r, #8
  914       tail_size = 1;  // the imm8
  915       break;
  916 
  917     case 0xAC: // shrd r, a, #8
  918       debug_only(has_disp32 = true);
  919       tail_size = 1;  // the imm8
  920       break;
  921 
  922     case REP16(0x80): // jcc rdisp32
  923       if (which == end_pc_operand)  return ip + 4;
  924       assert(which == call32_operand, "jcc has no disp32 or imm");
  925       return ip;
  926     default:
  927       ShouldNotReachHere();
  928     }
  929     break;
  930 
  931   case 0x81: // addl a, #32; addl r, #32
  932     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  933     // on 32bit in the case of cmpl, the imm might be an oop
  934     tail_size = 4;
  935     debug_only(has_disp32 = true); // has both kinds of operands!
  936     break;
  937 
  938   case 0x83: // addl a, #8; addl r, #8
  939     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  940     debug_only(has_disp32 = true); // has both kinds of operands!
  941     tail_size = 1;
  942     break;
  943 
  944   case 0x15: // adc rax, #32
  945   case 0x05: // add rax, #32
  946   case 0x25: // and rax, #32
  947   case 0x3D: // cmp rax, #32
  948   case 0x0D: // or  rax, #32
  949   case 0x1D: // sbb rax, #32
  950   case 0x2D: // sub rax, #32
  951   case 0x35: // xor rax, #32
  952     return which == end_pc_operand ? ip + 4 : ip;
  953 
  954   case 0x9B:
  955     switch (0xFF & *ip++) {
  956     case 0xD9: // fnstcw a
  957       debug_only(has_disp32 = true);
  958       break;
  959     default:
  960       ShouldNotReachHere();
  961     }
  962     break;
  963 
  964   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  965   case REP4(0x10): // adc...
  966   case REP4(0x20): // and...
  967   case REP4(0x30): // xor...
  968   case REP4(0x08): // or...
  969   case REP4(0x18): // sbb...
  970   case REP4(0x28): // sub...
  971   case 0xF7: // mull a
  972   case 0x8D: // lea r, a
  973   case 0x87: // xchg r, a
  974   case REP4(0x38): // cmp...
  975   case 0x85: // test r, a
  976     debug_only(has_disp32 = true); // has both kinds of operands!
  977     break;
  978 
  979   case 0xA8: // testb rax, #8
  980     return which == end_pc_operand ? ip + 1 : ip;
  981   case 0xA9: // testl/testq rax, #32
  982     return which == end_pc_operand ? ip + 4 : ip;
  983 
  984   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  985   case 0xC6: // movb a, #8
  986   case 0x80: // cmpb a, #8
  987   case 0x6B: // imul r, a, #8
  988     debug_only(has_disp32 = true); // has both kinds of operands!
  989     tail_size = 1; // the imm8
  990     break;
  991 
  992   case 0xC4: // VEX_3bytes
  993   case 0xC5: // VEX_2bytes
  994     assert((UseAVX > 0), "shouldn't have VEX prefix");
  995     assert(ip == inst+1, "no prefixes allowed");
  996     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  997     // but they have prefix 0x0F and processed when 0x0F processed above.
  998     //
  999     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1000     // instructions (these instructions are not supported in 64-bit mode).
 1001     // To distinguish them bits [7:6] are set in the VEX second byte since
 1002     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1003     // those VEX bits REX and vvvv bits are inverted.
 1004     //
 1005     // Fortunately C2 doesn't generate these instructions so we don't need
 1006     // to check for them in product version.
 1007 
 1008     // Check second byte
 1009     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1010 
 1011     int vex_opcode;
 1012     // First byte
 1013     if ((0xFF & *inst) == VEX_3bytes) {
 1014       vex_opcode = VEX_OPCODE_MASK & *ip;
 1015       ip++; // third byte
 1016       is_64bit = ((VEX_W & *ip) == VEX_W);
 1017     } else {
 1018       vex_opcode = VEX_OPCODE_0F;
 1019     }
 1020     ip++; // opcode
 1021     // To find the end of instruction (which == end_pc_operand).
 1022     switch (vex_opcode) {
 1023       case VEX_OPCODE_0F:
 1024         switch (0xFF & *ip) {
 1025         case 0x70: // pshufd r, r/a, #8
 1026         case 0x71: // ps[rl|ra|ll]w r, #8
 1027         case 0x72: // ps[rl|ra|ll]d r, #8
 1028         case 0x73: // ps[rl|ra|ll]q r, #8
 1029         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1030         case 0xC4: // pinsrw r, r, r/a, #8
 1031         case 0xC5: // pextrw r/a, r, #8
 1032         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1033           tail_size = 1;  // the imm8
 1034           break;
 1035         }
 1036         break;
 1037       case VEX_OPCODE_0F_3A:
 1038         tail_size = 1;
 1039         break;
 1040     }
 1041     ip++; // skip opcode
 1042     debug_only(has_disp32 = true); // has both kinds of operands!
 1043     break;
 1044 
 1045   case 0x62: // EVEX_4bytes
 1046     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1047     assert(ip == inst+1, "no prefixes allowed");
 1048     // no EVEX collisions, all instructions that have 0x62 opcodes
 1049     // have EVEX versions and are subopcodes of 0x66
 1050     ip++; // skip P0 and examine W in P1
 1051     is_64bit = ((VEX_W & *ip) == VEX_W);
 1052     ip++; // move to P2
 1053     ip++; // skip P2, move to opcode
 1054     // To find the end of instruction (which == end_pc_operand).
 1055     switch (0xFF & *ip) {
 1056     case 0x22: // pinsrd r, r/a, #8
 1057     case 0x61: // pcmpestri r, r/a, #8
 1058     case 0x70: // pshufd r, r/a, #8
 1059     case 0x73: // psrldq r, #8
 1060     case 0x1f: // evpcmpd/evpcmpq
 1061     case 0x3f: // evpcmpb/evpcmpw
 1062       tail_size = 1;  // the imm8
 1063       break;
 1064     default:
 1065       break;
 1066     }
 1067     ip++; // skip opcode
 1068     debug_only(has_disp32 = true); // has both kinds of operands!
 1069     break;
 1070 
 1071   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1072   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1073   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1074   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1075   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1076   case 0xDF: // fild_d a; fistp_d a
 1077   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1078   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1079   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1080     debug_only(has_disp32 = true);
 1081     break;
 1082 
 1083   case 0xE8: // call rdisp32
 1084   case 0xE9: // jmp  rdisp32
 1085     if (which == end_pc_operand)  return ip + 4;
 1086     assert(which == call32_operand, "call has no disp32 or imm");
 1087     return ip;
 1088 
 1089   case 0xF0:                    // Lock
 1090     goto again_after_prefix;
 1091 
 1092   case 0xF3:                    // For SSE
 1093   case 0xF2:                    // For SSE2
 1094     switch (0xFF & *ip++) {
 1095     case REX:
 1096     case REX_B:
 1097     case REX_X:
 1098     case REX_XB:
 1099     case REX_R:
 1100     case REX_RB:
 1101     case REX_RX:
 1102     case REX_RXB:
 1103     case REX_W:
 1104     case REX_WB:
 1105     case REX_WX:
 1106     case REX_WXB:
 1107     case REX_WR:
 1108     case REX_WRB:
 1109     case REX_WRX:
 1110     case REX_WRXB:
 1111       NOT_LP64(assert(false, "found 64bit prefix"));
 1112       ip++;
 1113     default:
 1114       ip++;
 1115     }
 1116     debug_only(has_disp32 = true); // has both kinds of operands!
 1117     break;
 1118 
 1119   default:
 1120     ShouldNotReachHere();
 1121 
 1122 #undef REP8
 1123 #undef REP16
 1124   }
 1125 
 1126   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1127 #ifdef _LP64
 1128   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1129 #else
 1130   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1131   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1132 #endif // LP64
 1133   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1134 
 1135   // parse the output of emit_operand
 1136   int op2 = 0xFF & *ip++;
 1137   int base = op2 & 0x07;
 1138   int op3 = -1;
 1139   const int b100 = 4;
 1140   const int b101 = 5;
 1141   if (base == b100 && (op2 >> 6) != 3) {
 1142     op3 = 0xFF & *ip++;
 1143     base = op3 & 0x07;   // refetch the base
 1144   }
 1145   // now ip points at the disp (if any)
 1146 
 1147   switch (op2 >> 6) {
 1148   case 0:
 1149     // [00 reg  100][ss index base]
 1150     // [00 reg  100][00   100  esp]
 1151     // [00 reg base]
 1152     // [00 reg  100][ss index  101][disp32]
 1153     // [00 reg  101]               [disp32]
 1154 
 1155     if (base == b101) {
 1156       if (which == disp32_operand)
 1157         return ip;              // caller wants the disp32
 1158       ip += 4;                  // skip the disp32
 1159     }
 1160     break;
 1161 
 1162   case 1:
 1163     // [01 reg  100][ss index base][disp8]
 1164     // [01 reg  100][00   100  esp][disp8]
 1165     // [01 reg base]               [disp8]
 1166     ip += 1;                    // skip the disp8
 1167     break;
 1168 
 1169   case 2:
 1170     // [10 reg  100][ss index base][disp32]
 1171     // [10 reg  100][00   100  esp][disp32]
 1172     // [10 reg base]               [disp32]
 1173     if (which == disp32_operand)
 1174       return ip;                // caller wants the disp32
 1175     ip += 4;                    // skip the disp32
 1176     break;
 1177 
 1178   case 3:
 1179     // [11 reg base]  (not a memory addressing mode)
 1180     break;
 1181   }
 1182 
 1183   if (which == end_pc_operand) {
 1184     return ip + tail_size;
 1185   }
 1186 
 1187 #ifdef _LP64
 1188   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1189 #else
 1190   assert(which == imm_operand, "instruction has only an imm field");
 1191 #endif // LP64
 1192   return ip;
 1193 }
 1194 
 1195 address Assembler::locate_next_instruction(address inst) {
 1196   // Secretly share code with locate_operand:
 1197   return locate_operand(inst, end_pc_operand);
 1198 }
 1199 
 1200 
 1201 #ifdef ASSERT
 1202 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1203   address inst = inst_mark();
 1204   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1205   address opnd;
 1206 
 1207   Relocation* r = rspec.reloc();
 1208   if (r->type() == relocInfo::none) {
 1209     return;
 1210   } else if (r->is_call() || format == call32_operand) {
 1211     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1212     opnd = locate_operand(inst, call32_operand);
 1213   } else if (r->is_data()) {
 1214     assert(format == imm_operand || format == disp32_operand
 1215            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1216     opnd = locate_operand(inst, (WhichOperand)format);
 1217   } else {
 1218     assert(format == imm_operand, "cannot specify a format");
 1219     return;
 1220   }
 1221   assert(opnd == pc(), "must put operand where relocs can find it");
 1222 }
 1223 #endif // ASSERT
 1224 
 1225 void Assembler::emit_operand(Register reg, Address adr,
 1226                              int rip_relative_correction) {
 1227   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1228                adr._rspec,
 1229                rip_relative_correction);
 1230 }
 1231 
 1232 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1233     if (adr.isxmmindex()) {
 1234        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1235     } else {
 1236        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1237        adr._rspec);
 1238     }
 1239 }
 1240 
 1241 // Now the Assembler instructions (identical for 32/64 bits)
 1242 
 1243 void Assembler::adcl(Address dst, int32_t imm32) {
 1244   InstructionMark im(this);
 1245   prefix(dst);
 1246   emit_arith_operand(0x81, rdx, dst, imm32);
 1247 }
 1248 
 1249 void Assembler::adcl(Address dst, Register src) {
 1250   InstructionMark im(this);
 1251   prefix(dst, src);
 1252   emit_int8(0x11);
 1253   emit_operand(src, dst);
 1254 }
 1255 
 1256 void Assembler::adcl(Register dst, int32_t imm32) {
 1257   prefix(dst);
 1258   emit_arith(0x81, 0xD0, dst, imm32);
 1259 }
 1260 
 1261 void Assembler::adcl(Register dst, Address src) {
 1262   InstructionMark im(this);
 1263   prefix(src, dst);
 1264   emit_int8(0x13);
 1265   emit_operand(dst, src);
 1266 }
 1267 
 1268 void Assembler::adcl(Register dst, Register src) {
 1269   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1270   emit_arith(0x13, 0xC0, dst, src);
 1271 }
 1272 
 1273 void Assembler::addl(Address dst, int32_t imm32) {
 1274   InstructionMark im(this);
 1275   prefix(dst);
 1276   emit_arith_operand(0x81, rax, dst, imm32);
 1277 }
 1278 
 1279 void Assembler::addb(Address dst, int imm8) {
 1280   InstructionMark im(this);
 1281   prefix(dst);
 1282   emit_int8((unsigned char)0x80);
 1283   emit_operand(rax, dst, 1);
 1284   emit_int8(imm8);
 1285 }
 1286 
 1287 void Assembler::addw(Register dst, Register src) {
 1288   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1289   emit_arith(0x03, 0xC0, dst, src);
 1290 }
 1291 
 1292 void Assembler::addw(Address dst, int imm16) {
 1293   InstructionMark im(this);
 1294   emit_int8(0x66);
 1295   prefix(dst);
 1296   emit_int8((unsigned char)0x81);
 1297   emit_operand(rax, dst, 2);
 1298   emit_int16(imm16);
 1299 }
 1300 
 1301 void Assembler::addl(Address dst, Register src) {
 1302   InstructionMark im(this);
 1303   prefix(dst, src);
 1304   emit_int8(0x01);
 1305   emit_operand(src, dst);
 1306 }
 1307 
 1308 void Assembler::addl(Register dst, int32_t imm32) {
 1309   prefix(dst);
 1310   emit_arith(0x81, 0xC0, dst, imm32);
 1311 }
 1312 
 1313 void Assembler::addl(Register dst, Address src) {
 1314   InstructionMark im(this);
 1315   prefix(src, dst);
 1316   emit_int8(0x03);
 1317   emit_operand(dst, src);
 1318 }
 1319 
 1320 void Assembler::addl(Register dst, Register src) {
 1321   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1322   emit_arith(0x03, 0xC0, dst, src);
 1323 }
 1324 
 1325 void Assembler::addr_nop_4() {
 1326   assert(UseAddressNop, "no CPU support");
 1327   // 4 bytes: NOP DWORD PTR [EAX+0]
 1328   emit_int32(0x0F,
 1329              0x1F,
 1330              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1331              0);   // 8-bits offset (1 byte)
 1332 }
 1333 
 1334 void Assembler::addr_nop_5() {
 1335   assert(UseAddressNop, "no CPU support");
 1336   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1337   emit_int32(0x0F,
 1338              0x1F,
 1339              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1340              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1341   emit_int8(0);     // 8-bits offset (1 byte)
 1342 }
 1343 
 1344 void Assembler::addr_nop_7() {
 1345   assert(UseAddressNop, "no CPU support");
 1346   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1347   emit_int24(0x0F,
 1348              0x1F,
 1349              (unsigned char)0x80);
 1350                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1351   emit_int32(0);   // 32-bits offset (4 bytes)
 1352 }
 1353 
 1354 void Assembler::addr_nop_8() {
 1355   assert(UseAddressNop, "no CPU support");
 1356   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1357   emit_int32(0x0F,
 1358              0x1F,
 1359              (unsigned char)0x84,
 1360                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1361              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1362   emit_int32(0);    // 32-bits offset (4 bytes)
 1363 }
 1364 
 1365 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_rex_vex_w_reverted();
 1369   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1370   emit_int16(0x58, (0xC0 | encode));
 1371 }
 1372 
 1373 void Assembler::addsd(XMMRegister dst, Address src) {
 1374   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1375   InstructionMark im(this);
 1376   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1377   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1378   attributes.set_rex_vex_w_reverted();
 1379   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1380   emit_int8(0x58);
 1381   emit_operand(dst, src);
 1382 }
 1383 
 1384 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1385   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1387   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1388   emit_int16(0x58, (0xC0 | encode));
 1389 }
 1390 
 1391 void Assembler::addss(XMMRegister dst, Address src) {
 1392   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1393   InstructionMark im(this);
 1394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1395   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1396   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1397   emit_int8(0x58);
 1398   emit_operand(dst, src);
 1399 }
 1400 
 1401 void Assembler::aesdec(XMMRegister dst, Address src) {
 1402   assert(VM_Version::supports_aes(), "");
 1403   InstructionMark im(this);
 1404   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1405   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1406   emit_int8((unsigned char)0xDE);
 1407   emit_operand(dst, src);
 1408 }
 1409 
 1410 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1411   assert(VM_Version::supports_aes(), "");
 1412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1413   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1414   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1415 }
 1416 
 1417 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1418   assert(VM_Version::supports_avx512_vaes(), "");
 1419   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1420   attributes.set_is_evex_instruction();
 1421   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1422   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1423 }
 1424 
 1425 
 1426 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1427   assert(VM_Version::supports_aes(), "");
 1428   InstructionMark im(this);
 1429   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1430   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1431   emit_int8((unsigned char)0xDF);
 1432   emit_operand(dst, src);
 1433 }
 1434 
 1435 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1436   assert(VM_Version::supports_aes(), "");
 1437   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1438   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1439   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1440 }
 1441 
 1442 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1443   assert(VM_Version::supports_avx512_vaes(), "");
 1444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1445   attributes.set_is_evex_instruction();
 1446   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1447   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1448 }
 1449 
 1450 void Assembler::aesenc(XMMRegister dst, Address src) {
 1451   assert(VM_Version::supports_aes(), "");
 1452   InstructionMark im(this);
 1453   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1454   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1455   emit_int8((unsigned char)0xDC);
 1456   emit_operand(dst, src);
 1457 }
 1458 
 1459 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1460   assert(VM_Version::supports_aes(), "");
 1461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1462   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1463   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1464 }
 1465 
 1466 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1467   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1469   attributes.set_is_evex_instruction();
 1470   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1471   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1472 }
 1473 
 1474 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1475   assert(VM_Version::supports_aes(), "");
 1476   InstructionMark im(this);
 1477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1478   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1479   emit_int8((unsigned char)0xDD);
 1480   emit_operand(dst, src);
 1481 }
 1482 
 1483 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1484   assert(VM_Version::supports_aes(), "");
 1485   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1486   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1487   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1488 }
 1489 
 1490 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1491   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1493   attributes.set_is_evex_instruction();
 1494   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1495   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1496 }
 1497 
 1498 void Assembler::andb(Address dst, Register src) {
 1499   InstructionMark im(this);
 1500   prefix(dst, src, true);
 1501   emit_int8(0x20);
 1502   emit_operand(src, dst);
 1503 }
 1504 
 1505 void Assembler::andw(Register dst, Register src) {
 1506   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1507   emit_arith(0x23, 0xC0, dst, src);
 1508 }
 1509 
 1510 void Assembler::andl(Address dst, int32_t imm32) {
 1511   InstructionMark im(this);
 1512   prefix(dst);
 1513   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1514 }
 1515 
 1516 void Assembler::andl(Register dst, int32_t imm32) {
 1517   prefix(dst);
 1518   emit_arith(0x81, 0xE0, dst, imm32);
 1519 }
 1520 
 1521 void Assembler::andl(Address dst, Register src) {
 1522   InstructionMark im(this);
 1523   prefix(dst, src);
 1524   emit_int8(0x21);
 1525   emit_operand(src, dst);
 1526 }
 1527 
 1528 void Assembler::andl(Register dst, Address src) {
 1529   InstructionMark im(this);
 1530   prefix(src, dst);
 1531   emit_int8(0x23);
 1532   emit_operand(dst, src);
 1533 }
 1534 
 1535 void Assembler::andl(Register dst, Register src) {
 1536   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1537   emit_arith(0x23, 0xC0, dst, src);
 1538 }
 1539 
 1540 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1541   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1543   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1544   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1545 }
 1546 
 1547 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1548   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1549   InstructionMark im(this);
 1550   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int8((unsigned char)0xF2);
 1553   emit_operand(dst, src2);
 1554 }
 1555 
 1556 void Assembler::bsfl(Register dst, Register src) {
 1557   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1558   emit_int24(0x0F,
 1559              (unsigned char)0xBC,
 1560              0xC0 | encode);
 1561 }
 1562 
 1563 void Assembler::bsrl(Register dst, Register src) {
 1564   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1565   emit_int24(0x0F,
 1566              (unsigned char)0xBD,
 1567              0xC0 | encode);
 1568 }
 1569 
 1570 void Assembler::bswapl(Register reg) { // bswap
 1571   int encode = prefix_and_encode(reg->encoding());
 1572   emit_int16(0x0F, (0xC8 | encode));
 1573 }
 1574 
 1575 void Assembler::blsil(Register dst, Register src) {
 1576   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1577   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1578   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1579   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1580 }
 1581 
 1582 void Assembler::blsil(Register dst, Address src) {
 1583   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1584   InstructionMark im(this);
 1585   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1586   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1587   emit_int8((unsigned char)0xF3);
 1588   emit_operand(rbx, src);
 1589 }
 1590 
 1591 void Assembler::blsmskl(Register dst, Register src) {
 1592   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1593   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1594   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1595   emit_int16((unsigned char)0xF3,
 1596              0xC0 | encode);
 1597 }
 1598 
 1599 void Assembler::blsmskl(Register dst, Address src) {
 1600   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1601   InstructionMark im(this);
 1602   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1603   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1604   emit_int8((unsigned char)0xF3);
 1605   emit_operand(rdx, src);
 1606 }
 1607 
 1608 void Assembler::blsrl(Register dst, Register src) {
 1609   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1610   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1611   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1612   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1613 }
 1614 
 1615 void Assembler::blsrl(Register dst, Address src) {
 1616   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1617   InstructionMark im(this);
 1618   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1619   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1620   emit_int8((unsigned char)0xF3);
 1621   emit_operand(rcx, src);
 1622 }
 1623 
 1624 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1625   // suspect disp32 is always good
 1626   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1627 
 1628   if (L.is_bound()) {
 1629     const int long_size = 5;
 1630     int offs = (int)( target(L) - pc() );
 1631     assert(offs <= 0, "assembler error");
 1632     InstructionMark im(this);
 1633     // 1110 1000 #32-bit disp
 1634     emit_int8((unsigned char)0xE8);
 1635     emit_data(offs - long_size, rtype, operand);
 1636   } else {
 1637     InstructionMark im(this);
 1638     // 1110 1000 #32-bit disp
 1639     L.add_patch_at(code(), locator());
 1640 
 1641     emit_int8((unsigned char)0xE8);
 1642     emit_data(int(0), rtype, operand);
 1643   }
 1644 }
 1645 
 1646 void Assembler::call(Register dst) {
 1647   int encode = prefix_and_encode(dst->encoding());
 1648   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1649 }
 1650 
 1651 
 1652 void Assembler::call(Address adr) {
 1653   InstructionMark im(this);
 1654   prefix(adr);
 1655   emit_int8((unsigned char)0xFF);
 1656   emit_operand(rdx, adr);
 1657 }
 1658 
 1659 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1660   InstructionMark im(this);
 1661   emit_int8((unsigned char)0xE8);
 1662   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1663   // Entry is NULL in case of a scratch emit.
 1664   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1665   // Technically, should use call32_operand, but this format is
 1666   // implied by the fact that we're emitting a call instruction.
 1667 
 1668   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1669   emit_data((int) disp, rspec, operand);
 1670 }
 1671 
 1672 void Assembler::cdql() {
 1673   emit_int8((unsigned char)0x99);
 1674 }
 1675 
 1676 void Assembler::cld() {
 1677   emit_int8((unsigned char)0xFC);
 1678 }
 1679 
 1680 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1681   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1682   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1683   emit_int24(0x0F,
 1684              0x40 | cc,
 1685              0xC0 | encode);
 1686 }
 1687 
 1688 
 1689 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1690   InstructionMark im(this);
 1691   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1692   prefix(src, dst);
 1693   emit_int16(0x0F, (0x40 | cc));
 1694   emit_operand(dst, src);
 1695 }
 1696 
 1697 void Assembler::cmpb(Address dst, int imm8) {
 1698   InstructionMark im(this);
 1699   prefix(dst);
 1700   emit_int8((unsigned char)0x80);
 1701   emit_operand(rdi, dst, 1);
 1702   emit_int8(imm8);
 1703 }
 1704 
 1705 void Assembler::cmpl(Address dst, int32_t imm32) {
 1706   InstructionMark im(this);
 1707   prefix(dst);
 1708   emit_int8((unsigned char)0x81);
 1709   emit_operand(rdi, dst, 4);
 1710   emit_int32(imm32);
 1711 }
 1712 
 1713 void Assembler::cmpl(Register dst, int32_t imm32) {
 1714   prefix(dst);
 1715   emit_arith(0x81, 0xF8, dst, imm32);
 1716 }
 1717 
 1718 void Assembler::cmpl(Register dst, Register src) {
 1719   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1720   emit_arith(0x3B, 0xC0, dst, src);
 1721 }
 1722 
 1723 void Assembler::cmpl(Register dst, Address  src) {
 1724   InstructionMark im(this);
 1725   prefix(src, dst);
 1726   emit_int8(0x3B);
 1727   emit_operand(dst, src);
 1728 }
 1729 
 1730 void Assembler::cmpw(Address dst, int imm16) {
 1731   InstructionMark im(this);
 1732   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1733   emit_int16(0x66, (unsigned char)0x81);
 1734   emit_operand(rdi, dst, 2);
 1735   emit_int16(imm16);
 1736 }
 1737 
 1738 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1739 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1740 // The ZF is set if the compared values were equal, and cleared otherwise.
 1741 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1742   InstructionMark im(this);
 1743   prefix(adr, reg);
 1744   emit_int16(0x0F, (unsigned char)0xB1);
 1745   emit_operand(reg, adr);
 1746 }
 1747 
 1748 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1749   InstructionMark im(this);
 1750   size_prefix();
 1751   prefix(adr, reg);
 1752   emit_int16(0x0F, (unsigned char)0xB1);
 1753   emit_operand(reg, adr);
 1754 }
 1755 
 1756 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1757 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1758 // The ZF is set if the compared values were equal, and cleared otherwise.
 1759 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1760   InstructionMark im(this);
 1761   prefix(adr, reg, true);
 1762   emit_int16(0x0F, (unsigned char)0xB0);
 1763   emit_operand(reg, adr);
 1764 }
 1765 
 1766 void Assembler::comisd(XMMRegister dst, Address src) {
 1767   // NOTE: dbx seems to decode this as comiss even though the
 1768   // 0x66 is there. Strangely ucomisd comes out correct
 1769   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1770   InstructionMark im(this);
 1771   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1772   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1773   attributes.set_rex_vex_w_reverted();
 1774   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1775   emit_int8(0x2F);
 1776   emit_operand(dst, src);
 1777 }
 1778 
 1779 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1780   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1781   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1782   attributes.set_rex_vex_w_reverted();
 1783   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1784   emit_int16(0x2F, (0xC0 | encode));
 1785 }
 1786 
 1787 void Assembler::comiss(XMMRegister dst, Address src) {
 1788   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1789   InstructionMark im(this);
 1790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1791   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1792   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1793   emit_int8(0x2F);
 1794   emit_operand(dst, src);
 1795 }
 1796 
 1797 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1798   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1800   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1801   emit_int16(0x2F, (0xC0 | encode));
 1802 }
 1803 
 1804 void Assembler::cpuid() {
 1805   emit_int16(0x0F, (unsigned char)0xA2);
 1806 }
 1807 
 1808 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1809 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1810 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1811 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1812 //
 1813 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1814 //
 1815 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1816 //
 1817 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1818 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1819   assert(VM_Version::supports_sse4_2(), "");
 1820   int8_t w = 0x01;
 1821   Prefix p = Prefix_EMPTY;
 1822 
 1823   emit_int8((unsigned char)0xF2);
 1824   switch (sizeInBytes) {
 1825   case 1:
 1826     w = 0;
 1827     break;
 1828   case 2:
 1829   case 4:
 1830     break;
 1831   LP64_ONLY(case 8:)
 1832     // This instruction is not valid in 32 bits
 1833     // Note:
 1834     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1835     //
 1836     // Page B - 72   Vol. 2C says
 1837     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1838     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1839     //                                                                            F0!!!
 1840     // while 3 - 208 Vol. 2A
 1841     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1842     //
 1843     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1844     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1845     p = REX_W;
 1846     break;
 1847   default:
 1848     assert(0, "Unsupported value for a sizeInBytes argument");
 1849     break;
 1850   }
 1851   LP64_ONLY(prefix(crc, v, p);)
 1852   emit_int32(0x0F,
 1853              0x38,
 1854              0xF0 | w,
 1855              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1856 }
 1857 
 1858 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1859   assert(VM_Version::supports_sse4_2(), "");
 1860   InstructionMark im(this);
 1861   int8_t w = 0x01;
 1862   Prefix p = Prefix_EMPTY;
 1863 
 1864   emit_int8((int8_t)0xF2);
 1865   switch (sizeInBytes) {
 1866   case 1:
 1867     w = 0;
 1868     break;
 1869   case 2:
 1870   case 4:
 1871     break;
 1872   LP64_ONLY(case 8:)
 1873     // This instruction is not valid in 32 bits
 1874     p = REX_W;
 1875     break;
 1876   default:
 1877     assert(0, "Unsupported value for a sizeInBytes argument");
 1878     break;
 1879   }
 1880   LP64_ONLY(prefix(crc, adr, p);)
 1881   emit_int24(0x0F, 0x38, (0xF0 | w));
 1882   emit_operand(crc, adr);
 1883 }
 1884 
 1885 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1886   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1889   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1893   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1894   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1895   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1896   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1897 }
 1898 
 1899 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1900   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1901   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1902   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1903   emit_int16(0x5B, (0xC0 | encode));
 1904 }
 1905 
 1906 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1907   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1908   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1909   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1910   emit_int16(0x5B, (0xC0 | encode));
 1911 }
 1912 
 1913 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1914   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1915   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1916   attributes.set_rex_vex_w_reverted();
 1917   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1918   emit_int16(0x5A, (0xC0 | encode));
 1919 }
 1920 
 1921 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1922   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1923   InstructionMark im(this);
 1924   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1925   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1926   attributes.set_rex_vex_w_reverted();
 1927   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1928   emit_int8(0x5A);
 1929   emit_operand(dst, src);
 1930 }
 1931 
 1932 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1933   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1934   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1935   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1936   emit_int16(0x2A, (0xC0 | encode));
 1937 }
 1938 
 1939 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1940   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1941   InstructionMark im(this);
 1942   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1943   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1944   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1945   emit_int8(0x2A);
 1946   emit_operand(dst, src);
 1947 }
 1948 
 1949 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1950   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1951   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1952   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1953   emit_int16(0x2A, (0xC0 | encode));
 1954 }
 1955 
 1956 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1957   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1958   InstructionMark im(this);
 1959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1960   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1961   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1962   emit_int8(0x2A);
 1963   emit_operand(dst, src);
 1964 }
 1965 
 1966 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1967   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1968   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1969   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1970   emit_int16(0x2A, (0xC0 | encode));
 1971 }
 1972 
 1973 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1974   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1975   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1976   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1977   emit_int16(0x5A, (0xC0 | encode));
 1978 }
 1979 
 1980 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1981   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1982   InstructionMark im(this);
 1983   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1984   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1985   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1986   emit_int8(0x5A);
 1987   emit_operand(dst, src);
 1988 }
 1989 
 1990 
 1991 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1993   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1994   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1995   emit_int16(0x2C, (0xC0 | encode));
 1996 }
 1997 
 1998 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 1999   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2000   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2001   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2002   emit_int16(0x2D, (0xC0 | encode));
 2003 }
 2004 
 2005 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2006   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2007   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2008   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2009   emit_int16(0x2C, (0xC0 | encode));
 2010 }
 2011 
 2012 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2013   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2014   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2015   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2016   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2017   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2018 }
 2019 
 2020 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2021   assert(VM_Version::supports_ssse3(), "");
 2022   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2023   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2024   emit_int16(0x1C, (0xC0 | encode));
 2025 }
 2026 
 2027 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2028   assert(VM_Version::supports_ssse3(), "");
 2029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2031   emit_int16(0x1D, (0xC0 | encode));
 2032 }
 2033 
 2034 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2035   assert(VM_Version::supports_ssse3(), "");
 2036   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2037   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2038   emit_int16(0x1E, (0xC0 | encode));
 2039 }
 2040 
 2041 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2042   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2043          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2044          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2045   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2046   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2047   emit_int16(0x1C, (0xC0 | encode));
 2048 }
 2049 
 2050 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2051   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2052          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2053          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2054   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2055   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2056   emit_int16(0x1D, (0xC0 | encode));
 2057 }
 2058 
 2059 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2060   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2061   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2062   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2063   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2065   emit_int16(0x1E, (0xC0 | encode));
 2066 }
 2067 
 2068 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2069   assert(UseAVX > 2, "");
 2070   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2071   attributes.set_is_evex_instruction();
 2072   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2073   emit_int16(0x1F, (0xC0 | encode));
 2074 }
 2075 
 2076 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2077   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2078   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2079   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2080   emit_int16(0x5A, (0xC0 | encode));
 2081 }
 2082 
 2083 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2084   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2085   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2086   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2087   attributes.set_rex_vex_w_reverted();
 2088   emit_int16(0x5A, (0xC0 | encode));
 2089 }
 2090 
 2091 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2092   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2093   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2094   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2095   emit_int16(0x5B, (0xC0 | encode));
 2096 }
 2097 
 2098 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2099   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2100   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2101   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2102   emit_int16(0x5B, (0xC0 | encode));
 2103 }
 2104 
 2105 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2106   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2107   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2108   attributes.set_is_evex_instruction();
 2109   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2110   emit_int16(0x7B, (0xC0 | encode));
 2111 }
 2112 
 2113 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2114   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2115   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2116   attributes.set_is_evex_instruction();
 2117   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2118   emit_int16(0x5B, (0xC0 | encode));
 2119 }
 2120 
 2121 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2122   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2123   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2124   attributes.set_is_evex_instruction();
 2125   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2126   emit_int16(0x7A, (0xC0 | encode));
 2127 }
 2128 
 2129 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2130   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2131   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2132   attributes.set_is_evex_instruction();
 2133   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2134   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2135 }
 2136 
 2137 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2138   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2139   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2140   attributes.set_is_evex_instruction();
 2141   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2142   emit_int16(0x30, (0xC0 | encode));
 2143 }
 2144 
 2145 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2146   assert(UseAVX > 2, "");
 2147   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   attributes.set_is_evex_instruction();
 2149   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2150   emit_int16(0x33, (0xC0 | encode));
 2151 }
 2152 
 2153 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2154   assert(UseAVX > 2, "");
 2155   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2156   attributes.set_is_evex_instruction();
 2157   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2158   emit_int16(0x31, (0xC0 | encode));
 2159 }
 2160 
 2161 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2162   assert(UseAVX > 2, "");
 2163   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2164   attributes.set_is_evex_instruction();
 2165   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2166   emit_int16(0x35, (0xC0 | encode));
 2167 }
 2168 
 2169 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2170   assert(UseAVX > 2, "");
 2171   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2172   attributes.set_is_evex_instruction();
 2173   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2174   emit_int16(0x32, (0xC0 | encode));
 2175 }
 2176 
 2177 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2178   assert(UseAVX > 2, "");
 2179   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2180   attributes.set_is_evex_instruction();
 2181   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2182   emit_int16(0x34, (0xC0 | encode));
 2183 }
 2184 
 2185 void Assembler::decl(Address dst) {
 2186   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2187   InstructionMark im(this);
 2188   prefix(dst);
 2189   emit_int8((unsigned char)0xFF);
 2190   emit_operand(rcx, dst);
 2191 }
 2192 
 2193 void Assembler::divsd(XMMRegister dst, Address src) {
 2194   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2195   InstructionMark im(this);
 2196   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2197   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2198   attributes.set_rex_vex_w_reverted();
 2199   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2200   emit_int8(0x5E);
 2201   emit_operand(dst, src);
 2202 }
 2203 
 2204 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2205   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2206   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2207   attributes.set_rex_vex_w_reverted();
 2208   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2209   emit_int16(0x5E, (0xC0 | encode));
 2210 }
 2211 
 2212 void Assembler::divss(XMMRegister dst, Address src) {
 2213   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2214   InstructionMark im(this);
 2215   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2216   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2217   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2218   emit_int8(0x5E);
 2219   emit_operand(dst, src);
 2220 }
 2221 
 2222 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2223   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2225   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2226   emit_int16(0x5E, (0xC0 | encode));
 2227 }
 2228 
 2229 void Assembler::hlt() {
 2230   emit_int8((unsigned char)0xF4);
 2231 }
 2232 
 2233 void Assembler::idivl(Register src) {
 2234   int encode = prefix_and_encode(src->encoding());
 2235   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2236 }
 2237 
 2238 void Assembler::idivl(Address src) {
 2239   InstructionMark im(this);
 2240   prefix(src);
 2241   emit_int8((unsigned char)0xF7);
 2242   emit_operand(as_Register(7), src);
 2243 }
 2244 
 2245 void Assembler::divl(Register src) { // Unsigned
 2246   int encode = prefix_and_encode(src->encoding());
 2247   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2248 }
 2249 
 2250 void Assembler::imull(Register src) {
 2251   int encode = prefix_and_encode(src->encoding());
 2252   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2253 }
 2254 
 2255 void Assembler::imull(Register dst, Register src) {
 2256   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2257   emit_int24(0x0F,
 2258              (unsigned char)0xAF,
 2259              (0xC0 | encode));
 2260 }
 2261 
 2262 void Assembler::imull(Register dst, Address src, int32_t value) {
 2263   InstructionMark im(this);
 2264   prefix(src, dst);
 2265   if (is8bit(value)) {
 2266     emit_int8((unsigned char)0x6B);
 2267     emit_operand(dst, src);
 2268     emit_int8(value);
 2269   } else {
 2270     emit_int8((unsigned char)0x69);
 2271     emit_operand(dst, src);
 2272     emit_int32(value);
 2273   }
 2274 }
 2275 
 2276 void Assembler::imull(Register dst, Register src, int value) {
 2277   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2278   if (is8bit(value)) {
 2279     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2280   } else {
 2281     emit_int16(0x69, (0xC0 | encode));
 2282     emit_int32(value);
 2283   }
 2284 }
 2285 
 2286 void Assembler::imull(Register dst, Address src) {
 2287   InstructionMark im(this);
 2288   prefix(src, dst);
 2289   emit_int16(0x0F, (unsigned char)0xAF);
 2290   emit_operand(dst, src);
 2291 }
 2292 
 2293 
 2294 void Assembler::incl(Address dst) {
 2295   // Don't use it directly. Use MacroAssembler::increment() instead.
 2296   InstructionMark im(this);
 2297   prefix(dst);
 2298   emit_int8((unsigned char)0xFF);
 2299   emit_operand(rax, dst);
 2300 }
 2301 
 2302 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2303   InstructionMark im(this);
 2304   assert((0 <= cc) && (cc < 16), "illegal cc");
 2305   if (L.is_bound()) {
 2306     address dst = target(L);
 2307     assert(dst != NULL, "jcc most probably wrong");
 2308 
 2309     const int short_size = 2;
 2310     const int long_size = 6;
 2311     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2312     if (maybe_short && is8bit(offs - short_size)) {
 2313       // 0111 tttn #8-bit disp
 2314       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2315     } else {
 2316       // 0000 1111 1000 tttn #32-bit disp
 2317       assert(is_simm32(offs - long_size),
 2318              "must be 32bit offset (call4)");
 2319       emit_int16(0x0F, (0x80 | cc));
 2320       emit_int32(offs - long_size);
 2321     }
 2322   } else {
 2323     // Note: could eliminate cond. jumps to this jump if condition
 2324     //       is the same however, seems to be rather unlikely case.
 2325     // Note: use jccb() if label to be bound is very close to get
 2326     //       an 8-bit displacement
 2327     L.add_patch_at(code(), locator());
 2328     emit_int16(0x0F, (0x80 | cc));
 2329     emit_int32(0);
 2330   }
 2331 }
 2332 
 2333 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2334   if (L.is_bound()) {
 2335     const int short_size = 2;
 2336     address entry = target(L);
 2337 #ifdef ASSERT
 2338     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2339     intptr_t delta = short_branch_delta();
 2340     if (delta != 0) {
 2341       dist += (dist < 0 ? (-delta) :delta);
 2342     }
 2343     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2344 #endif
 2345     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2346     // 0111 tttn #8-bit disp
 2347     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2348   } else {
 2349     InstructionMark im(this);
 2350     L.add_patch_at(code(), locator(), file, line);
 2351     emit_int16(0x70 | cc, 0);
 2352   }
 2353 }
 2354 
 2355 void Assembler::jmp(Address adr) {
 2356   InstructionMark im(this);
 2357   prefix(adr);
 2358   emit_int8((unsigned char)0xFF);
 2359   emit_operand(rsp, adr);
 2360 }
 2361 
 2362 void Assembler::jmp(Label& L, bool maybe_short) {
 2363   if (L.is_bound()) {
 2364     address entry = target(L);
 2365     assert(entry != NULL, "jmp most probably wrong");
 2366     InstructionMark im(this);
 2367     const int short_size = 2;
 2368     const int long_size = 5;
 2369     intptr_t offs = entry - pc();
 2370     if (maybe_short && is8bit(offs - short_size)) {
 2371       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2372     } else {
 2373       emit_int8((unsigned char)0xE9);
 2374       emit_int32(offs - long_size);
 2375     }
 2376   } else {
 2377     // By default, forward jumps are always 32-bit displacements, since
 2378     // we can't yet know where the label will be bound.  If you're sure that
 2379     // the forward jump will not run beyond 256 bytes, use jmpb to
 2380     // force an 8-bit displacement.
 2381     InstructionMark im(this);
 2382     L.add_patch_at(code(), locator());
 2383     emit_int8((unsigned char)0xE9);
 2384     emit_int32(0);
 2385   }
 2386 }
 2387 
 2388 void Assembler::jmp(Register entry) {
 2389   int encode = prefix_and_encode(entry->encoding());
 2390   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2391 }
 2392 
 2393 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2394   InstructionMark im(this);
 2395   emit_int8((unsigned char)0xE9);
 2396   assert(dest != NULL, "must have a target");
 2397   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2398   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2399   emit_data(disp, rspec.reloc(), call32_operand);
 2400 }
 2401 
 2402 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2403   if (L.is_bound()) {
 2404     const int short_size = 2;
 2405     address entry = target(L);
 2406     assert(entry != NULL, "jmp most probably wrong");
 2407 #ifdef ASSERT
 2408     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2409     intptr_t delta = short_branch_delta();
 2410     if (delta != 0) {
 2411       dist += (dist < 0 ? (-delta) :delta);
 2412     }
 2413     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2414 #endif
 2415     intptr_t offs = entry - pc();
 2416     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2417   } else {
 2418     InstructionMark im(this);
 2419     L.add_patch_at(code(), locator(), file, line);
 2420     emit_int16((unsigned char)0xEB, 0);
 2421   }
 2422 }
 2423 
 2424 void Assembler::ldmxcsr( Address src) {
 2425   if (UseAVX > 0 ) {
 2426     InstructionMark im(this);
 2427     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2428     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2429     emit_int8((unsigned char)0xAE);
 2430     emit_operand(as_Register(2), src);
 2431   } else {
 2432     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2433     InstructionMark im(this);
 2434     prefix(src);
 2435     emit_int16(0x0F, (unsigned char)0xAE);
 2436     emit_operand(as_Register(2), src);
 2437   }
 2438 }
 2439 
 2440 void Assembler::leal(Register dst, Address src) {
 2441   InstructionMark im(this);
 2442   prefix(src, dst);
 2443   emit_int8((unsigned char)0x8D);
 2444   emit_operand(dst, src);
 2445 }
 2446 
 2447 void Assembler::lfence() {
 2448   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2449 }
 2450 
 2451 void Assembler::lock() {
 2452   emit_int8((unsigned char)0xF0);
 2453 }
 2454 
 2455 void Assembler::size_prefix() {
 2456   emit_int8(0x66);
 2457 }
 2458 
 2459 void Assembler::lzcntl(Register dst, Register src) {
 2460   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2461   emit_int8((unsigned char)0xF3);
 2462   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2463   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2464 }
 2465 
 2466 // Emit mfence instruction
 2467 void Assembler::mfence() {
 2468   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2469   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2470 }
 2471 
 2472 // Emit sfence instruction
 2473 void Assembler::sfence() {
 2474   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2475   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2476 }
 2477 
 2478 void Assembler::mov(Register dst, Register src) {
 2479   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2480 }
 2481 
 2482 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2483   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2484   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2485   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2486   attributes.set_rex_vex_w_reverted();
 2487   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2488   emit_int16(0x28, (0xC0 | encode));
 2489 }
 2490 
 2491 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2492   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2493   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2494   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2495   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2496   emit_int16(0x28, (0xC0 | encode));
 2497 }
 2498 
 2499 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2500   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2501   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2502   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2503   emit_int16(0x16, (0xC0 | encode));
 2504 }
 2505 
 2506 void Assembler::movb(Register dst, Address src) {
 2507   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2508   InstructionMark im(this);
 2509   prefix(src, dst, true);
 2510   emit_int8((unsigned char)0x8A);
 2511   emit_operand(dst, src);
 2512 }
 2513 
 2514 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2515   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2516   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2517   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2518   attributes.set_rex_vex_w_reverted();
 2519   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2520   emit_int16(0x12, 0xC0 | encode);
 2521 }
 2522 
 2523 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2524   assert(VM_Version::supports_avx(), "");
 2525   InstructionMark im(this);
 2526   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2527   attributes.set_rex_vex_w_reverted();
 2528   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2529   emit_int8(0x12);
 2530   emit_operand(dst, src);
 2531 }
 2532 
 2533 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2534   assert(VM_Version::supports_avx512dq(), "");
 2535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2536   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2537   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2538 }
 2539 
 2540 void Assembler::kmovbl(KRegister dst, Register src) {
 2541   assert(VM_Version::supports_avx512dq(), "");
 2542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2543   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2544   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2545 }
 2546 
 2547 void Assembler::kmovbl(Register dst, KRegister src) {
 2548   assert(VM_Version::supports_avx512dq(), "");
 2549   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2550   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2551   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2552 }
 2553 
 2554 void Assembler::kmovwl(KRegister dst, Register src) {
 2555   assert(VM_Version::supports_evex(), "");
 2556   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2557   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2558   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2559 }
 2560 
 2561 void Assembler::kmovwl(Register dst, KRegister src) {
 2562   assert(VM_Version::supports_evex(), "");
 2563   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2564   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2565   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2566 }
 2567 
 2568 void Assembler::kmovwl(KRegister dst, Address src) {
 2569   assert(VM_Version::supports_evex(), "");
 2570   InstructionMark im(this);
 2571   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2572   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2573   emit_int8((unsigned char)0x90);
 2574   emit_operand((Register)dst, src);
 2575 }
 2576 
 2577 void Assembler::kmovwl(Address dst, KRegister src) {
 2578   assert(VM_Version::supports_evex(), "");
 2579   InstructionMark im(this);
 2580   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2581   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2582   emit_int8((unsigned char)0x91);
 2583   emit_operand((Register)src, dst);
 2584 }
 2585 
 2586 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2587   assert(VM_Version::supports_evex(), "");
 2588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2589   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2590   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2591 }
 2592 
 2593 void Assembler::kmovdl(KRegister dst, Register src) {
 2594   assert(VM_Version::supports_avx512bw(), "");
 2595   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2596   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2597   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2598 }
 2599 
 2600 void Assembler::kmovdl(Register dst, KRegister src) {
 2601   assert(VM_Version::supports_avx512bw(), "");
 2602   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2603   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2604   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2605 }
 2606 
 2607 void Assembler::kmovql(KRegister dst, KRegister src) {
 2608   assert(VM_Version::supports_avx512bw(), "");
 2609   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2610   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2611   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2612 }
 2613 
 2614 void Assembler::kmovql(KRegister dst, Address src) {
 2615   assert(VM_Version::supports_avx512bw(), "");
 2616   InstructionMark im(this);
 2617   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2618   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2619   emit_int8((unsigned char)0x90);
 2620   emit_operand((Register)dst, src);
 2621 }
 2622 
 2623 void Assembler::kmovql(Address dst, KRegister src) {
 2624   assert(VM_Version::supports_avx512bw(), "");
 2625   InstructionMark im(this);
 2626   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2627   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2628   emit_int8((unsigned char)0x91);
 2629   emit_operand((Register)src, dst);
 2630 }
 2631 
 2632 void Assembler::kmovql(KRegister dst, Register src) {
 2633   assert(VM_Version::supports_avx512bw(), "");
 2634   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2635   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2636   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2637 }
 2638 
 2639 void Assembler::kmovql(Register dst, KRegister src) {
 2640   assert(VM_Version::supports_avx512bw(), "");
 2641   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2642   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2643   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2644 }
 2645 
 2646 void Assembler::knotwl(KRegister dst, KRegister src) {
 2647   assert(VM_Version::supports_evex(), "");
 2648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2649   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2650   emit_int16(0x44, (0xC0 | encode));
 2651 }
 2652 
 2653 void Assembler::knotbl(KRegister dst, KRegister src) {
 2654   assert(VM_Version::supports_evex(), "");
 2655   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2656   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2657   emit_int16(0x44, (0xC0 | encode));
 2658 }
 2659 
 2660 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2661   assert(VM_Version::supports_avx512dq(), "");
 2662   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2663   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2664   emit_int16(0x45, (0xC0 | encode));
 2665 }
 2666 
 2667 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2668   assert(VM_Version::supports_evex(), "");
 2669   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2670   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2671   emit_int16(0x45, (0xC0 | encode));
 2672 }
 2673 
 2674 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2675   assert(VM_Version::supports_avx512bw(), "");
 2676   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2677   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2678   emit_int16(0x45, (0xC0 | encode));
 2679 }
 2680 
 2681 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2682   assert(VM_Version::supports_avx512bw(), "");
 2683   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2684   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2685   emit_int16(0x45, (0xC0 | encode));
 2686 }
 2687 
 2688 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2689   assert(VM_Version::supports_avx512dq(), "");
 2690   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2691   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2692   emit_int16(0x47, (0xC0 | encode));
 2693 }
 2694 
 2695 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2696   assert(VM_Version::supports_evex(), "");
 2697   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2698   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2699   emit_int16(0x47, (0xC0 | encode));
 2700 }
 2701 
 2702 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2703   assert(VM_Version::supports_avx512bw(), "");
 2704   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2705   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2706   emit_int16(0x47, (0xC0 | encode));
 2707 }
 2708 
 2709 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2710   assert(VM_Version::supports_avx512bw(), "");
 2711   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2712   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2713   emit_int16(0x47, (0xC0 | encode));
 2714 }
 2715 
 2716 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2717   assert(VM_Version::supports_avx512dq(), "");
 2718   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2719   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2720   emit_int16(0x41, (0xC0 | encode));
 2721 }
 2722 
 2723 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2724   assert(VM_Version::supports_evex(), "");
 2725   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2726   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2727   emit_int16(0x41, (0xC0 | encode));
 2728 }
 2729 
 2730 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2731   assert(VM_Version::supports_avx512bw(), "");
 2732   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2733   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2734   emit_int16(0x41, (0xC0 | encode));
 2735 }
 2736 
 2737 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2738   assert(VM_Version::supports_avx512bw(), "");
 2739   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2740   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2741   emit_int16(0x41, (0xC0 | encode));
 2742 }
 2743 
 2744 void Assembler::knotdl(KRegister dst, KRegister src) {
 2745   assert(VM_Version::supports_avx512bw(), "");
 2746   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2747   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2748   emit_int16(0x44, (0xC0 | encode));
 2749 }
 2750 
 2751 void Assembler::knotql(KRegister dst, KRegister src) {
 2752   assert(VM_Version::supports_avx512bw(), "");
 2753   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2754   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2755   emit_int16(0x44, (0xC0 | encode));
 2756 }
 2757 
 2758 // This instruction produces ZF or CF flags
 2759 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2760   assert(VM_Version::supports_avx512dq(), "");
 2761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2762   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2763   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2764 }
 2765 
 2766 // This instruction produces ZF or CF flags
 2767 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2768   assert(VM_Version::supports_evex(), "");
 2769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2770   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2771   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2772 }
 2773 
 2774 // This instruction produces ZF or CF flags
 2775 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2776   assert(VM_Version::supports_avx512bw(), "");
 2777   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2778   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2779   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2780 }
 2781 
 2782 // This instruction produces ZF or CF flags
 2783 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2784   assert(VM_Version::supports_avx512bw(), "");
 2785   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2786   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2787   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2788 }
 2789 
 2790 // This instruction produces ZF or CF flags
 2791 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2792   assert(VM_Version::supports_avx512bw(), "");
 2793   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2794   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2795   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2796 }
 2797 
 2798 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2799   assert(VM_Version::supports_avx512bw(), "");
 2800   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2801   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2802   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2803 }
 2804 
 2805 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2806   assert(VM_Version::supports_avx512dq(), "");
 2807   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2808   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2809   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2810 }
 2811 
 2812 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2813   assert(VM_Version::supports_avx512dq(), "");
 2814   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2815   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2816   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2817 }
 2818 
 2819 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2820   assert(VM_Version::supports_avx512bw(), "");
 2821   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2822   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2823   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2824 }
 2825 
 2826 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2827   assert(VM_Version::supports_avx512bw(), "");
 2828   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2829   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2830   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2831 }
 2832 
 2833 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2834   assert(VM_Version::supports_avx512dq(), "");
 2835   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2836   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2837   emit_int16(0x46, (0xC0 | encode));
 2838 }
 2839 
 2840 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2841   assert(VM_Version::supports_avx512dq(), "");
 2842   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2843   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2844   emit_int16(0x32, (0xC0 | encode));
 2845   emit_int8(imm8);
 2846 }
 2847 
 2848 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2849   assert(VM_Version::supports_avx512bw(), "");
 2850   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2851   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2852   emit_int16(0x33, (0xC0 | encode));
 2853   emit_int8(imm8);
 2854 }
 2855 
 2856 
 2857 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2858   assert(VM_Version::supports_avx512dq(), "");
 2859   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2860   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2861   emit_int16(0x30, (0xC0 | encode));
 2862 }
 2863 
 2864 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2865   assert(VM_Version::supports_evex(), "");
 2866   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2867   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2868   emit_int16(0x30, (0xC0 | encode));
 2869   emit_int8(imm8);
 2870 }
 2871 
 2872 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2873   assert(VM_Version::supports_avx512bw(), "");
 2874   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2875   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2876   emit_int16(0x31, (0xC0 | encode));
 2877   emit_int8(imm8);
 2878 }
 2879 
 2880 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2881   assert(VM_Version::supports_avx512bw(), "");
 2882   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2883   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2884   emit_int16(0x31, (0xC0 | encode));
 2885   emit_int8(imm8);
 2886 }
 2887 
 2888 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 2889   assert(VM_Version::supports_avx512bw(), "");
 2890   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2891   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2892   emit_int16(0x4B, (0xC0 | encode));
 2893 }
 2894 
 2895 void Assembler::movb(Address dst, int imm8) {
 2896   InstructionMark im(this);
 2897    prefix(dst);
 2898   emit_int8((unsigned char)0xC6);
 2899   emit_operand(rax, dst, 1);
 2900   emit_int8(imm8);
 2901 }
 2902 
 2903 
 2904 void Assembler::movb(Address dst, Register src) {
 2905   assert(src->has_byte_register(), "must have byte register");
 2906   InstructionMark im(this);
 2907   prefix(dst, src, true);
 2908   emit_int8((unsigned char)0x88);
 2909   emit_operand(src, dst);
 2910 }
 2911 
 2912 void Assembler::movdl(XMMRegister dst, Register src) {
 2913   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2914   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2915   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2916   emit_int16(0x6E, (0xC0 | encode));
 2917 }
 2918 
 2919 void Assembler::movdl(Register dst, XMMRegister src) {
 2920   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2922   // swap src/dst to get correct prefix
 2923   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2924   emit_int16(0x7E, (0xC0 | encode));
 2925 }
 2926 
 2927 void Assembler::movdl(XMMRegister dst, Address src) {
 2928   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2929   InstructionMark im(this);
 2930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2931   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2932   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2933   emit_int8(0x6E);
 2934   emit_operand(dst, src);
 2935 }
 2936 
 2937 void Assembler::movdl(Address dst, XMMRegister src) {
 2938   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2939   InstructionMark im(this);
 2940   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2941   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2942   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2943   emit_int8(0x7E);
 2944   emit_operand(src, dst);
 2945 }
 2946 
 2947 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2949   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2950   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2951   emit_int16(0x6F, (0xC0 | encode));
 2952 }
 2953 
 2954 void Assembler::movdqa(XMMRegister dst, Address src) {
 2955   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2956   InstructionMark im(this);
 2957   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2958   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2959   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2960   emit_int8(0x6F);
 2961   emit_operand(dst, src);
 2962 }
 2963 
 2964 void Assembler::movdqu(XMMRegister dst, Address src) {
 2965   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2966   InstructionMark im(this);
 2967   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2968   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2969   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2970   emit_int8(0x6F);
 2971   emit_operand(dst, src);
 2972 }
 2973 
 2974 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2975   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2976   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2977   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2978   emit_int16(0x6F, (0xC0 | encode));
 2979 }
 2980 
 2981 void Assembler::movdqu(Address dst, XMMRegister src) {
 2982   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2983   InstructionMark im(this);
 2984   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2985   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2986   attributes.reset_is_clear_context();
 2987   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2988   emit_int8(0x7F);
 2989   emit_operand(src, dst);
 2990 }
 2991 
 2992 // Move Unaligned 256bit Vector
 2993 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2994   assert(UseAVX > 0, "");
 2995   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2996   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2997   emit_int16(0x6F, (0xC0 | encode));
 2998 }
 2999 
 3000 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3001   assert(UseAVX > 0, "");
 3002   InstructionMark im(this);
 3003   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3004   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3005   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3006   emit_int8(0x6F);
 3007   emit_operand(dst, src);
 3008 }
 3009 
 3010 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3011   assert(UseAVX > 0, "");
 3012   InstructionMark im(this);
 3013   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3014   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3015   attributes.reset_is_clear_context();
 3016   // swap src<->dst for encoding
 3017   assert(src != xnoreg, "sanity");
 3018   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3019   emit_int8(0x7F);
 3020   emit_operand(src, dst);
 3021 }
 3022 
 3023 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3024 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 3025   assert(VM_Version::supports_evex(), "");
 3026   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3027   attributes.set_is_evex_instruction();
 3028   if (merge) {
 3029     attributes.reset_is_clear_context();
 3030   }
 3031   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3032   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3033   emit_int16(0x6F, (0xC0 | encode));
 3034 }
 3035 
 3036 void Assembler::evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) {
 3037   assert(VM_Version::supports_evex(), "");
 3038   InstructionMark im(this);
 3039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3040   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3041   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3042   attributes.set_is_evex_instruction();
 3043   if (merge) {
 3044     attributes.reset_is_clear_context();
 3045   }
 3046   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3047   emit_int8(0x6F);
 3048   emit_operand(dst, src);
 3049 }
 3050 
 3051 void Assembler::evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) {
 3052   assert(VM_Version::supports_evex(), "");
 3053   assert(src != xnoreg, "sanity");
 3054   InstructionMark im(this);
 3055   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3056   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3057   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3058   attributes.set_is_evex_instruction();
 3059   if (merge) {
 3060     attributes.reset_is_clear_context();
 3061   }
 3062   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3063   emit_int8(0x7F);
 3064   emit_operand(src, dst);
 3065 }
 3066 
 3067 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3068   assert(VM_Version::supports_avx512vlbw(), "");
 3069   InstructionMark im(this);
 3070   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3071   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3072   attributes.set_embedded_opmask_register_specifier(mask);
 3073   attributes.set_is_evex_instruction();
 3074   if (merge) {
 3075     attributes.reset_is_clear_context();
 3076   }
 3077   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3078   emit_int8(0x6F);
 3079   emit_operand(dst, src);
 3080 }
 3081 
 3082 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3083   assert(VM_Version::supports_avx512vlbw(), "");
 3084   assert(src != xnoreg, "sanity");
 3085   InstructionMark im(this);
 3086   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3087   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3088   attributes.set_embedded_opmask_register_specifier(mask);
 3089   attributes.set_is_evex_instruction();
 3090   if (merge) {
 3091     attributes.reset_is_clear_context();
 3092   }
 3093   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3094   emit_int8(0x7F);
 3095   emit_operand(src, dst);
 3096 }
 3097 
 3098 void Assembler::evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) {
 3099   assert(VM_Version::supports_evex(), "");
 3100   InstructionMark im(this);
 3101   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3102   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3103   attributes.set_is_evex_instruction();
 3104   if (merge) {
 3105     attributes.reset_is_clear_context();
 3106   }
 3107   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3108   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3109   emit_int8(0x6F);
 3110   emit_operand(dst, src);
 3111 }
 3112 
 3113 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3114   assert(VM_Version::supports_avx512vlbw(), "");
 3115   InstructionMark im(this);
 3116   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3117   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3118   attributes.set_embedded_opmask_register_specifier(mask);
 3119   attributes.set_is_evex_instruction();
 3120   if (merge) {
 3121     attributes.reset_is_clear_context();
 3122   }
 3123   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3124   emit_int8(0x6F);
 3125   emit_operand(dst, src);
 3126 }
 3127 
 3128 void Assembler::evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) {
 3129   assert(VM_Version::supports_evex(), "");
 3130   assert(src != xnoreg, "sanity");
 3131   InstructionMark im(this);
 3132   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3133   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3134   attributes.set_is_evex_instruction();
 3135   if (merge) {
 3136     attributes.reset_is_clear_context();
 3137   }
 3138   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3139   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3140   emit_int8(0x7F);
 3141   emit_operand(src, dst);
 3142 }
 3143 
 3144 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3145   assert(VM_Version::supports_avx512vlbw(), "");
 3146   assert(src != xnoreg, "sanity");
 3147   InstructionMark im(this);
 3148   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3149   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3150   attributes.set_embedded_opmask_register_specifier(mask);
 3151   attributes.set_is_evex_instruction();
 3152   if (merge) {
 3153     attributes.reset_is_clear_context();
 3154   }
 3155   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3156   emit_int8(0x7F);
 3157   emit_operand(src, dst);
 3158 }
 3159 
 3160 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3161   // Unmasked instruction
 3162   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3163 }
 3164 
 3165 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3166   assert(VM_Version::supports_evex(), "");
 3167   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3168   attributes.set_embedded_opmask_register_specifier(mask);
 3169   attributes.set_is_evex_instruction();
 3170   if (merge) {
 3171     attributes.reset_is_clear_context();
 3172   }
 3173   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3174   emit_int16(0x6F, (0xC0 | encode));
 3175 }
 3176 
 3177 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3178   // Unmasked instruction
 3179   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3180 }
 3181 
 3182 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3183   assert(VM_Version::supports_evex(), "");
 3184   InstructionMark im(this);
 3185   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3186   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3187   attributes.set_embedded_opmask_register_specifier(mask);
 3188   attributes.set_is_evex_instruction();
 3189   if (merge) {
 3190     attributes.reset_is_clear_context();
 3191   }
 3192   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3193   emit_int8(0x6F);
 3194   emit_operand(dst, src);
 3195 }
 3196 
 3197 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3198   // Unmasked isntruction
 3199   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3200 }
 3201 
 3202 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3203   assert(VM_Version::supports_evex(), "");
 3204   assert(src != xnoreg, "sanity");
 3205   InstructionMark im(this);
 3206   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3207   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3208   attributes.set_embedded_opmask_register_specifier(mask);
 3209   attributes.set_is_evex_instruction();
 3210   if (merge) {
 3211     attributes.reset_is_clear_context();
 3212   }
 3213   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3214   emit_int8(0x7F);
 3215   emit_operand(src, dst);
 3216 }
 3217 
 3218 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3219   // Unmasked instruction
 3220   if (dst->encoding() == src->encoding()) return;
 3221   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3222 }
 3223 
 3224 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3225   assert(VM_Version::supports_evex(), "");
 3226   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3227   attributes.set_embedded_opmask_register_specifier(mask);
 3228   attributes.set_is_evex_instruction();
 3229   if (merge) {
 3230     attributes.reset_is_clear_context();
 3231   }
 3232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3233   emit_int16(0x6F, (0xC0 | encode));
 3234 }
 3235 
 3236 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3237   // Unmasked instruction
 3238   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3239 }
 3240 
 3241 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3242   assert(VM_Version::supports_evex(), "");
 3243   InstructionMark im(this);
 3244   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3245   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3246   attributes.set_embedded_opmask_register_specifier(mask);
 3247   attributes.set_is_evex_instruction();
 3248   if (merge) {
 3249     attributes.reset_is_clear_context();
 3250   }
 3251   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3252   emit_int8(0x6F);
 3253   emit_operand(dst, src);
 3254 }
 3255 
 3256 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3257   // Unmasked instruction
 3258   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3259 }
 3260 
 3261 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3262   assert(VM_Version::supports_evex(), "");
 3263   assert(src != xnoreg, "sanity");
 3264   InstructionMark im(this);
 3265   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3266   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3267   attributes.set_embedded_opmask_register_specifier(mask);
 3268   if (merge) {
 3269     attributes.reset_is_clear_context();
 3270   }
 3271   attributes.set_is_evex_instruction();
 3272   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3273   emit_int8(0x7F);
 3274   emit_operand(src, dst);
 3275 }
 3276 
 3277 // Uses zero extension on 64bit
 3278 
 3279 void Assembler::movl(Register dst, int32_t imm32) {
 3280   int encode = prefix_and_encode(dst->encoding());
 3281   emit_int8(0xB8 | encode);
 3282   emit_int32(imm32);
 3283 }
 3284 
 3285 void Assembler::movl(Register dst, Register src) {
 3286   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3287   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3288 }
 3289 
 3290 void Assembler::movl(Register dst, Address src) {
 3291   InstructionMark im(this);
 3292   prefix(src, dst);
 3293   emit_int8((unsigned char)0x8B);
 3294   emit_operand(dst, src);
 3295 }
 3296 
 3297 void Assembler::movl(Address dst, int32_t imm32) {
 3298   InstructionMark im(this);
 3299   prefix(dst);
 3300   emit_int8((unsigned char)0xC7);
 3301   emit_operand(rax, dst, 4);
 3302   emit_int32(imm32);
 3303 }
 3304 
 3305 void Assembler::movl(Address dst, Register src) {
 3306   InstructionMark im(this);
 3307   prefix(dst, src);
 3308   emit_int8((unsigned char)0x89);
 3309   emit_operand(src, dst);
 3310 }
 3311 
 3312 // New cpus require to use movsd and movss to avoid partial register stall
 3313 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3314 // The selection is done in MacroAssembler::movdbl() and movflt().
 3315 void Assembler::movlpd(XMMRegister dst, Address src) {
 3316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3317   InstructionMark im(this);
 3318   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3319   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3320   attributes.set_rex_vex_w_reverted();
 3321   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3322   emit_int8(0x12);
 3323   emit_operand(dst, src);
 3324 }
 3325 
 3326 void Assembler::movq(XMMRegister dst, Address src) {
 3327   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3328   InstructionMark im(this);
 3329   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3330   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3331   attributes.set_rex_vex_w_reverted();
 3332   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3333   emit_int8(0x7E);
 3334   emit_operand(dst, src);
 3335 }
 3336 
 3337 void Assembler::movq(Address dst, XMMRegister src) {
 3338   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3339   InstructionMark im(this);
 3340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3341   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3342   attributes.set_rex_vex_w_reverted();
 3343   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3344   emit_int8((unsigned char)0xD6);
 3345   emit_operand(src, dst);
 3346 }
 3347 
 3348 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3349   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3350   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3351   attributes.set_rex_vex_w_reverted();
 3352   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3353   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3354 }
 3355 
 3356 void Assembler::movq(Register dst, XMMRegister src) {
 3357   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3358   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3359   // swap src/dst to get correct prefix
 3360   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3361   emit_int16(0x7E, (0xC0 | encode));
 3362 }
 3363 
 3364 void Assembler::movq(XMMRegister dst, Register src) {
 3365   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3366   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3367   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3368   emit_int16(0x6E, (0xC0 | encode));
 3369 }
 3370 
 3371 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3372   InstructionMark im(this);
 3373   prefix(src, dst);
 3374   emit_int16(0x0F, (unsigned char)0xBE);
 3375   emit_operand(dst, src);
 3376 }
 3377 
 3378 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3379   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3380   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3381   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3382 }
 3383 
 3384 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3386   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3387   attributes.set_rex_vex_w_reverted();
 3388   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3389   emit_int16(0x10, (0xC0 | encode));
 3390 }
 3391 
 3392 void Assembler::movsd(XMMRegister dst, Address src) {
 3393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3394   InstructionMark im(this);
 3395   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3396   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3397   attributes.set_rex_vex_w_reverted();
 3398   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3399   emit_int8(0x10);
 3400   emit_operand(dst, src);
 3401 }
 3402 
 3403 void Assembler::movsd(Address dst, XMMRegister src) {
 3404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3405   InstructionMark im(this);
 3406   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3407   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3408   attributes.reset_is_clear_context();
 3409   attributes.set_rex_vex_w_reverted();
 3410   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3411   emit_int8(0x11);
 3412   emit_operand(src, dst);
 3413 }
 3414 
 3415 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3416   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3417   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3418   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3419   emit_int16(0x10, (0xC0 | encode));
 3420 }
 3421 
 3422 void Assembler::movss(XMMRegister dst, Address src) {
 3423   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3424   InstructionMark im(this);
 3425   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3426   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3427   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3428   emit_int8(0x10);
 3429   emit_operand(dst, src);
 3430 }
 3431 
 3432 void Assembler::movss(Address dst, XMMRegister src) {
 3433   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3434   InstructionMark im(this);
 3435   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3436   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3437   attributes.reset_is_clear_context();
 3438   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3439   emit_int8(0x11);
 3440   emit_operand(src, dst);
 3441 }
 3442 
 3443 void Assembler::movswl(Register dst, Address src) { // movsxw
 3444   InstructionMark im(this);
 3445   prefix(src, dst);
 3446   emit_int16(0x0F, (unsigned char)0xBF);
 3447   emit_operand(dst, src);
 3448 }
 3449 
 3450 void Assembler::movswl(Register dst, Register src) { // movsxw
 3451   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3452   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3453 }
 3454 
 3455 void Assembler::movw(Address dst, int imm16) {
 3456   InstructionMark im(this);
 3457 
 3458   emit_int8(0x66); // switch to 16-bit mode
 3459   prefix(dst);
 3460   emit_int8((unsigned char)0xC7);
 3461   emit_operand(rax, dst, 2);
 3462   emit_int16(imm16);
 3463 }
 3464 
 3465 void Assembler::movw(Register dst, Address src) {
 3466   InstructionMark im(this);
 3467   emit_int8(0x66);
 3468   prefix(src, dst);
 3469   emit_int8((unsigned char)0x8B);
 3470   emit_operand(dst, src);
 3471 }
 3472 
 3473 void Assembler::movw(Address dst, Register src) {
 3474   InstructionMark im(this);
 3475   emit_int8(0x66);
 3476   prefix(dst, src);
 3477   emit_int8((unsigned char)0x89);
 3478   emit_operand(src, dst);
 3479 }
 3480 
 3481 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3482   InstructionMark im(this);
 3483   prefix(src, dst);
 3484   emit_int16(0x0F, (unsigned char)0xB6);
 3485   emit_operand(dst, src);
 3486 }
 3487 
 3488 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3489   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3490   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3491   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3492 }
 3493 
 3494 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3495   InstructionMark im(this);
 3496   prefix(src, dst);
 3497   emit_int16(0x0F, (unsigned char)0xB7);
 3498   emit_operand(dst, src);
 3499 }
 3500 
 3501 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3502   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3503   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3504 }
 3505 
 3506 void Assembler::mull(Address src) {
 3507   InstructionMark im(this);
 3508   prefix(src);
 3509   emit_int8((unsigned char)0xF7);
 3510   emit_operand(rsp, src);
 3511 }
 3512 
 3513 void Assembler::mull(Register src) {
 3514   int encode = prefix_and_encode(src->encoding());
 3515   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3516 }
 3517 
 3518 void Assembler::mulsd(XMMRegister dst, Address src) {
 3519   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3520   InstructionMark im(this);
 3521   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3522   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3523   attributes.set_rex_vex_w_reverted();
 3524   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3525   emit_int8(0x59);
 3526   emit_operand(dst, src);
 3527 }
 3528 
 3529 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3530   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3531   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3532   attributes.set_rex_vex_w_reverted();
 3533   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3534   emit_int16(0x59, (0xC0 | encode));
 3535 }
 3536 
 3537 void Assembler::mulss(XMMRegister dst, Address src) {
 3538   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3539   InstructionMark im(this);
 3540   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3541   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3542   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3543   emit_int8(0x59);
 3544   emit_operand(dst, src);
 3545 }
 3546 
 3547 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3548   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3549   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3550   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3551   emit_int16(0x59, (0xC0 | encode));
 3552 }
 3553 
 3554 void Assembler::negl(Register dst) {
 3555   int encode = prefix_and_encode(dst->encoding());
 3556   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3557 }
 3558 
 3559 void Assembler::negl(Address dst) {
 3560   InstructionMark im(this);
 3561   prefix(dst);
 3562   emit_int8((unsigned char)0xF7);
 3563   emit_operand(as_Register(3), dst);
 3564 }
 3565 
 3566 void Assembler::nop(int i) {
 3567 #ifdef ASSERT
 3568   assert(i > 0, " ");
 3569   // The fancy nops aren't currently recognized by debuggers making it a
 3570   // pain to disassemble code while debugging. If asserts are on clearly
 3571   // speed is not an issue so simply use the single byte traditional nop
 3572   // to do alignment.
 3573 
 3574   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3575   return;
 3576 
 3577 #endif // ASSERT
 3578 
 3579   if (UseAddressNop && VM_Version::is_intel()) {
 3580     //
 3581     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3582     //  1: 0x90
 3583     //  2: 0x66 0x90
 3584     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3585     //  4: 0x0F 0x1F 0x40 0x00
 3586     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3587     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3588     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3589     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3590     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3591     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3592     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3593 
 3594     // The rest coding is Intel specific - don't use consecutive address nops
 3595 
 3596     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3597     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3598     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3599     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3600 
 3601     while(i >= 15) {
 3602       // For Intel don't generate consecutive address nops (mix with regular nops)
 3603       i -= 15;
 3604       emit_int24(0x66, 0x66, 0x66);
 3605       addr_nop_8();
 3606       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3607     }
 3608     switch (i) {
 3609       case 14:
 3610         emit_int8(0x66); // size prefix
 3611       case 13:
 3612         emit_int8(0x66); // size prefix
 3613       case 12:
 3614         addr_nop_8();
 3615         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3616         break;
 3617       case 11:
 3618         emit_int8(0x66); // size prefix
 3619       case 10:
 3620         emit_int8(0x66); // size prefix
 3621       case 9:
 3622         emit_int8(0x66); // size prefix
 3623       case 8:
 3624         addr_nop_8();
 3625         break;
 3626       case 7:
 3627         addr_nop_7();
 3628         break;
 3629       case 6:
 3630         emit_int8(0x66); // size prefix
 3631       case 5:
 3632         addr_nop_5();
 3633         break;
 3634       case 4:
 3635         addr_nop_4();
 3636         break;
 3637       case 3:
 3638         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3639         emit_int8(0x66); // size prefix
 3640       case 2:
 3641         emit_int8(0x66); // size prefix
 3642       case 1:
 3643         emit_int8((unsigned char)0x90);
 3644                          // nop
 3645         break;
 3646       default:
 3647         assert(i == 0, " ");
 3648     }
 3649     return;
 3650   }
 3651   if (UseAddressNop && VM_Version::is_amd_family()) {
 3652     //
 3653     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3654     //  1: 0x90
 3655     //  2: 0x66 0x90
 3656     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3657     //  4: 0x0F 0x1F 0x40 0x00
 3658     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3659     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3660     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3661     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3662     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3663     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3664     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3665 
 3666     // The rest coding is AMD specific - use consecutive address nops
 3667 
 3668     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3669     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3670     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3671     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3672     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3673     //     Size prefixes (0x66) are added for larger sizes
 3674 
 3675     while(i >= 22) {
 3676       i -= 11;
 3677       emit_int24(0x66, 0x66, 0x66);
 3678       addr_nop_8();
 3679     }
 3680     // Generate first nop for size between 21-12
 3681     switch (i) {
 3682       case 21:
 3683         i -= 1;
 3684         emit_int8(0x66); // size prefix
 3685       case 20:
 3686       case 19:
 3687         i -= 1;
 3688         emit_int8(0x66); // size prefix
 3689       case 18:
 3690       case 17:
 3691         i -= 1;
 3692         emit_int8(0x66); // size prefix
 3693       case 16:
 3694       case 15:
 3695         i -= 8;
 3696         addr_nop_8();
 3697         break;
 3698       case 14:
 3699       case 13:
 3700         i -= 7;
 3701         addr_nop_7();
 3702         break;
 3703       case 12:
 3704         i -= 6;
 3705         emit_int8(0x66); // size prefix
 3706         addr_nop_5();
 3707         break;
 3708       default:
 3709         assert(i < 12, " ");
 3710     }
 3711 
 3712     // Generate second nop for size between 11-1
 3713     switch (i) {
 3714       case 11:
 3715         emit_int8(0x66); // size prefix
 3716       case 10:
 3717         emit_int8(0x66); // size prefix
 3718       case 9:
 3719         emit_int8(0x66); // size prefix
 3720       case 8:
 3721         addr_nop_8();
 3722         break;
 3723       case 7:
 3724         addr_nop_7();
 3725         break;
 3726       case 6:
 3727         emit_int8(0x66); // size prefix
 3728       case 5:
 3729         addr_nop_5();
 3730         break;
 3731       case 4:
 3732         addr_nop_4();
 3733         break;
 3734       case 3:
 3735         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3736         emit_int8(0x66); // size prefix
 3737       case 2:
 3738         emit_int8(0x66); // size prefix
 3739       case 1:
 3740         emit_int8((unsigned char)0x90);
 3741                          // nop
 3742         break;
 3743       default:
 3744         assert(i == 0, " ");
 3745     }
 3746     return;
 3747   }
 3748 
 3749   if (UseAddressNop && VM_Version::is_zx()) {
 3750     //
 3751     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3752     //  1: 0x90
 3753     //  2: 0x66 0x90
 3754     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3755     //  4: 0x0F 0x1F 0x40 0x00
 3756     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3757     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3758     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3759     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3760     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3761     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3762     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3763 
 3764     // The rest coding is ZX specific - don't use consecutive address nops
 3765 
 3766     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3767     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3768     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3769     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3770 
 3771     while (i >= 15) {
 3772       // For ZX don't generate consecutive address nops (mix with regular nops)
 3773       i -= 15;
 3774       emit_int24(0x66, 0x66, 0x66);
 3775       addr_nop_8();
 3776       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3777     }
 3778     switch (i) {
 3779       case 14:
 3780         emit_int8(0x66); // size prefix
 3781       case 13:
 3782         emit_int8(0x66); // size prefix
 3783       case 12:
 3784         addr_nop_8();
 3785         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3786         break;
 3787       case 11:
 3788         emit_int8(0x66); // size prefix
 3789       case 10:
 3790         emit_int8(0x66); // size prefix
 3791       case 9:
 3792         emit_int8(0x66); // size prefix
 3793       case 8:
 3794         addr_nop_8();
 3795         break;
 3796       case 7:
 3797         addr_nop_7();
 3798         break;
 3799       case 6:
 3800         emit_int8(0x66); // size prefix
 3801       case 5:
 3802         addr_nop_5();
 3803         break;
 3804       case 4:
 3805         addr_nop_4();
 3806         break;
 3807       case 3:
 3808         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3809         emit_int8(0x66); // size prefix
 3810       case 2:
 3811         emit_int8(0x66); // size prefix
 3812       case 1:
 3813         emit_int8((unsigned char)0x90);
 3814                          // nop
 3815         break;
 3816       default:
 3817         assert(i == 0, " ");
 3818     }
 3819     return;
 3820   }
 3821 
 3822   // Using nops with size prefixes "0x66 0x90".
 3823   // From AMD Optimization Guide:
 3824   //  1: 0x90
 3825   //  2: 0x66 0x90
 3826   //  3: 0x66 0x66 0x90
 3827   //  4: 0x66 0x66 0x66 0x90
 3828   //  5: 0x66 0x66 0x90 0x66 0x90
 3829   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3830   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3831   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3832   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3833   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3834   //
 3835   while (i > 12) {
 3836     i -= 4;
 3837     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3838   }
 3839   // 1 - 12 nops
 3840   if (i > 8) {
 3841     if (i > 9) {
 3842       i -= 1;
 3843       emit_int8(0x66);
 3844     }
 3845     i -= 3;
 3846     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3847   }
 3848   // 1 - 8 nops
 3849   if (i > 4) {
 3850     if (i > 6) {
 3851       i -= 1;
 3852       emit_int8(0x66);
 3853     }
 3854     i -= 3;
 3855     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3856   }
 3857   switch (i) {
 3858     case 4:
 3859       emit_int8(0x66);
 3860     case 3:
 3861       emit_int8(0x66);
 3862     case 2:
 3863       emit_int8(0x66);
 3864     case 1:
 3865       emit_int8((unsigned char)0x90);
 3866       break;
 3867     default:
 3868       assert(i == 0, " ");
 3869   }
 3870 }
 3871 
 3872 void Assembler::notl(Register dst) {
 3873   int encode = prefix_and_encode(dst->encoding());
 3874   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 3875 }
 3876 
 3877 void Assembler::orw(Register dst, Register src) {
 3878   (void)prefix_and_encode(dst->encoding(), src->encoding());
 3879   emit_arith(0x0B, 0xC0, dst, src);
 3880 }
 3881 
 3882 void Assembler::orl(Address dst, int32_t imm32) {
 3883   InstructionMark im(this);
 3884   prefix(dst);
 3885   emit_arith_operand(0x81, rcx, dst, imm32);
 3886 }
 3887 
 3888 void Assembler::orl(Register dst, int32_t imm32) {
 3889   prefix(dst);
 3890   emit_arith(0x81, 0xC8, dst, imm32);
 3891 }
 3892 
 3893 void Assembler::orl(Register dst, Address src) {
 3894   InstructionMark im(this);
 3895   prefix(src, dst);
 3896   emit_int8(0x0B);
 3897   emit_operand(dst, src);
 3898 }
 3899 
 3900 void Assembler::orl(Register dst, Register src) {
 3901   (void) prefix_and_encode(dst->encoding(), src->encoding());
 3902   emit_arith(0x0B, 0xC0, dst, src);
 3903 }
 3904 
 3905 void Assembler::orl(Address dst, Register src) {
 3906   InstructionMark im(this);
 3907   prefix(dst, src);
 3908   emit_int8(0x09);
 3909   emit_operand(src, dst);
 3910 }
 3911 
 3912 void Assembler::orb(Address dst, int imm8) {
 3913   InstructionMark im(this);
 3914   prefix(dst);
 3915   emit_int8((unsigned char)0x80);
 3916   emit_operand(rcx, dst, 1);
 3917   emit_int8(imm8);
 3918 }
 3919 
 3920 void Assembler::orb(Address dst, Register src) {
 3921   InstructionMark im(this);
 3922   prefix(dst, src, true);
 3923   emit_int8(0x08);
 3924   emit_operand(src, dst);
 3925 }
 3926 
 3927 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 3928   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3929   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3930   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3931   emit_int16(0x63, (0xC0 | encode));
 3932 }
 3933 
 3934 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3935   assert(UseAVX > 0, "some form of AVX must be enabled");
 3936   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3937   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3938   emit_int16(0x63, (0xC0 | encode));
 3939 }
 3940 
 3941 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 3942   assert(VM_Version::supports_sse2(), "");
 3943   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3944   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3945   emit_int16(0x6B, (0xC0 | encode));
 3946 }
 3947 
 3948 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3949   assert(UseAVX > 0, "some form of AVX must be enabled");
 3950   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3951   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3952   emit_int16(0x6B, (0xC0 | encode));
 3953 }
 3954 
 3955 void Assembler::packuswb(XMMRegister dst, Address src) {
 3956   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3957   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 3958   InstructionMark im(this);
 3959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3960   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 3961   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3962   emit_int8(0x67);
 3963   emit_operand(dst, src);
 3964 }
 3965 
 3966 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 3967   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3968   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3969   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3970   emit_int16(0x67, (0xC0 | encode));
 3971 }
 3972 
 3973 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3974   assert(UseAVX > 0, "some form of AVX must be enabled");
 3975   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3976   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3977   emit_int16(0x67, (0xC0 | encode));
 3978 }
 3979 
 3980 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 3981   assert(VM_Version::supports_sse4_1(), "");
 3982   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3983   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3984   emit_int16(0x2B, (0xC0 | encode));
 3985 }
 3986 
 3987 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3988   assert(UseAVX > 0, "some form of AVX must be enabled");
 3989   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3990   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3991   emit_int16(0x2B, (0xC0 | encode));
 3992 }
 3993 
 3994 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3995   assert(VM_Version::supports_avx2(), "");
 3996   assert(vector_len != AVX_128bit, "");
 3997   // VEX.256.66.0F3A.W1 00 /r ib
 3998   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3999   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4000   emit_int24(0x00, (0xC0 | encode), imm8);
 4001 }
 4002 
 4003 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4004   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4005          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4006   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4007   attributes.set_is_evex_instruction();
 4008   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4009   emit_int16(0x36, (0xC0 | encode));
 4010 }
 4011 
 4012 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4013   assert(VM_Version::supports_avx512_vbmi(), "");
 4014   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4015   attributes.set_is_evex_instruction();
 4016   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4017   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4018 }
 4019 
 4020 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4021   assert(VM_Version::supports_avx512_vbmi(), "");
 4022   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4023   attributes.set_is_evex_instruction();
 4024   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4025   emit_int8((unsigned char)0x8D);
 4026   emit_operand(dst, src);
 4027 }
 4028 
 4029 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4030   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4031          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4032          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4033   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4034   attributes.set_is_evex_instruction();
 4035   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4036   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4037 }
 4038 
 4039 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4040   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4041   // VEX.NDS.256.66.0F38.W0 36 /r
 4042   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4043   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4044   emit_int16(0x36, (0xC0 | encode));
 4045 }
 4046 
 4047 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4048   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4049   // VEX.NDS.256.66.0F38.W0 36 /r
 4050   InstructionMark im(this);
 4051   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4052   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4053   emit_int8(0x36);
 4054   emit_operand(dst, src);
 4055 }
 4056 
 4057 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4058   assert(VM_Version::supports_avx2(), "");
 4059   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4060   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4061   emit_int24(0x46, (0xC0 | encode), imm8);
 4062 }
 4063 
 4064 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4065   assert(VM_Version::supports_avx(), "");
 4066   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4067   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4068   emit_int24(0x06, (0xC0 | encode), imm8);
 4069 }
 4070 
 4071 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4072   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4073   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4074   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4075   emit_int24(0x04, (0xC0 | encode), imm8);
 4076 }
 4077 
 4078 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4079   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4080   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4081   attributes.set_rex_vex_w_reverted();
 4082   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4083   emit_int24(0x05, (0xC0 | encode), imm8);
 4084 }
 4085 
 4086 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4087   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4088   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4089   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4090   emit_int24(0x01, (0xC0 | encode), imm8);
 4091 }
 4092 
 4093 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4094   assert(VM_Version::supports_evex(), "");
 4095   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4096   attributes.set_is_evex_instruction();
 4097   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4098   emit_int16(0x76, (0xC0 | encode));
 4099 }
 4100 
 4101 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4102   assert(VM_Version::supports_avx512_vbmi(), "");
 4103   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4104   attributes.set_is_evex_instruction();
 4105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4106   emit_int16(0x7D, (0xC0 | encode));
 4107 }
 4108 
 4109 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4110   assert(VM_Version::supports_avx512_vbmi(), "");
 4111   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4112   attributes.set_is_evex_instruction();
 4113   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4114   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4115 }
 4116 
 4117 void Assembler::pause() {
 4118   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4119 }
 4120 
 4121 void Assembler::ud2() {
 4122   emit_int16(0x0F, 0x0B);
 4123 }
 4124 
 4125 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4126   assert(VM_Version::supports_sse4_2(), "");
 4127   InstructionMark im(this);
 4128   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4129   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4130   emit_int8(0x61);
 4131   emit_operand(dst, src);
 4132   emit_int8(imm8);
 4133 }
 4134 
 4135 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4136   assert(VM_Version::supports_sse4_2(), "");
 4137   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4138   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4139   emit_int24(0x61, (0xC0 | encode), imm8);
 4140 }
 4141 
 4142 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4143 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4144   assert(VM_Version::supports_sse2(), "");
 4145   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4146   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4147   emit_int16(0x74, (0xC0 | encode));
 4148 }
 4149 
 4150 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4151   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4152   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4153   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4154   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4155   emit_int16(cond_encoding, (0xC0 | encode));
 4156 }
 4157 
 4158 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4159 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4160   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4161   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4162   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4163   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4164   emit_int16(0x74, (0xC0 | encode));
 4165 }
 4166 
 4167 // In this context, kdst is written the mask used to process the equal components
 4168 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4169   assert(VM_Version::supports_avx512bw(), "");
 4170   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4171   attributes.set_is_evex_instruction();
 4172   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4173   emit_int16(0x74, (0xC0 | encode));
 4174 }
 4175 
 4176 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4177   assert(VM_Version::supports_avx512vlbw(), "");
 4178   InstructionMark im(this);
 4179   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4180   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4181   attributes.set_is_evex_instruction();
 4182   int dst_enc = kdst->encoding();
 4183   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4184   emit_int8(0x64);
 4185   emit_operand(as_Register(dst_enc), src);
 4186 }
 4187 
 4188 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4189   assert(VM_Version::supports_avx512vlbw(), "");
 4190   InstructionMark im(this);
 4191   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4192   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4193   attributes.reset_is_clear_context();
 4194   attributes.set_embedded_opmask_register_specifier(mask);
 4195   attributes.set_is_evex_instruction();
 4196   int dst_enc = kdst->encoding();
 4197   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4198   emit_int8(0x64);
 4199   emit_operand(as_Register(dst_enc), src);
 4200 }
 4201 
 4202 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4203   assert(VM_Version::supports_avx512vlbw(), "");
 4204   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4205   attributes.set_is_evex_instruction();
 4206   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4207   emit_int24(0x3E, (0xC0 | encode), vcc);
 4208 }
 4209 
 4210 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4211   assert(VM_Version::supports_avx512vlbw(), "");
 4212   InstructionMark im(this);
 4213   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4214   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4215   attributes.set_is_evex_instruction();
 4216   int dst_enc = kdst->encoding();
 4217   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4218   emit_int8(0x3E);
 4219   emit_operand(as_Register(dst_enc), src);
 4220   emit_int8(vcc);
 4221 }
 4222 
 4223 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4224   assert(VM_Version::supports_avx512bw(), "");
 4225   InstructionMark im(this);
 4226   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4227   attributes.set_is_evex_instruction();
 4228   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4229   int dst_enc = kdst->encoding();
 4230   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4231   emit_int8(0x74);
 4232   emit_operand(as_Register(dst_enc), src);
 4233 }
 4234 
 4235 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4236   assert(VM_Version::supports_avx512vlbw(), "");
 4237   InstructionMark im(this);
 4238   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4239   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4240   attributes.reset_is_clear_context();
 4241   attributes.set_embedded_opmask_register_specifier(mask);
 4242   attributes.set_is_evex_instruction();
 4243   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4244   emit_int8(0x74);
 4245   emit_operand(as_Register(kdst->encoding()), src);
 4246 }
 4247 
 4248 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4249 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4250   assert(VM_Version::supports_sse2(), "");
 4251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4252   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4253   emit_int16(0x75, (0xC0 | encode));
 4254 }
 4255 
 4256 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4257 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4258   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4259   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4260   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4261   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4262   emit_int16(0x75, (0xC0 | encode));
 4263 }
 4264 
 4265 // In this context, kdst is written the mask used to process the equal components
 4266 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4267   assert(VM_Version::supports_avx512bw(), "");
 4268   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4269   attributes.set_is_evex_instruction();
 4270   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4271   emit_int16(0x75, (0xC0 | encode));
 4272 }
 4273 
 4274 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4275   assert(VM_Version::supports_avx512bw(), "");
 4276   InstructionMark im(this);
 4277   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4278   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4279   attributes.set_is_evex_instruction();
 4280   int dst_enc = kdst->encoding();
 4281   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4282   emit_int8(0x75);
 4283   emit_operand(as_Register(dst_enc), src);
 4284 }
 4285 
 4286 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4287 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4288   assert(VM_Version::supports_sse2(), "");
 4289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4291   emit_int16(0x76, (0xC0 | encode));
 4292 }
 4293 
 4294 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4295 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4296   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4297   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4299   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4300   emit_int16(0x76, (0xC0 | encode));
 4301 }
 4302 
 4303 // In this context, kdst is written the mask used to process the equal components
 4304 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4305   assert(VM_Version::supports_evex(), "");
 4306   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4307   attributes.set_is_evex_instruction();
 4308   attributes.reset_is_clear_context();
 4309   attributes.set_embedded_opmask_register_specifier(mask);
 4310   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4311   emit_int16(0x76, (0xC0 | encode));
 4312 }
 4313 
 4314 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4315   assert(VM_Version::supports_evex(), "");
 4316   InstructionMark im(this);
 4317   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4318   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4319   attributes.set_is_evex_instruction();
 4320   attributes.reset_is_clear_context();
 4321   attributes.set_embedded_opmask_register_specifier(mask);
 4322   int dst_enc = kdst->encoding();
 4323   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4324   emit_int8(0x76);
 4325   emit_operand(as_Register(dst_enc), src);
 4326 }
 4327 
 4328 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4329 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4330   assert(VM_Version::supports_sse4_1(), "");
 4331   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4332   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4333   emit_int16(0x29, (0xC0 | encode));
 4334 }
 4335 
 4336 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4337   assert(VM_Version::supports_avx(), "");
 4338   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4339   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4340   emit_int16(cond_encoding, (0xC0 | encode));
 4341 }
 4342 
 4343 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4344 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4345   assert(VM_Version::supports_avx(), "");
 4346   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4347   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4348   emit_int16(0x29, (0xC0 | encode));
 4349 }
 4350 
 4351 // In this context, kdst is written the mask used to process the equal components
 4352 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4353   assert(VM_Version::supports_evex(), "");
 4354   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4355   attributes.reset_is_clear_context();
 4356   attributes.set_is_evex_instruction();
 4357   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4358   emit_int16(0x29, (0xC0 | encode));
 4359 }
 4360 
 4361 // In this context, kdst is written the mask used to process the equal components
 4362 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4363   assert(VM_Version::supports_evex(), "");
 4364   InstructionMark im(this);
 4365   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4366   attributes.reset_is_clear_context();
 4367   attributes.set_is_evex_instruction();
 4368   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4369   int dst_enc = kdst->encoding();
 4370   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4371   emit_int8(0x29);
 4372   emit_operand(as_Register(dst_enc), src);
 4373 }
 4374 
 4375 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4376   assert(VM_Version::supports_sse4_1(), "");
 4377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4378   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4379   emit_int16(0x37, (0xC0 | encode));
 4380 }
 4381 
 4382 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4383   assert(VM_Version::supports_sse2(), "");
 4384   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4385   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4386   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4387 }
 4388 
 4389 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4390   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4391          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4392   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4393   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4394   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4395 }
 4396 
 4397 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4398   assert(VM_Version::supports_avx(), "");
 4399   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4400   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4401   emit_int16(0x50, (0xC0 | encode));
 4402 }
 4403 
 4404 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4405   assert(VM_Version::supports_avx(), "");
 4406   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4407   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4408   emit_int16(0x50, (0xC0 | encode));
 4409 }
 4410 
 4411 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4412   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 4413   InstructionMark im(this);
 4414   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 4415   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4416   emit_int8((unsigned char)0x8C);
 4417   emit_operand(dst, src);
 4418 }
 4419 
 4420 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4421   assert(VM_Version::supports_sse4_1(), "");
 4422   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4423   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4424   emit_int24(0x16, (0xC0 | encode), imm8);
 4425 }
 4426 
 4427 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4428   assert(VM_Version::supports_sse4_1(), "");
 4429   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4430   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4431   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4432   emit_int8(0x16);
 4433   emit_operand(src, dst);
 4434   emit_int8(imm8);
 4435 }
 4436 
 4437 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4438   assert(VM_Version::supports_sse4_1(), "");
 4439   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4440   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4441   emit_int24(0x16, (0xC0 | encode), imm8);
 4442 }
 4443 
 4444 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4445   assert(VM_Version::supports_sse4_1(), "");
 4446   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4447   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4448   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4449   emit_int8(0x16);
 4450   emit_operand(src, dst);
 4451   emit_int8(imm8);
 4452 }
 4453 
 4454 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4455   assert(VM_Version::supports_sse2(), "");
 4456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4457   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4458   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4459 }
 4460 
 4461 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4462   assert(VM_Version::supports_sse4_1(), "");
 4463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4464   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4465   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4466   emit_int8(0x15);
 4467   emit_operand(src, dst);
 4468   emit_int8(imm8);
 4469 }
 4470 
 4471 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4472   assert(VM_Version::supports_sse4_1(), "");
 4473   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4474   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4475   emit_int24(0x14, (0xC0 | encode), imm8);
 4476 }
 4477 
 4478 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4479   assert(VM_Version::supports_sse4_1(), "");
 4480   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4482   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4483   emit_int8(0x14);
 4484   emit_operand(src, dst);
 4485   emit_int8(imm8);
 4486 }
 4487 
 4488 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4489   assert(VM_Version::supports_sse4_1(), "");
 4490   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4491   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4492   emit_int24(0x22, (0xC0 | encode), imm8);
 4493 }
 4494 
 4495 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4496   assert(VM_Version::supports_sse4_1(), "");
 4497   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4498   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4499   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4500   emit_int8(0x22);
 4501   emit_operand(dst,src);
 4502   emit_int8(imm8);
 4503 }
 4504 
 4505 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4506   assert(VM_Version::supports_avx(), "");
 4507   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4508   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4509   emit_int24(0x22, (0xC0 | encode), imm8);
 4510 }
 4511 
 4512 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4513   assert(VM_Version::supports_sse4_1(), "");
 4514   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4515   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4516   emit_int24(0x22, (0xC0 | encode), imm8);
 4517 }
 4518 
 4519 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4520   assert(VM_Version::supports_sse4_1(), "");
 4521   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4522   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4523   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4524   emit_int8(0x22);
 4525   emit_operand(dst, src);
 4526   emit_int8(imm8);
 4527 }
 4528 
 4529 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4530   assert(VM_Version::supports_avx(), "");
 4531   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4532   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4533   emit_int24(0x22, (0xC0 | encode), imm8);
 4534 }
 4535 
 4536 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4537   assert(VM_Version::supports_sse2(), "");
 4538   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4539   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4540   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4541 }
 4542 
 4543 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4544   assert(VM_Version::supports_sse2(), "");
 4545   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4547   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4548   emit_int8((unsigned char)0xC4);
 4549   emit_operand(dst, src);
 4550   emit_int8(imm8);
 4551 }
 4552 
 4553 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4554   assert(VM_Version::supports_avx(), "");
 4555   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4556   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4557   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4558 }
 4559 
 4560 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4561   assert(VM_Version::supports_sse4_1(), "");
 4562   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4563   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4564   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4565   emit_int8(0x20);
 4566   emit_operand(dst, src);
 4567   emit_int8(imm8);
 4568 }
 4569 
 4570 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4571   assert(VM_Version::supports_sse4_1(), "");
 4572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4573   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4574   emit_int24(0x20, (0xC0 | encode), imm8);
 4575 }
 4576 
 4577 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4578   assert(VM_Version::supports_avx(), "");
 4579   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4580   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4581   emit_int24(0x20, (0xC0 | encode), imm8);
 4582 }
 4583 
 4584 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4585   assert(VM_Version::supports_sse4_1(), "");
 4586   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4587   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4588   emit_int24(0x21, (0xC0 | encode), imm8);
 4589 }
 4590 
 4591 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4592   assert(VM_Version::supports_avx(), "");
 4593   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4595   emit_int24(0x21, (0xC0 | encode), imm8);
 4596 }
 4597 
 4598 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4599   assert(VM_Version::supports_sse4_1(), "");
 4600   InstructionMark im(this);
 4601   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4602   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4603   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4604   emit_int8(0x30);
 4605   emit_operand(dst, src);
 4606 }
 4607 
 4608 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4609   assert(VM_Version::supports_sse4_1(), "");
 4610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4611   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4612   emit_int16(0x30, (0xC0 | encode));
 4613 }
 4614 
 4615 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4616   assert(VM_Version::supports_sse4_1(), "");
 4617   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4618   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4619   emit_int16(0x20, (0xC0 | encode));
 4620 }
 4621 
 4622 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4623   assert(VM_Version::supports_sse4_1(), "");
 4624   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4625   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4626   emit_int16(0x35, (0xC0 | encode));
 4627 }
 4628 
 4629 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4630   assert(VM_Version::supports_sse4_1(), "");
 4631   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4632   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4633   emit_int16(0x21, (0xC0 | encode));
 4634 }
 4635 
 4636 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4637   assert(VM_Version::supports_sse4_1(), "");
 4638   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4639   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4640   emit_int16(0x31, (0xC0 | encode));
 4641 }
 4642 
 4643 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4644   assert(VM_Version::supports_sse4_1(), "");
 4645   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4646   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4647   emit_int16(0x22, (0xC0 | encode));
 4648 }
 4649 
 4650 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4651   assert(VM_Version::supports_sse4_1(), "");
 4652   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4653   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4654   emit_int16(0x23, (0xC0 | encode));
 4655 }
 4656 
 4657 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4658   assert(VM_Version::supports_avx(), "");
 4659   InstructionMark im(this);
 4660   assert(dst != xnoreg, "sanity");
 4661   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4662   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4663   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4664   emit_int8(0x30);
 4665   emit_operand(dst, src);
 4666 }
 4667 
 4668 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4669   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4670   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4671   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4672   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4673   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4674   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4675 }
 4676 
 4677 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4678   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4679   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4680   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4681   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4682   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4683   emit_int16(0x20, (0xC0 | encode));
 4684 }
 4685 
 4686 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4687   assert(VM_Version::supports_avx512vlbw(), "");
 4688   assert(dst != xnoreg, "sanity");
 4689   InstructionMark im(this);
 4690   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4691   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4692   attributes.set_embedded_opmask_register_specifier(mask);
 4693   attributes.set_is_evex_instruction();
 4694   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4695   emit_int8(0x30);
 4696   emit_operand(dst, src);
 4697 }
 4698 
 4699 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4700   assert(VM_Version::supports_evex(), "");
 4701   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4702   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4703   attributes.set_is_evex_instruction();
 4704   attributes.set_embedded_opmask_register_specifier(mask);
 4705   if (merge) {
 4706     attributes.reset_is_clear_context();
 4707   }
 4708   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4709   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4710 }
 4711 
 4712 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4713   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4714   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4715   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4716   emit_int16(0x35, (0xC0 | encode));
 4717 }
 4718 
 4719 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4720   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4721   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4722   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4723   emit_int16(0x31, (0xC0 | encode));
 4724 }
 4725 
 4726 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4727   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4728   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4729   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4730   emit_int16(0x32, (0xC0 | encode));
 4731 }
 4732 
 4733 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4734   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4735          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4736              VM_Version::supports_evex(), "");
 4737   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4738   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4739   emit_int16(0x21, (0xC0 | encode));
 4740 }
 4741 
 4742 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4743   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4744          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4745              VM_Version::supports_evex(), "");
 4746   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4747   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4748   emit_int16(0x22, (0xC0 | encode));
 4749 }
 4750 
 4751 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4752   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4753          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4754              VM_Version::supports_evex(), "");
 4755   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4756   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4757   emit_int16(0x23, (0xC0 | encode));
 4758 }
 4759 
 4760 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4761   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4762          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4763              VM_Version::supports_evex(), "");
 4764   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4765   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4766   emit_int16(0x24, (0xC0 | encode));
 4767 }
 4768 
 4769 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4770   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4771          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4772              VM_Version::supports_evex(), "");
 4773   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4774   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4775   emit_int16(0x25, (0xC0 | encode));
 4776 }
 4777 
 4778 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4779   assert(VM_Version::supports_avx512vlbw(), "");
 4780   assert(src != xnoreg, "sanity");
 4781   InstructionMark im(this);
 4782   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4783   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4784   attributes.set_is_evex_instruction();
 4785   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4786   emit_int8(0x30);
 4787   emit_operand(src, dst);
 4788 }
 4789 
 4790 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4791   assert(VM_Version::supports_avx512vlbw(), "");
 4792   assert(src != xnoreg, "sanity");
 4793   InstructionMark im(this);
 4794   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4795   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4796   attributes.reset_is_clear_context();
 4797   attributes.set_embedded_opmask_register_specifier(mask);
 4798   attributes.set_is_evex_instruction();
 4799   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4800   emit_int8(0x30);
 4801   emit_operand(src, dst);
 4802 }
 4803 
 4804 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4805   assert(VM_Version::supports_evex(), "");
 4806   assert(src != xnoreg, "sanity");
 4807   InstructionMark im(this);
 4808   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4809   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4810   attributes.set_is_evex_instruction();
 4811   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4812   emit_int8(0x31);
 4813   emit_operand(src, dst);
 4814 }
 4815 
 4816 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4817   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4818   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4819   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4820   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4821   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4822   emit_int16(0x33, (0xC0 | encode));
 4823 }
 4824 
 4825 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4826   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4827   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4828   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4829   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4830   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4831   emit_int16(0x34, (0xC0 | encode));
 4832 }
 4833 
 4834 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4835   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4836   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4837   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4838   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4839 }
 4840 
 4841 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4842   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4843     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4844     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4845   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4846   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4847   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4848 }
 4849 
 4850 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4851 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4852        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4853        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4854   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4855   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4856   emit_int16(0x04, (0xC0 | encode));
 4857 }
 4858 
 4859 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4860   assert(VM_Version::supports_evex(), "");
 4861   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4862   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4863   attributes.set_is_evex_instruction();
 4864   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4865   emit_int16(0x52, (0xC0 | encode));
 4866 }
 4867 
 4868 // generic
 4869 void Assembler::pop(Register dst) {
 4870   int encode = prefix_and_encode(dst->encoding());
 4871   emit_int8(0x58 | encode);
 4872 }
 4873 
 4874 void Assembler::popcntl(Register dst, Address src) {
 4875   assert(VM_Version::supports_popcnt(), "must support");
 4876   InstructionMark im(this);
 4877   emit_int8((unsigned char)0xF3);
 4878   prefix(src, dst);
 4879   emit_int16(0x0F, (unsigned char)0xB8);
 4880   emit_operand(dst, src);
 4881 }
 4882 
 4883 void Assembler::popcntl(Register dst, Register src) {
 4884   assert(VM_Version::supports_popcnt(), "must support");
 4885   emit_int8((unsigned char)0xF3);
 4886   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 4887   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 4888 }
 4889 
 4890 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4891   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 4892   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4893   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4894   attributes.set_embedded_opmask_register_specifier(mask);
 4895   attributes.set_is_evex_instruction();
 4896   if (merge) {
 4897     attributes.reset_is_clear_context();
 4898   }
 4899   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4900   emit_int16(0x54, (0xC0 | encode));
 4901 }
 4902 
 4903 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4904   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 4905   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4906   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4907   attributes.set_is_evex_instruction();
 4908   attributes.set_embedded_opmask_register_specifier(mask);
 4909   if (merge) {
 4910     attributes.reset_is_clear_context();
 4911   }
 4912   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4913   emit_int16(0x54, (0xC0 | encode));
 4914 }
 4915 
 4916 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4917   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4918   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4919   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4920   attributes.set_is_evex_instruction();
 4921   attributes.set_embedded_opmask_register_specifier(mask);
 4922   if (merge) {
 4923     attributes.reset_is_clear_context();
 4924   }
 4925   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4926   emit_int16(0x55, (0xC0 | encode));
 4927 }
 4928 
 4929 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 4930   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4931   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4932   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4933   attributes.set_is_evex_instruction();
 4934   attributes.set_embedded_opmask_register_specifier(mask);
 4935   if (merge) {
 4936     attributes.reset_is_clear_context();
 4937   }
 4938   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4939   emit_int16(0x55, (0xC0 | encode));
 4940 }
 4941 
 4942 void Assembler::popf() {
 4943   emit_int8((unsigned char)0x9D);
 4944 }
 4945 
 4946 #ifndef _LP64 // no 32bit push/pop on amd64
 4947 void Assembler::popl(Address dst) {
 4948   // NOTE: this will adjust stack by 8byte on 64bits
 4949   InstructionMark im(this);
 4950   prefix(dst);
 4951   emit_int8((unsigned char)0x8F);
 4952   emit_operand(rax, dst);
 4953 }
 4954 #endif
 4955 
 4956 void Assembler::prefetchnta(Address src) {
 4957   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4958   InstructionMark im(this);
 4959   prefix(src);
 4960   emit_int16(0x0F, 0x18);
 4961   emit_operand(rax, src); // 0, src
 4962 }
 4963 
 4964 void Assembler::prefetchr(Address src) {
 4965   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4966   InstructionMark im(this);
 4967   prefix(src);
 4968   emit_int16(0x0F, 0x0D);
 4969   emit_operand(rax, src); // 0, src
 4970 }
 4971 
 4972 void Assembler::prefetcht0(Address src) {
 4973   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4974   InstructionMark im(this);
 4975   prefix(src);
 4976   emit_int16(0x0F, 0x18);
 4977   emit_operand(rcx, src); // 1, src
 4978 }
 4979 
 4980 void Assembler::prefetcht1(Address src) {
 4981   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4982   InstructionMark im(this);
 4983   prefix(src);
 4984   emit_int16(0x0F, 0x18);
 4985   emit_operand(rdx, src); // 2, src
 4986 }
 4987 
 4988 void Assembler::prefetcht2(Address src) {
 4989   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4990   InstructionMark im(this);
 4991   prefix(src);
 4992   emit_int16(0x0F, 0x18);
 4993   emit_operand(rbx, src); // 3, src
 4994 }
 4995 
 4996 void Assembler::prefetchw(Address src) {
 4997   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4998   InstructionMark im(this);
 4999   prefix(src);
 5000   emit_int16(0x0F, 0x0D);
 5001   emit_operand(rcx, src); // 1, src
 5002 }
 5003 
 5004 void Assembler::prefix(Prefix p) {
 5005   emit_int8(p);
 5006 }
 5007 
 5008 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5009   assert(VM_Version::supports_ssse3(), "");
 5010   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5011   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5012   emit_int16(0x00, (0xC0 | encode));
 5013 }
 5014 
 5015 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5016   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5017          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5018          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5019   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5020   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5021   emit_int16(0x00, (0xC0 | encode));
 5022 }
 5023 
 5024 void Assembler::pshufb(XMMRegister dst, Address src) {
 5025   assert(VM_Version::supports_ssse3(), "");
 5026   InstructionMark im(this);
 5027   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5028   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5029   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5030   emit_int8(0x00);
 5031   emit_operand(dst, src);
 5032 }
 5033 
 5034 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5035   assert(isByte(mode), "invalid value");
 5036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5037   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5038   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5039   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5040   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5041 }
 5042 
 5043 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5044   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5045          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5046          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5047   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5048   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5049   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5050   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5051 }
 5052 
 5053 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5054   assert(isByte(mode), "invalid value");
 5055   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5056   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5057   InstructionMark im(this);
 5058   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5059   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5060   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5061   emit_int8(0x70);
 5062   emit_operand(dst, src);
 5063   emit_int8(mode & 0xFF);
 5064 }
 5065 
 5066 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5067   assert(isByte(mode), "invalid value");
 5068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5070   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5071   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5072 }
 5073 
 5074 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5075   assert(isByte(mode), "invalid value");
 5076   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5077   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5078   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5079   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5080 }
 5081 
 5082 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5083   assert(isByte(mode), "invalid value");
 5084   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5085   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5086   InstructionMark im(this);
 5087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5088   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5089   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5090   emit_int8(0x70);
 5091   emit_operand(dst, src);
 5092   emit_int8(mode & 0xFF);
 5093 }
 5094 
 5095 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5096   assert(VM_Version::supports_evex(), "requires EVEX support");
 5097   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5098   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5099   attributes.set_is_evex_instruction();
 5100   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5101   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5102 }
 5103 
 5104 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5105   assert(isByte(imm8), "invalid value");
 5106   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5107   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5108   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5109   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5110 }
 5111 
 5112 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5113   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5114   attributes.set_rex_vex_w_reverted();
 5115   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5116   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5117 }
 5118 
 5119 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 5120   assert(isByte(imm8), "invalid value");
 5121   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5122   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5123   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5124   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5125 }
 5126 
 5127 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5128   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5129   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5130   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5131 }
 5132 
 5133 void Assembler::psrldq(XMMRegister dst, int shift) {
 5134   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5137   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5138   emit_int24(0x73, (0xC0 | encode), shift);
 5139 }
 5140 
 5141 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5142   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5143          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5144          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5145   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5146   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5147   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5148 }
 5149 
 5150 void Assembler::pslldq(XMMRegister dst, int shift) {
 5151   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5152   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5153   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5154   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5155   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5156   emit_int24(0x73, (0xC0 | encode), shift);
 5157 }
 5158 
 5159 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5160   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5161          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5162          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5163   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5164   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5165   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5166 }
 5167 
 5168 void Assembler::ptest(XMMRegister dst, Address src) {
 5169   assert(VM_Version::supports_sse4_1(), "");
 5170   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5171   InstructionMark im(this);
 5172   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5173   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5174   emit_int8(0x17);
 5175   emit_operand(dst, src);
 5176 }
 5177 
 5178 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5179   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5180   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5181   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5182   emit_int8(0x17);
 5183   emit_int8((0xC0 | encode));
 5184 }
 5185 
 5186 void Assembler::vptest(XMMRegister dst, Address src) {
 5187   assert(VM_Version::supports_avx(), "");
 5188   InstructionMark im(this);
 5189   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5190   assert(dst != xnoreg, "sanity");
 5191   // swap src<->dst for encoding
 5192   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5193   emit_int8(0x17);
 5194   emit_operand(dst, src);
 5195 }
 5196 
 5197 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5198   assert(VM_Version::supports_avx(), "");
 5199   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5200   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5201   emit_int16(0x17, (0xC0 | encode));
 5202 }
 5203 
 5204 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5205   assert(VM_Version::supports_avx(), "");
 5206   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5207   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5208   emit_int16(0x17, (0xC0 | encode));
 5209 }
 5210 
 5211 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5212   assert(VM_Version::supports_avx512vlbw(), "");
 5213   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5214   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5215   attributes.set_is_evex_instruction();
 5216   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5217   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5218 }
 5219 
 5220 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5221   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5222   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5223   InstructionMark im(this);
 5224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5225   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5226   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5227   emit_int8(0x60);
 5228   emit_operand(dst, src);
 5229 }
 5230 
 5231 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5232   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5233   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5234   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5235   emit_int16(0x60, (0xC0 | encode));
 5236 }
 5237 
 5238 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5239   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5240   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5241   InstructionMark im(this);
 5242   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5243   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5244   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5245   emit_int8(0x62);
 5246   emit_operand(dst, src);
 5247 }
 5248 
 5249 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5252   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5253   emit_int16(0x62, (0xC0 | encode));
 5254 }
 5255 
 5256 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5258   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5259   attributes.set_rex_vex_w_reverted();
 5260   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5261   emit_int16(0x6C, (0xC0 | encode));
 5262 }
 5263 
 5264 void Assembler::push(int32_t imm32) {
 5265   // in 64bits we push 64bits onto the stack but only
 5266   // take a 32bit immediate
 5267   emit_int8(0x68);
 5268   emit_int32(imm32);
 5269 }
 5270 
 5271 void Assembler::push(Register src) {
 5272   int encode = prefix_and_encode(src->encoding());
 5273   emit_int8(0x50 | encode);
 5274 }
 5275 
 5276 void Assembler::pushf() {
 5277   emit_int8((unsigned char)0x9C);
 5278 }
 5279 
 5280 #ifndef _LP64 // no 32bit push/pop on amd64
 5281 void Assembler::pushl(Address src) {
 5282   // Note this will push 64bit on 64bit
 5283   InstructionMark im(this);
 5284   prefix(src);
 5285   emit_int8((unsigned char)0xFF);
 5286   emit_operand(rsi, src);
 5287 }
 5288 #endif
 5289 
 5290 void Assembler::rcll(Register dst, int imm8) {
 5291   assert(isShiftCount(imm8), "illegal shift count");
 5292   int encode = prefix_and_encode(dst->encoding());
 5293   if (imm8 == 1) {
 5294     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5295   } else {
 5296     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5297   }
 5298 }
 5299 
 5300 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5301   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5303   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5304   emit_int16(0x53, (0xC0 | encode));
 5305 }
 5306 
 5307 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5308   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5310   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5311   emit_int16(0x53, (0xC0 | encode));
 5312 }
 5313 
 5314 void Assembler::rdtsc() {
 5315   emit_int16(0x0F, 0x31);
 5316 }
 5317 
 5318 // copies data from [esi] to [edi] using rcx pointer sized words
 5319 // generic
 5320 void Assembler::rep_mov() {
 5321   // REP
 5322   // MOVSQ
 5323   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5324   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5325 }
 5326 
 5327 // sets rcx bytes with rax, value at [edi]
 5328 void Assembler::rep_stosb() {
 5329   // REP
 5330   // STOSB
 5331   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5332   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5333 }
 5334 
 5335 // sets rcx pointer sized words with rax, value at [edi]
 5336 // generic
 5337 void Assembler::rep_stos() {
 5338   // REP
 5339   // LP64:STOSQ, LP32:STOSD
 5340   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5341   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5342 }
 5343 
 5344 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5345 // generic
 5346 void Assembler::repne_scan() { // repne_scan
 5347   // SCASQ
 5348   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5349   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5350 }
 5351 
 5352 #ifdef _LP64
 5353 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5354 // generic
 5355 void Assembler::repne_scanl() { // repne_scan
 5356   // SCASL
 5357   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5358 }
 5359 #endif
 5360 
 5361 void Assembler::ret(int imm16) {
 5362   if (imm16 == 0) {
 5363     emit_int8((unsigned char)0xC3);
 5364   } else {
 5365     emit_int8((unsigned char)0xC2);
 5366     emit_int16(imm16);
 5367   }
 5368 }
 5369 
 5370 void Assembler::roll(Register dst, int imm8) {
 5371   assert(isShiftCount(imm8), "illegal shift count");
 5372   int encode = prefix_and_encode(dst->encoding());
 5373   if (imm8 == 1) {
 5374     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5375   } else {
 5376     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5377   }
 5378 }
 5379 
 5380 void Assembler::roll(Register dst) {
 5381   int encode = prefix_and_encode(dst->encoding());
 5382   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5383 }
 5384 
 5385 void Assembler::rorl(Register dst, int imm8) {
 5386   assert(isShiftCount(imm8), "illegal shift count");
 5387   int encode = prefix_and_encode(dst->encoding());
 5388   if (imm8 == 1) {
 5389     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5390   } else {
 5391     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5392   }
 5393 }
 5394 
 5395 void Assembler::rorl(Register dst) {
 5396   int encode = prefix_and_encode(dst->encoding());
 5397   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5398 }
 5399 
 5400 #ifdef _LP64
 5401 void Assembler::rorq(Register dst) {
 5402   int encode = prefixq_and_encode(dst->encoding());
 5403   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5404 }
 5405 
 5406 void Assembler::rorq(Register dst, int imm8) {
 5407   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5408   int encode = prefixq_and_encode(dst->encoding());
 5409   if (imm8 == 1) {
 5410     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5411   } else {
 5412     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5413   }
 5414 }
 5415 
 5416 void Assembler::rolq(Register dst) {
 5417   int encode = prefixq_and_encode(dst->encoding());
 5418   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5419 }
 5420 
 5421 void Assembler::rolq(Register dst, int imm8) {
 5422   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5423   int encode = prefixq_and_encode(dst->encoding());
 5424   if (imm8 == 1) {
 5425     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5426   } else {
 5427     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5428   }
 5429 }
 5430 #endif
 5431 
 5432 void Assembler::sahf() {
 5433 #ifdef _LP64
 5434   // Not supported in 64bit mode
 5435   ShouldNotReachHere();
 5436 #endif
 5437   emit_int8((unsigned char)0x9E);
 5438 }
 5439 
 5440 void Assembler::sall(Address dst, int imm8) {
 5441   InstructionMark im(this);
 5442   assert(isShiftCount(imm8), "illegal shift count");
 5443   prefix(dst);
 5444   if (imm8 == 1) {
 5445     emit_int8((unsigned char)0xD1);
 5446     emit_operand(as_Register(4), dst);
 5447   }
 5448   else {
 5449     emit_int8((unsigned char)0xC1);
 5450     emit_operand(as_Register(4), dst);
 5451     emit_int8(imm8);
 5452   }
 5453 }
 5454 
 5455 void Assembler::sall(Address dst) {
 5456   InstructionMark im(this);
 5457   prefix(dst);
 5458   emit_int8((unsigned char)0xD3);
 5459   emit_operand(as_Register(4), dst);
 5460 }
 5461 
 5462 void Assembler::sall(Register dst, int imm8) {
 5463   assert(isShiftCount(imm8), "illegal shift count");
 5464   int encode = prefix_and_encode(dst->encoding());
 5465   if (imm8 == 1) {
 5466     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5467   } else {
 5468     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5469   }
 5470 }
 5471 
 5472 void Assembler::sall(Register dst) {
 5473   int encode = prefix_and_encode(dst->encoding());
 5474   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5475 }
 5476 
 5477 void Assembler::sarl(Address dst, int imm8) {
 5478   assert(isShiftCount(imm8), "illegal shift count");
 5479   InstructionMark im(this);
 5480   prefix(dst);
 5481   if (imm8 == 1) {
 5482     emit_int8((unsigned char)0xD1);
 5483     emit_operand(as_Register(7), dst);
 5484   }
 5485   else {
 5486     emit_int8((unsigned char)0xC1);
 5487     emit_operand(as_Register(7), dst);
 5488     emit_int8(imm8);
 5489   }
 5490 }
 5491 
 5492 void Assembler::sarl(Address dst) {
 5493   InstructionMark im(this);
 5494   prefix(dst);
 5495   emit_int8((unsigned char)0xD3);
 5496   emit_operand(as_Register(7), dst);
 5497 }
 5498 
 5499 void Assembler::sarl(Register dst, int imm8) {
 5500   int encode = prefix_and_encode(dst->encoding());
 5501   assert(isShiftCount(imm8), "illegal shift count");
 5502   if (imm8 == 1) {
 5503     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5504   } else {
 5505     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5506   }
 5507 }
 5508 
 5509 void Assembler::sarl(Register dst) {
 5510   int encode = prefix_and_encode(dst->encoding());
 5511   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5512 }
 5513 
 5514 void Assembler::sbbl(Address dst, int32_t imm32) {
 5515   InstructionMark im(this);
 5516   prefix(dst);
 5517   emit_arith_operand(0x81, rbx, dst, imm32);
 5518 }
 5519 
 5520 void Assembler::sbbl(Register dst, int32_t imm32) {
 5521   prefix(dst);
 5522   emit_arith(0x81, 0xD8, dst, imm32);
 5523 }
 5524 
 5525 
 5526 void Assembler::sbbl(Register dst, Address src) {
 5527   InstructionMark im(this);
 5528   prefix(src, dst);
 5529   emit_int8(0x1B);
 5530   emit_operand(dst, src);
 5531 }
 5532 
 5533 void Assembler::sbbl(Register dst, Register src) {
 5534   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5535   emit_arith(0x1B, 0xC0, dst, src);
 5536 }
 5537 
 5538 void Assembler::setb(Condition cc, Register dst) {
 5539   assert(0 <= cc && cc < 16, "illegal cc");
 5540   int encode = prefix_and_encode(dst->encoding(), true);
 5541   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5542 }
 5543 
 5544 void Assembler::sete(Register dst) {
 5545   int encode = prefix_and_encode(dst->encoding(), true);
 5546   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5547 }
 5548 
 5549 void Assembler::setl(Register dst) {
 5550   int encode = prefix_and_encode(dst->encoding(), true);
 5551   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5552 }
 5553 
 5554 void Assembler::setne(Register dst) {
 5555   int encode = prefix_and_encode(dst->encoding(), true);
 5556   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5557 }
 5558 
 5559 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5560   assert(VM_Version::supports_ssse3(), "");
 5561   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5562   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5563   emit_int24(0x0F, (0xC0 | encode), imm8);
 5564 }
 5565 
 5566 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5567   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5568          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5569          0, "");
 5570   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5571   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5572   emit_int24(0x0F, (0xC0 | encode), imm8);
 5573 }
 5574 
 5575 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5576   assert(VM_Version::supports_evex(), "");
 5577   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5578   attributes.set_is_evex_instruction();
 5579   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5580   emit_int24(0x3, (0xC0 | encode), imm8);
 5581 }
 5582 
 5583 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5584   assert(VM_Version::supports_sse4_1(), "");
 5585   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5586   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5587   emit_int24(0x0E, (0xC0 | encode), imm8);
 5588 }
 5589 
 5590 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5591   assert(VM_Version::supports_sha(), "");
 5592   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5593   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5594 }
 5595 
 5596 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5597   assert(VM_Version::supports_sha(), "");
 5598   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5599   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5600 }
 5601 
 5602 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5603   assert(VM_Version::supports_sha(), "");
 5604   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5605   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5606 }
 5607 
 5608 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5609   assert(VM_Version::supports_sha(), "");
 5610   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5611   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5612 }
 5613 
 5614 // xmm0 is implicit additional source to this instruction.
 5615 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5616   assert(VM_Version::supports_sha(), "");
 5617   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5618   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5619 }
 5620 
 5621 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5622   assert(VM_Version::supports_sha(), "");
 5623   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5624   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5625 }
 5626 
 5627 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5628   assert(VM_Version::supports_sha(), "");
 5629   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5630   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5631 }
 5632 
 5633 
 5634 void Assembler::shll(Register dst, int imm8) {
 5635   assert(isShiftCount(imm8), "illegal shift count");
 5636   int encode = prefix_and_encode(dst->encoding());
 5637   if (imm8 == 1 ) {
 5638     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5639   } else {
 5640     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5641   }
 5642 }
 5643 
 5644 void Assembler::shll(Register dst) {
 5645   int encode = prefix_and_encode(dst->encoding());
 5646   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5647 }
 5648 
 5649 void Assembler::shrl(Register dst, int imm8) {
 5650   assert(isShiftCount(imm8), "illegal shift count");
 5651   int encode = prefix_and_encode(dst->encoding());
 5652   if (imm8 == 1) {
 5653     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5654   }
 5655   else {
 5656     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5657   }
 5658 }
 5659 
 5660 void Assembler::shrl(Register dst) {
 5661   int encode = prefix_and_encode(dst->encoding());
 5662   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5663 }
 5664 
 5665 void Assembler::shrl(Address dst) {
 5666   InstructionMark im(this);
 5667   prefix(dst);
 5668   emit_int8((unsigned char)0xD3);
 5669   emit_operand(as_Register(5), dst);
 5670 }
 5671 
 5672 void Assembler::shrl(Address dst, int imm8) {
 5673   InstructionMark im(this);
 5674   assert(isShiftCount(imm8), "illegal shift count");
 5675   prefix(dst);
 5676   if (imm8 == 1) {
 5677     emit_int8((unsigned char)0xD1);
 5678     emit_operand(as_Register(5), dst);
 5679   }
 5680   else {
 5681     emit_int8((unsigned char)0xC1);
 5682     emit_operand(as_Register(5), dst);
 5683     emit_int8(imm8);
 5684   }
 5685 }
 5686 
 5687 
 5688 void Assembler::shldl(Register dst, Register src) {
 5689   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5690   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5691 }
 5692 
 5693 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5694   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5695   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5696 }
 5697 
 5698 void Assembler::shrdl(Register dst, Register src) {
 5699   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5700   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5701 }
 5702 
 5703 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5704   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5705   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5706 }
 5707 
 5708 // copies a single word from [esi] to [edi]
 5709 void Assembler::smovl() {
 5710   emit_int8((unsigned char)0xA5);
 5711 }
 5712 
 5713 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5714   assert(VM_Version::supports_sse4_1(), "");
 5715   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5716   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5717   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5718 }
 5719 
 5720 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5721   assert(VM_Version::supports_sse4_1(), "");
 5722   InstructionMark im(this);
 5723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5724   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5725   emit_int8(0x0B);
 5726   emit_operand(dst, src);
 5727   emit_int8((unsigned char)rmode);
 5728 }
 5729 
 5730 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5732   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5733   attributes.set_rex_vex_w_reverted();
 5734   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5735   emit_int16(0x51, (0xC0 | encode));
 5736 }
 5737 
 5738 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5740   InstructionMark im(this);
 5741   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5742   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5743   attributes.set_rex_vex_w_reverted();
 5744   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5745   emit_int8(0x51);
 5746   emit_operand(dst, src);
 5747 }
 5748 
 5749 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5750   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5751   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5752   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5753   emit_int16(0x51, (0xC0 | encode));
 5754 }
 5755 
 5756 void Assembler::std() {
 5757   emit_int8((unsigned char)0xFD);
 5758 }
 5759 
 5760 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5761   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5762   InstructionMark im(this);
 5763   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5764   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5765   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5766   emit_int8(0x51);
 5767   emit_operand(dst, src);
 5768 }
 5769 
 5770 void Assembler::stmxcsr( Address dst) {
 5771   if (UseAVX > 0 ) {
 5772     assert(VM_Version::supports_avx(), "");
 5773     InstructionMark im(this);
 5774     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5775     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5776     emit_int8((unsigned char)0xAE);
 5777     emit_operand(as_Register(3), dst);
 5778   } else {
 5779     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5780     InstructionMark im(this);
 5781     prefix(dst);
 5782     emit_int16(0x0F, (unsigned char)0xAE);
 5783     emit_operand(as_Register(3), dst);
 5784   }
 5785 }
 5786 
 5787 void Assembler::subl(Address dst, int32_t imm32) {
 5788   InstructionMark im(this);
 5789   prefix(dst);
 5790   emit_arith_operand(0x81, rbp, dst, imm32);
 5791 }
 5792 
 5793 void Assembler::subl(Address dst, Register src) {
 5794   InstructionMark im(this);
 5795   prefix(dst, src);
 5796   emit_int8(0x29);
 5797   emit_operand(src, dst);
 5798 }
 5799 
 5800 void Assembler::subl(Register dst, int32_t imm32) {
 5801   prefix(dst);
 5802   emit_arith(0x81, 0xE8, dst, imm32);
 5803 }
 5804 
 5805 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5806 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5807   prefix(dst);
 5808   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5809 }
 5810 
 5811 void Assembler::subl(Register dst, Address src) {
 5812   InstructionMark im(this);
 5813   prefix(src, dst);
 5814   emit_int8(0x2B);
 5815   emit_operand(dst, src);
 5816 }
 5817 
 5818 void Assembler::subl(Register dst, Register src) {
 5819   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5820   emit_arith(0x2B, 0xC0, dst, src);
 5821 }
 5822 
 5823 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5824   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5825   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5826   attributes.set_rex_vex_w_reverted();
 5827   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5828   emit_int16(0x5C, (0xC0 | encode));
 5829 }
 5830 
 5831 void Assembler::subsd(XMMRegister dst, Address src) {
 5832   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5833   InstructionMark im(this);
 5834   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5835   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5836   attributes.set_rex_vex_w_reverted();
 5837   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5838   emit_int8(0x5C);
 5839   emit_operand(dst, src);
 5840 }
 5841 
 5842 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5843   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5844   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5845   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5846   emit_int16(0x5C, (0xC0 | encode));
 5847 }
 5848 
 5849 void Assembler::subss(XMMRegister dst, Address src) {
 5850   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5851   InstructionMark im(this);
 5852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5853   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5854   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5855   emit_int8(0x5C);
 5856   emit_operand(dst, src);
 5857 }
 5858 
 5859 void Assembler::testb(Register dst, int imm8) {
 5860   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5861   if (dst == rax) {
 5862     emit_int8((unsigned char)0xA8);
 5863     emit_int8(imm8);
 5864   } else {
 5865     (void) prefix_and_encode(dst->encoding(), true);
 5866     emit_arith_b(0xF6, 0xC0, dst, imm8);
 5867   }
 5868 }
 5869 
 5870 void Assembler::testb(Address dst, int imm8) {
 5871   InstructionMark im(this);
 5872   prefix(dst);
 5873   emit_int8((unsigned char)0xF6);
 5874   emit_operand(rax, dst, 1);
 5875   emit_int8(imm8);
 5876 }
 5877 
 5878 void Assembler::testl(Address dst, int32_t imm32) {
 5879   if (imm32 >= 0 && is8bit(imm32)) {
 5880     testb(dst, imm32);
 5881     return;
 5882   }
 5883   InstructionMark im(this);
 5884   emit_int8((unsigned char)0xF7);
 5885   emit_operand(as_Register(0), dst);
 5886   emit_int32(imm32);
 5887 }
 5888 
 5889 void Assembler::testl(Register dst, int32_t imm32) {
 5890   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 5891     testb(dst, imm32);
 5892     return;
 5893   }
 5894   // not using emit_arith because test
 5895   // doesn't support sign-extension of
 5896   // 8bit operands
 5897   if (dst == rax) {
 5898     emit_int8((unsigned char)0xA9);
 5899     emit_int32(imm32);
 5900   } else {
 5901     int encode = dst->encoding();
 5902     encode = prefix_and_encode(encode);
 5903     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5904     emit_int32(imm32);
 5905   }
 5906 }
 5907 
 5908 void Assembler::testl(Register dst, Register src) {
 5909   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5910   emit_arith(0x85, 0xC0, dst, src);
 5911 }
 5912 
 5913 void Assembler::testl(Register dst, Address src) {
 5914   InstructionMark im(this);
 5915   prefix(src, dst);
 5916   emit_int8((unsigned char)0x85);
 5917   emit_operand(dst, src);
 5918 }
 5919 
 5920 void Assembler::tzcntl(Register dst, Register src) {
 5921   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5922   emit_int8((unsigned char)0xF3);
 5923   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5924   emit_int24(0x0F,
 5925              (unsigned char)0xBC,
 5926              0xC0 | encode);
 5927 }
 5928 
 5929 void Assembler::tzcntq(Register dst, Register src) {
 5930   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5931   emit_int8((unsigned char)0xF3);
 5932   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5933   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5934 }
 5935 
 5936 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5937   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5938   InstructionMark im(this);
 5939   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5940   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5941   attributes.set_rex_vex_w_reverted();
 5942   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5943   emit_int8(0x2E);
 5944   emit_operand(dst, src);
 5945 }
 5946 
 5947 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 5948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5949   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5950   attributes.set_rex_vex_w_reverted();
 5951   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5952   emit_int16(0x2E, (0xC0 | encode));
 5953 }
 5954 
 5955 void Assembler::ucomiss(XMMRegister dst, Address src) {
 5956   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5957   InstructionMark im(this);
 5958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5959   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5960   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5961   emit_int8(0x2E);
 5962   emit_operand(dst, src);
 5963 }
 5964 
 5965 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 5966   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5967   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5968   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5969   emit_int16(0x2E, (0xC0 | encode));
 5970 }
 5971 
 5972 void Assembler::xabort(int8_t imm8) {
 5973   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 5974 }
 5975 
 5976 void Assembler::xaddb(Address dst, Register src) {
 5977   InstructionMark im(this);
 5978   prefix(dst, src, true);
 5979   emit_int16(0x0F, (unsigned char)0xC0);
 5980   emit_operand(src, dst);
 5981 }
 5982 
 5983 void Assembler::xaddw(Address dst, Register src) {
 5984   InstructionMark im(this);
 5985   emit_int8(0x66);
 5986   prefix(dst, src);
 5987   emit_int16(0x0F, (unsigned char)0xC1);
 5988   emit_operand(src, dst);
 5989 }
 5990 
 5991 void Assembler::xaddl(Address dst, Register src) {
 5992   InstructionMark im(this);
 5993   prefix(dst, src);
 5994   emit_int16(0x0F, (unsigned char)0xC1);
 5995   emit_operand(src, dst);
 5996 }
 5997 
 5998 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 5999   InstructionMark im(this);
 6000   relocate(rtype);
 6001   if (abort.is_bound()) {
 6002     address entry = target(abort);
 6003     assert(entry != NULL, "abort entry NULL");
 6004     intptr_t offset = entry - pc();
 6005     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6006     emit_int32(offset - 6); // 2 opcode + 4 address
 6007   } else {
 6008     abort.add_patch_at(code(), locator());
 6009     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6010     emit_int32(0);
 6011   }
 6012 }
 6013 
 6014 void Assembler::xchgb(Register dst, Address src) { // xchg
 6015   InstructionMark im(this);
 6016   prefix(src, dst, true);
 6017   emit_int8((unsigned char)0x86);
 6018   emit_operand(dst, src);
 6019 }
 6020 
 6021 void Assembler::xchgw(Register dst, Address src) { // xchg
 6022   InstructionMark im(this);
 6023   emit_int8(0x66);
 6024   prefix(src, dst);
 6025   emit_int8((unsigned char)0x87);
 6026   emit_operand(dst, src);
 6027 }
 6028 
 6029 void Assembler::xchgl(Register dst, Address src) { // xchg
 6030   InstructionMark im(this);
 6031   prefix(src, dst);
 6032   emit_int8((unsigned char)0x87);
 6033   emit_operand(dst, src);
 6034 }
 6035 
 6036 void Assembler::xchgl(Register dst, Register src) {
 6037   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6038   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6039 }
 6040 
 6041 void Assembler::xend() {
 6042   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6043 }
 6044 
 6045 void Assembler::xgetbv() {
 6046   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6047 }
 6048 
 6049 void Assembler::xorl(Address dst, int32_t imm32) {
 6050   InstructionMark im(this);
 6051   prefix(dst);
 6052   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6053 }
 6054 
 6055 void Assembler::xorl(Register dst, int32_t imm32) {
 6056   prefix(dst);
 6057   emit_arith(0x81, 0xF0, dst, imm32);
 6058 }
 6059 
 6060 void Assembler::xorl(Register dst, Address src) {
 6061   InstructionMark im(this);
 6062   prefix(src, dst);
 6063   emit_int8(0x33);
 6064   emit_operand(dst, src);
 6065 }
 6066 
 6067 void Assembler::xorl(Register dst, Register src) {
 6068   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6069   emit_arith(0x33, 0xC0, dst, src);
 6070 }
 6071 
 6072 void Assembler::xorl(Address dst, Register src) {
 6073   InstructionMark im(this);
 6074   prefix(dst, src);
 6075   emit_int8(0x31);
 6076   emit_operand(src, dst);
 6077 }
 6078 
 6079 void Assembler::xorb(Register dst, Address src) {
 6080   InstructionMark im(this);
 6081   prefix(src, dst);
 6082   emit_int8(0x32);
 6083   emit_operand(dst, src);
 6084 }
 6085 
 6086 void Assembler::xorb(Address dst, Register src) {
 6087   InstructionMark im(this);
 6088   prefix(dst, src, true);
 6089   emit_int8(0x30);
 6090   emit_operand(src, dst);
 6091 }
 6092 
 6093 void Assembler::xorw(Register dst, Register src) {
 6094   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6095   emit_arith(0x33, 0xC0, dst, src);
 6096 }
 6097 
 6098 // AVX 3-operands scalar float-point arithmetic instructions
 6099 
 6100 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6101   assert(VM_Version::supports_avx(), "");
 6102   InstructionMark im(this);
 6103   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6104   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6105   attributes.set_rex_vex_w_reverted();
 6106   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6107   emit_int8(0x58);
 6108   emit_operand(dst, src);
 6109 }
 6110 
 6111 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6112   assert(VM_Version::supports_avx(), "");
 6113   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6114   attributes.set_rex_vex_w_reverted();
 6115   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6116   emit_int16(0x58, (0xC0 | encode));
 6117 }
 6118 
 6119 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6120   assert(VM_Version::supports_avx(), "");
 6121   InstructionMark im(this);
 6122   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6123   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6124   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6125   emit_int8(0x58);
 6126   emit_operand(dst, src);
 6127 }
 6128 
 6129 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6130   assert(VM_Version::supports_avx(), "");
 6131   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6132   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6133   emit_int16(0x58, (0xC0 | encode));
 6134 }
 6135 
 6136 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6137   assert(VM_Version::supports_avx(), "");
 6138   InstructionMark im(this);
 6139   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6140   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6141   attributes.set_rex_vex_w_reverted();
 6142   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6143   emit_int8(0x5E);
 6144   emit_operand(dst, src);
 6145 }
 6146 
 6147 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6148   assert(VM_Version::supports_avx(), "");
 6149   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6150   attributes.set_rex_vex_w_reverted();
 6151   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6152   emit_int16(0x5E, (0xC0 | encode));
 6153 }
 6154 
 6155 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6156   assert(VM_Version::supports_avx(), "");
 6157   InstructionMark im(this);
 6158   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6159   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6160   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6161   emit_int8(0x5E);
 6162   emit_operand(dst, src);
 6163 }
 6164 
 6165 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6166   assert(VM_Version::supports_avx(), "");
 6167   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6168   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6169   emit_int16(0x5E, (0xC0 | encode));
 6170 }
 6171 
 6172 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6173   assert(VM_Version::supports_fma(), "");
 6174   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6175   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6176   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6177 }
 6178 
 6179 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6180   assert(VM_Version::supports_fma(), "");
 6181   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6182   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6183   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6184 }
 6185 
 6186 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6187   assert(VM_Version::supports_avx(), "");
 6188   InstructionMark im(this);
 6189   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6190   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6191   attributes.set_rex_vex_w_reverted();
 6192   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6193   emit_int8(0x59);
 6194   emit_operand(dst, src);
 6195 }
 6196 
 6197 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6198   assert(VM_Version::supports_avx(), "");
 6199   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6200   attributes.set_rex_vex_w_reverted();
 6201   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6202   emit_int16(0x59, (0xC0 | encode));
 6203 }
 6204 
 6205 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6206   assert(VM_Version::supports_avx(), "");
 6207   InstructionMark im(this);
 6208   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6209   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6210   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6211   emit_int8(0x59);
 6212   emit_operand(dst, src);
 6213 }
 6214 
 6215 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6216   assert(VM_Version::supports_avx(), "");
 6217   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6218   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6219   emit_int16(0x59, (0xC0 | encode));
 6220 }
 6221 
 6222 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6223   assert(VM_Version::supports_avx(), "");
 6224   InstructionMark im(this);
 6225   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6226   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6227   attributes.set_rex_vex_w_reverted();
 6228   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6229   emit_int8(0x5C);
 6230   emit_operand(dst, src);
 6231 }
 6232 
 6233 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6234   assert(VM_Version::supports_avx(), "");
 6235   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6236   attributes.set_rex_vex_w_reverted();
 6237   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6238   emit_int16(0x5C, (0xC0 | encode));
 6239 }
 6240 
 6241 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6242   assert(VM_Version::supports_avx(), "");
 6243   InstructionMark im(this);
 6244   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6245   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6246   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6247   emit_int8(0x5C);
 6248   emit_operand(dst, src);
 6249 }
 6250 
 6251 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6252   assert(VM_Version::supports_avx(), "");
 6253   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6254   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6255   emit_int16(0x5C, (0xC0 | encode));
 6256 }
 6257 
 6258 //====================VECTOR ARITHMETIC=====================================
 6259 
 6260 // Float-point vector arithmetic
 6261 
 6262 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6264   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6265   attributes.set_rex_vex_w_reverted();
 6266   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6267   emit_int16(0x58, (0xC0 | encode));
 6268 }
 6269 
 6270 void Assembler::addpd(XMMRegister dst, Address src) {
 6271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6272   InstructionMark im(this);
 6273   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6274   attributes.set_rex_vex_w_reverted();
 6275   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6276   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6277   emit_int8(0x58);
 6278   emit_operand(dst, src);
 6279 }
 6280 
 6281 
 6282 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6283   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6284   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6285   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6286   emit_int16(0x58, (0xC0 | encode));
 6287 }
 6288 
 6289 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6290   assert(VM_Version::supports_avx(), "");
 6291   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6292   attributes.set_rex_vex_w_reverted();
 6293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6294   emit_int16(0x58, (0xC0 | encode));
 6295 }
 6296 
 6297 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6298   assert(VM_Version::supports_avx(), "");
 6299   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6300   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6301   emit_int16(0x58, (0xC0 | encode));
 6302 }
 6303 
 6304 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6305   assert(VM_Version::supports_avx(), "");
 6306   InstructionMark im(this);
 6307   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6308   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6309   attributes.set_rex_vex_w_reverted();
 6310   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6311   emit_int8(0x58);
 6312   emit_operand(dst, src);
 6313 }
 6314 
 6315 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6316   assert(VM_Version::supports_avx(), "");
 6317   InstructionMark im(this);
 6318   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6319   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6320   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6321   emit_int8(0x58);
 6322   emit_operand(dst, src);
 6323 }
 6324 
 6325 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6327   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6328   attributes.set_rex_vex_w_reverted();
 6329   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6330   emit_int16(0x5C, (0xC0 | encode));
 6331 }
 6332 
 6333 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6337   emit_int16(0x5C, (0xC0 | encode));
 6338 }
 6339 
 6340 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6341   assert(VM_Version::supports_avx(), "");
 6342   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6343   attributes.set_rex_vex_w_reverted();
 6344   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6345   emit_int16(0x5C, (0xC0 | encode));
 6346 }
 6347 
 6348 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6349   assert(VM_Version::supports_avx(), "");
 6350   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6351   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6352   emit_int16(0x5C, (0xC0 | encode));
 6353 }
 6354 
 6355 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6356   assert(VM_Version::supports_avx(), "");
 6357   InstructionMark im(this);
 6358   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6359   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6360   attributes.set_rex_vex_w_reverted();
 6361   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6362   emit_int8(0x5C);
 6363   emit_operand(dst, src);
 6364 }
 6365 
 6366 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6367   assert(VM_Version::supports_avx(), "");
 6368   InstructionMark im(this);
 6369   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6370   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6371   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6372   emit_int8(0x5C);
 6373   emit_operand(dst, src);
 6374 }
 6375 
 6376 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6377   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6378   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6379   attributes.set_rex_vex_w_reverted();
 6380   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6381   emit_int16(0x59, (0xC0 | encode));
 6382 }
 6383 
 6384 void Assembler::mulpd(XMMRegister dst, Address src) {
 6385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6386   InstructionMark im(this);
 6387   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6388   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6389   attributes.set_rex_vex_w_reverted();
 6390   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6391   emit_int8(0x59);
 6392   emit_operand(dst, src);
 6393 }
 6394 
 6395 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6397   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6398   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6399   emit_int16(0x59, (0xC0 | encode));
 6400 }
 6401 
 6402 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6403   assert(VM_Version::supports_avx(), "");
 6404   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6405   attributes.set_rex_vex_w_reverted();
 6406   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6407   emit_int16(0x59, (0xC0 | encode));
 6408 }
 6409 
 6410 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6411   assert(VM_Version::supports_avx(), "");
 6412   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6413   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6414   emit_int16(0x59, (0xC0 | encode));
 6415 }
 6416 
 6417 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6418   assert(VM_Version::supports_avx(), "");
 6419   InstructionMark im(this);
 6420   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6421   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6422   attributes.set_rex_vex_w_reverted();
 6423   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6424   emit_int8(0x59);
 6425   emit_operand(dst, src);
 6426 }
 6427 
 6428 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6429   assert(VM_Version::supports_avx(), "");
 6430   InstructionMark im(this);
 6431   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6432   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6433   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6434   emit_int8(0x59);
 6435   emit_operand(dst, src);
 6436 }
 6437 
 6438 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6439   assert(VM_Version::supports_fma(), "");
 6440   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6441   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6442   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6443 }
 6444 
 6445 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6446   assert(VM_Version::supports_fma(), "");
 6447   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6448   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6449   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6450 }
 6451 
 6452 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6453   assert(VM_Version::supports_fma(), "");
 6454   InstructionMark im(this);
 6455   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6456   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6457   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6458   emit_int8((unsigned char)0xB8);
 6459   emit_operand(dst, src2);
 6460 }
 6461 
 6462 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6463   assert(VM_Version::supports_fma(), "");
 6464   InstructionMark im(this);
 6465   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6466   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6467   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6468   emit_int8((unsigned char)0xB8);
 6469   emit_operand(dst, src2);
 6470 }
 6471 
 6472 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6473   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6474   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6475   attributes.set_rex_vex_w_reverted();
 6476   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6477   emit_int16(0x5E, (0xC0 | encode));
 6478 }
 6479 
 6480 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6481   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6482   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6483   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6484   emit_int16(0x5E, (0xC0 | encode));
 6485 }
 6486 
 6487 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6488   assert(VM_Version::supports_avx(), "");
 6489   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6490   attributes.set_rex_vex_w_reverted();
 6491   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6492   emit_int16(0x5E, (0xC0 | encode));
 6493 }
 6494 
 6495 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6496   assert(VM_Version::supports_avx(), "");
 6497   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6498   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6499   emit_int16(0x5E, (0xC0 | encode));
 6500 }
 6501 
 6502 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6503   assert(VM_Version::supports_avx(), "");
 6504   InstructionMark im(this);
 6505   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6506   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6507   attributes.set_rex_vex_w_reverted();
 6508   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6509   emit_int8(0x5E);
 6510   emit_operand(dst, src);
 6511 }
 6512 
 6513 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6514   assert(VM_Version::supports_avx(), "");
 6515   InstructionMark im(this);
 6516   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6517   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6518   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6519   emit_int8(0x5E);
 6520   emit_operand(dst, src);
 6521 }
 6522 
 6523 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6524   assert(VM_Version::supports_avx(), "");
 6525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6526   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6527   emit_int24(0x09, (0xC0 | encode), (rmode));
 6528 }
 6529 
 6530 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6531   assert(VM_Version::supports_avx(), "");
 6532   InstructionMark im(this);
 6533   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6534   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6535   emit_int8(0x09);
 6536   emit_operand(dst, src);
 6537   emit_int8((rmode));
 6538 }
 6539 
 6540 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6541   assert(VM_Version::supports_evex(), "requires EVEX support");
 6542   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6543   attributes.set_is_evex_instruction();
 6544   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6545   emit_int24(0x09, (0xC0 | encode), (rmode));
 6546 }
 6547 
 6548 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6549   assert(VM_Version::supports_evex(), "requires EVEX support");
 6550   assert(dst != xnoreg, "sanity");
 6551   InstructionMark im(this);
 6552   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6553   attributes.set_is_evex_instruction();
 6554   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6555   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6556   emit_int8(0x09);
 6557   emit_operand(dst, src);
 6558   emit_int8((rmode));
 6559 }
 6560 
 6561 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6562   assert(VM_Version::supports_avx(), "");
 6563   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6564   attributes.set_rex_vex_w_reverted();
 6565   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6566   emit_int16(0x51, (0xC0 | encode));
 6567 }
 6568 
 6569 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6570   assert(VM_Version::supports_avx(), "");
 6571   InstructionMark im(this);
 6572   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6573   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6574   attributes.set_rex_vex_w_reverted();
 6575   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6576   emit_int8(0x51);
 6577   emit_operand(dst, src);
 6578 }
 6579 
 6580 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6581   assert(VM_Version::supports_avx(), "");
 6582   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6583   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6584   emit_int16(0x51, (0xC0 | encode));
 6585 }
 6586 
 6587 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6588   assert(VM_Version::supports_avx(), "");
 6589   InstructionMark im(this);
 6590   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6591   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6592   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6593   emit_int8(0x51);
 6594   emit_operand(dst, src);
 6595 }
 6596 
 6597 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6598   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6599   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6600   attributes.set_rex_vex_w_reverted();
 6601   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6602   emit_int16(0x54, (0xC0 | encode));
 6603 }
 6604 
 6605 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6606   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6607   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6608   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6609   emit_int16(0x54, (0xC0 | encode));
 6610 }
 6611 
 6612 void Assembler::andps(XMMRegister dst, Address src) {
 6613   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6614   InstructionMark im(this);
 6615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6616   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6617   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6618   emit_int8(0x54);
 6619   emit_operand(dst, src);
 6620 }
 6621 
 6622 void Assembler::andpd(XMMRegister dst, Address src) {
 6623   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6624   InstructionMark im(this);
 6625   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6626   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6627   attributes.set_rex_vex_w_reverted();
 6628   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6629   emit_int8(0x54);
 6630   emit_operand(dst, src);
 6631 }
 6632 
 6633 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6634   assert(VM_Version::supports_avx(), "");
 6635   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6636   attributes.set_rex_vex_w_reverted();
 6637   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6638   emit_int16(0x54, (0xC0 | encode));
 6639 }
 6640 
 6641 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6642   assert(VM_Version::supports_avx(), "");
 6643   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6644   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6645   emit_int16(0x54, (0xC0 | encode));
 6646 }
 6647 
 6648 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6649   assert(VM_Version::supports_avx(), "");
 6650   InstructionMark im(this);
 6651   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6652   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6653   attributes.set_rex_vex_w_reverted();
 6654   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6655   emit_int8(0x54);
 6656   emit_operand(dst, src);
 6657 }
 6658 
 6659 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6660   assert(VM_Version::supports_avx(), "");
 6661   InstructionMark im(this);
 6662   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6663   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6664   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6665   emit_int8(0x54);
 6666   emit_operand(dst, src);
 6667 }
 6668 
 6669 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6671   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6672   attributes.set_rex_vex_w_reverted();
 6673   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6674   emit_int8(0x15);
 6675   emit_int8((0xC0 | encode));
 6676 }
 6677 
 6678 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6679   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6680   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6681   attributes.set_rex_vex_w_reverted();
 6682   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6683   emit_int16(0x14, (0xC0 | encode));
 6684 }
 6685 
 6686 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6687   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6688   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6689   attributes.set_rex_vex_w_reverted();
 6690   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6691   emit_int16(0x57, (0xC0 | encode));
 6692 }
 6693 
 6694 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6695   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6696   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6697   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6698   emit_int16(0x57, (0xC0 | encode));
 6699 }
 6700 
 6701 void Assembler::xorpd(XMMRegister dst, Address src) {
 6702   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6703   InstructionMark im(this);
 6704   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6705   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6706   attributes.set_rex_vex_w_reverted();
 6707   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6708   emit_int8(0x57);
 6709   emit_operand(dst, src);
 6710 }
 6711 
 6712 void Assembler::xorps(XMMRegister dst, Address src) {
 6713   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6714   InstructionMark im(this);
 6715   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6716   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6717   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6718   emit_int8(0x57);
 6719   emit_operand(dst, src);
 6720 }
 6721 
 6722 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6723   assert(VM_Version::supports_avx(), "");
 6724   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6725   attributes.set_rex_vex_w_reverted();
 6726   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6727   emit_int16(0x57, (0xC0 | encode));
 6728 }
 6729 
 6730 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6731   assert(VM_Version::supports_avx(), "");
 6732   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6733   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6734   emit_int16(0x57, (0xC0 | encode));
 6735 }
 6736 
 6737 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6738   assert(VM_Version::supports_avx(), "");
 6739   InstructionMark im(this);
 6740   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6741   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6742   attributes.set_rex_vex_w_reverted();
 6743   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6744   emit_int8(0x57);
 6745   emit_operand(dst, src);
 6746 }
 6747 
 6748 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6749   assert(VM_Version::supports_avx(), "");
 6750   InstructionMark im(this);
 6751   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6752   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6753   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6754   emit_int8(0x57);
 6755   emit_operand(dst, src);
 6756 }
 6757 
 6758 // Integer vector arithmetic
 6759 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6760   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6761          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6762   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6763   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6764   emit_int16(0x01, (0xC0 | encode));
 6765 }
 6766 
 6767 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6768   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6769          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6770   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6771   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6772   emit_int16(0x02, (0xC0 | encode));
 6773 }
 6774 
 6775 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6776   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6778   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6779   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6780 }
 6781 
 6782 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6784   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6785   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6786   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6787 }
 6788 
 6789 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6790   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6791   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6792   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6793   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6794 }
 6795 
 6796 void Assembler::paddd(XMMRegister dst, Address src) {
 6797   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6798   InstructionMark im(this);
 6799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6800   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6801   emit_int8((unsigned char)0xFE);
 6802   emit_operand(dst, src);
 6803 }
 6804 
 6805 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6806   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6807   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6808   attributes.set_rex_vex_w_reverted();
 6809   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6810   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6811 }
 6812 
 6813 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6814   assert(VM_Version::supports_sse3(), "");
 6815   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6816   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6817   emit_int16(0x01, (0xC0 | encode));
 6818 }
 6819 
 6820 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6821   assert(VM_Version::supports_sse3(), "");
 6822   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6823   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6824   emit_int16(0x02, (0xC0 | encode));
 6825 }
 6826 
 6827 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6828   assert(UseAVX > 0, "requires some form of AVX");
 6829   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6830   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6831   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6832 }
 6833 
 6834 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6835   assert(UseAVX > 0, "requires some form of AVX");
 6836   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6837   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6838   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6839 }
 6840 
 6841 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6842   assert(UseAVX > 0, "requires some form of AVX");
 6843   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6844   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6845   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6846 }
 6847 
 6848 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6849   assert(UseAVX > 0, "requires some form of AVX");
 6850   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6851   attributes.set_rex_vex_w_reverted();
 6852   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6853   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6854 }
 6855 
 6856 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6857   assert(UseAVX > 0, "requires some form of AVX");
 6858   InstructionMark im(this);
 6859   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6860   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6861   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6862   emit_int8((unsigned char)0xFC);
 6863   emit_operand(dst, src);
 6864 }
 6865 
 6866 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6867   assert(UseAVX > 0, "requires some form of AVX");
 6868   InstructionMark im(this);
 6869   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6870   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6871   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6872   emit_int8((unsigned char)0xFD);
 6873   emit_operand(dst, src);
 6874 }
 6875 
 6876 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6877   assert(UseAVX > 0, "requires some form of AVX");
 6878   InstructionMark im(this);
 6879   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6880   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6881   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6882   emit_int8((unsigned char)0xFE);
 6883   emit_operand(dst, src);
 6884 }
 6885 
 6886 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6887   assert(UseAVX > 0, "requires some form of AVX");
 6888   InstructionMark im(this);
 6889   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6890   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6891   attributes.set_rex_vex_w_reverted();
 6892   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6893   emit_int8((unsigned char)0xD4);
 6894   emit_operand(dst, src);
 6895 }
 6896 
 6897 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6898   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6899   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6900   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6901   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6902 }
 6903 
 6904 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6905   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6906   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6907   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6908   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6909 }
 6910 
 6911 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6912   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6913   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6914   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6915 }
 6916 
 6917 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6918   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6919   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6920   attributes.set_rex_vex_w_reverted();
 6921   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6922   emit_int8((unsigned char)0xFB);
 6923   emit_int8((0xC0 | encode));
 6924 }
 6925 
 6926 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6927   assert(UseAVX > 0, "requires some form of AVX");
 6928   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6929   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6930   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6931 }
 6932 
 6933 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6934   assert(UseAVX > 0, "requires some form of AVX");
 6935   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6936   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6937   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6938 }
 6939 
 6940 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6941   assert(UseAVX > 0, "requires some form of AVX");
 6942   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6943   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6944   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6945 }
 6946 
 6947 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6948   assert(UseAVX > 0, "requires some form of AVX");
 6949   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6950   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6951   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6952 }
 6953 
 6954 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6955   assert(UseAVX > 0, "requires some form of AVX");
 6956   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6957   attributes.set_rex_vex_w_reverted();
 6958   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6959   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 6960 }
 6961 
 6962 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6963   assert(UseAVX > 0, "requires some form of AVX");
 6964   InstructionMark im(this);
 6965   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6966   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6967   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6968   emit_int8((unsigned char)0xF8);
 6969   emit_operand(dst, src);
 6970 }
 6971 
 6972 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6973   assert(UseAVX > 0, "requires some form of AVX");
 6974   InstructionMark im(this);
 6975   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6976   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6977   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6978   emit_int8((unsigned char)0xF9);
 6979   emit_operand(dst, src);
 6980 }
 6981 
 6982 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6983   assert(UseAVX > 0, "requires some form of AVX");
 6984   InstructionMark im(this);
 6985   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6986   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6987   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6988   emit_int8((unsigned char)0xFA);
 6989   emit_operand(dst, src);
 6990 }
 6991 
 6992 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6993   assert(UseAVX > 0, "requires some form of AVX");
 6994   InstructionMark im(this);
 6995   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6996   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6997   attributes.set_rex_vex_w_reverted();
 6998   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6999   emit_int8((unsigned char)0xFB);
 7000   emit_operand(dst, src);
 7001 }
 7002 
 7003 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7004   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7005   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7006   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7007   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7008 }
 7009 
 7010 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7011   assert(VM_Version::supports_sse4_1(), "");
 7012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7013   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7014   emit_int16(0x40, (0xC0 | encode));
 7015 }
 7016 
 7017 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7018   assert(VM_Version::supports_sse2(), "");
 7019   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7020   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7021   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7022 }
 7023 
 7024 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7025   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7026          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7027          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7028   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7029   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7030   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7031 }
 7032 
 7033 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7034   assert(UseAVX > 0, "requires some form of AVX");
 7035   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7036   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7037   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7038 }
 7039 
 7040 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7041   assert(UseAVX > 0, "requires some form of AVX");
 7042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7043   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7044   emit_int16(0x40, (0xC0 | encode));
 7045 }
 7046 
 7047 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7048   assert(UseAVX > 2, "requires some form of EVEX");
 7049   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7050   attributes.set_is_evex_instruction();
 7051   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7052   emit_int16(0x40, (0xC0 | encode));
 7053 }
 7054 
 7055 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7056   assert(UseAVX > 0, "requires some form of AVX");
 7057   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7058   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7059   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7060 }
 7061 
 7062 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7063   assert(UseAVX > 0, "requires some form of AVX");
 7064   InstructionMark im(this);
 7065   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7066   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7067   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7068   emit_int8((unsigned char)0xD5);
 7069   emit_operand(dst, src);
 7070 }
 7071 
 7072 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7073   assert(UseAVX > 0, "requires some form of AVX");
 7074   InstructionMark im(this);
 7075   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7076   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7077   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7078   emit_int8(0x40);
 7079   emit_operand(dst, src);
 7080 }
 7081 
 7082 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7083   assert(UseAVX > 2, "requires some form of EVEX");
 7084   InstructionMark im(this);
 7085   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7086   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7087   attributes.set_is_evex_instruction();
 7088   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7089   emit_int8(0x40);
 7090   emit_operand(dst, src);
 7091 }
 7092 
 7093 // Min, max
 7094 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7095   assert(VM_Version::supports_sse4_1(), "");
 7096   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7097   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7098   emit_int16(0x38, (0xC0 | encode));
 7099 }
 7100 
 7101 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7102   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7103         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7106   emit_int16(0x38, (0xC0 | encode));
 7107 }
 7108 
 7109 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7110   assert(VM_Version::supports_sse2(), "");
 7111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7112   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7113   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7114 }
 7115 
 7116 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7117   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7118         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7120   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7121   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7122 }
 7123 
 7124 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7125   assert(VM_Version::supports_sse4_1(), "");
 7126   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7127   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7128   emit_int16(0x39, (0xC0 | encode));
 7129 }
 7130 
 7131 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7132   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7133         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7134   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7135   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7136   emit_int16(0x39, (0xC0 | encode));
 7137 }
 7138 
 7139 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7140   assert(UseAVX > 2, "requires AVX512F");
 7141   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7142   attributes.set_is_evex_instruction();
 7143   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7144   emit_int16(0x39, (0xC0 | encode));
 7145 }
 7146 
 7147 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7148   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7149   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7150   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7151   emit_int16(0x5D, (0xC0 | encode));
 7152 }
 7153 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7154   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7155   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7156   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7157   emit_int16(0x5D, (0xC0 | encode));
 7158 }
 7159 
 7160 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7161   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7162   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7163   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7164   emit_int16(0x5D, (0xC0 | encode));
 7165 }
 7166 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7167   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7168   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7169   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7170   emit_int16(0x5D, (0xC0 | encode));
 7171 }
 7172 
 7173 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7174   assert(VM_Version::supports_sse4_1(), "");
 7175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7176   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7177   emit_int16(0x3C, (0xC0 | encode));
 7178 }
 7179 
 7180 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7181   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7182         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7184   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7185   emit_int16(0x3C, (0xC0 | encode));
 7186 }
 7187 
 7188 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7189   assert(VM_Version::supports_sse2(), "");
 7190   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7191   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7192   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7193 }
 7194 
 7195 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7196   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7197         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7198   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7199   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7200   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7201 }
 7202 
 7203 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7204   assert(VM_Version::supports_sse4_1(), "");
 7205   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7206   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7207   emit_int16(0x3D, (0xC0 | encode));
 7208 }
 7209 
 7210 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7211   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7212         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7213   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7214   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7215   emit_int16(0x3D, (0xC0 | encode));
 7216 }
 7217 
 7218 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7219   assert(UseAVX > 2, "requires AVX512F");
 7220   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7221   attributes.set_is_evex_instruction();
 7222   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7223   emit_int16(0x3D, (0xC0 | encode));
 7224 }
 7225 
 7226 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7227   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7228   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7229   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7230   emit_int16(0x5F, (0xC0 | encode));
 7231 }
 7232 
 7233 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7234   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7235   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7236   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7237   emit_int16(0x5F, (0xC0 | encode));
 7238 }
 7239 
 7240 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7241   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7242   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7243   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7244   emit_int16(0x5F, (0xC0 | encode));
 7245 }
 7246 
 7247 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7248   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7249   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7250   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7251   emit_int16(0x5F, (0xC0 | encode));
 7252 }
 7253 
 7254 // Shift packed integers left by specified number of bits.
 7255 void Assembler::psllw(XMMRegister dst, int shift) {
 7256   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7258   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7259   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7260   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7261 }
 7262 
 7263 void Assembler::pslld(XMMRegister dst, int shift) {
 7264   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7265   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7266   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7267   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7268   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7269 }
 7270 
 7271 void Assembler::psllq(XMMRegister dst, int shift) {
 7272   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7273   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7274   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7275   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7276   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7277 }
 7278 
 7279 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7280   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7282   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7283   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7284 }
 7285 
 7286 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7287   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7289   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7290   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7291 }
 7292 
 7293 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7294   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7295   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7296   attributes.set_rex_vex_w_reverted();
 7297   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7298   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7299 }
 7300 
 7301 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7302   assert(UseAVX > 0, "requires some form of AVX");
 7303   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7304   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7305   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7306   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7307 }
 7308 
 7309 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7310   assert(UseAVX > 0, "requires some form of AVX");
 7311   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7312   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7313   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7314   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7315   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7316 }
 7317 
 7318 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7319   assert(UseAVX > 0, "requires some form of AVX");
 7320   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7321   attributes.set_rex_vex_w_reverted();
 7322   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7323   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7324   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7325 }
 7326 
 7327 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7328   assert(UseAVX > 0, "requires some form of AVX");
 7329   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7330   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7331   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7332 }
 7333 
 7334 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7335   assert(UseAVX > 0, "requires some form of AVX");
 7336   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7337   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7338   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7339 }
 7340 
 7341 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7342   assert(UseAVX > 0, "requires some form of AVX");
 7343   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7344   attributes.set_rex_vex_w_reverted();
 7345   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7346   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7347 }
 7348 
 7349 // Shift packed integers logically right by specified number of bits.
 7350 void Assembler::psrlw(XMMRegister dst, int shift) {
 7351   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7352   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7353   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7354   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7355   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7356 }
 7357 
 7358 void Assembler::psrld(XMMRegister dst, int shift) {
 7359   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7360   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7361   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7362   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7363   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7364 }
 7365 
 7366 void Assembler::psrlq(XMMRegister dst, int shift) {
 7367   // Do not confuse it with psrldq SSE2 instruction which
 7368   // shifts 128 bit value in xmm register by number of bytes.
 7369   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7370   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7371   attributes.set_rex_vex_w_reverted();
 7372   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7373   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7374   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7375 }
 7376 
 7377 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7379   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7380   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7381   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7382 }
 7383 
 7384 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7387   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7388   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7389 }
 7390 
 7391 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7392   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7393   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7394   attributes.set_rex_vex_w_reverted();
 7395   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7396   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7397 }
 7398 
 7399 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7400   assert(UseAVX > 0, "requires some form of AVX");
 7401   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7402   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7403   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7404   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7405 }
 7406 
 7407 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7408   assert(UseAVX > 0, "requires some form of AVX");
 7409   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7410   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7411   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7412   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7413 }
 7414 
 7415 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7416   assert(UseAVX > 0, "requires some form of AVX");
 7417   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7418   attributes.set_rex_vex_w_reverted();
 7419   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7420   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7421   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7422 }
 7423 
 7424 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7425   assert(UseAVX > 0, "requires some form of AVX");
 7426   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7427   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7428   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7429 }
 7430 
 7431 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7432   assert(UseAVX > 0, "requires some form of AVX");
 7433   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7434   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7435   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7436 }
 7437 
 7438 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7439   assert(UseAVX > 0, "requires some form of AVX");
 7440   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7441   attributes.set_rex_vex_w_reverted();
 7442   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7443   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7444 }
 7445 
 7446 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7447   assert(VM_Version::supports_avx512bw(), "");
 7448   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7449   attributes.set_is_evex_instruction();
 7450   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7451   emit_int16(0x10, (0xC0 | encode));
 7452 }
 7453 
 7454 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7455   assert(VM_Version::supports_avx512bw(), "");
 7456   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7457   attributes.set_is_evex_instruction();
 7458   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7459   emit_int16(0x12, (0xC0 | encode));
 7460 }
 7461 
 7462 // Shift packed integers arithmetically right by specified number of bits.
 7463 void Assembler::psraw(XMMRegister dst, int shift) {
 7464   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7465   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7466   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7467   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7468   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7469 }
 7470 
 7471 void Assembler::psrad(XMMRegister dst, int shift) {
 7472   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7473   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7474   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7475   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7476   emit_int8(0x72);
 7477   emit_int8((0xC0 | encode));
 7478   emit_int8(shift & 0xFF);
 7479 }
 7480 
 7481 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7482   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7483   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7484   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7485   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7486 }
 7487 
 7488 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7489   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7490   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7491   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7492   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7493 }
 7494 
 7495 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7496   assert(UseAVX > 0, "requires some form of AVX");
 7497   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7498   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7499   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7500   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7501 }
 7502 
 7503 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7504   assert(UseAVX > 0, "requires some form of AVX");
 7505   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7506   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7507   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7508   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7509 }
 7510 
 7511 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7512   assert(UseAVX > 0, "requires some form of AVX");
 7513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7514   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7515   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7516 }
 7517 
 7518 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7519   assert(UseAVX > 0, "requires some form of AVX");
 7520   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7521   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7522   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7523 }
 7524 
 7525 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7526   assert(UseAVX > 2, "requires AVX512");
 7527   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7528   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7529   attributes.set_is_evex_instruction();
 7530   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7531   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7532 }
 7533 
 7534 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7535   assert(UseAVX > 2, "requires AVX512");
 7536   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7537   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7538   attributes.set_is_evex_instruction();
 7539   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7540   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7541 }
 7542 
 7543 // logical operations packed integers
 7544 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7545   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7546   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7547   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7548   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7549 }
 7550 
 7551 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7552   assert(UseAVX > 0, "requires some form of AVX");
 7553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7554   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7555   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7556 }
 7557 
 7558 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7559   assert(UseAVX > 0, "requires some form of AVX");
 7560   InstructionMark im(this);
 7561   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7562   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7563   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7564   emit_int8((unsigned char)0xDB);
 7565   emit_operand(dst, src);
 7566 }
 7567 
 7568 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7569   assert(VM_Version::supports_evex(), "");
 7570   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7571   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7572   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7573 }
 7574 
 7575 //Variable Shift packed integers logically left.
 7576 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7577   assert(UseAVX > 1, "requires AVX2");
 7578   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7579   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7580   emit_int16(0x47, (0xC0 | encode));
 7581 }
 7582 
 7583 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7584   assert(UseAVX > 1, "requires AVX2");
 7585   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7586   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7587   emit_int16(0x47, (0xC0 | encode));
 7588 }
 7589 
 7590 //Variable Shift packed integers logically right.
 7591 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7592   assert(UseAVX > 1, "requires AVX2");
 7593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7594   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7595   emit_int16(0x45, (0xC0 | encode));
 7596 }
 7597 
 7598 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7599   assert(UseAVX > 1, "requires AVX2");
 7600   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7601   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7602   emit_int16(0x45, (0xC0 | encode));
 7603 }
 7604 
 7605 //Variable right Shift arithmetic packed integers .
 7606 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7607   assert(UseAVX > 1, "requires AVX2");
 7608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7609   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7610   emit_int16(0x46, (0xC0 | encode));
 7611 }
 7612 
 7613 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7614   assert(VM_Version::supports_avx512bw(), "");
 7615   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7616   attributes.set_is_evex_instruction();
 7617   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7618   emit_int16(0x11, (0xC0 | encode));
 7619 }
 7620 
 7621 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7622   assert(UseAVX > 2, "requires AVX512");
 7623   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7624   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7625   attributes.set_is_evex_instruction();
 7626   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7627   emit_int16(0x46, (0xC0 | encode));
 7628 }
 7629 
 7630 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7631   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7633   attributes.set_is_evex_instruction();
 7634   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7635   emit_int16(0x71, (0xC0 | encode));
 7636 }
 7637 
 7638 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7639   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7641   attributes.set_is_evex_instruction();
 7642   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7643   emit_int16(0x73, (0xC0 | encode));
 7644 }
 7645 
 7646 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7647   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7648   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7649   attributes.set_rex_vex_w_reverted();
 7650   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7651   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7652 }
 7653 
 7654 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7655   assert(UseAVX > 0, "requires some form of AVX");
 7656   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7657   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7658   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7659 }
 7660 
 7661 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7662   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7663   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7664   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7665   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7666 }
 7667 
 7668 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7669   assert(UseAVX > 0, "requires some form of AVX");
 7670   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7671   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7672   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7673 }
 7674 
 7675 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7676   assert(UseAVX > 0, "requires some form of AVX");
 7677   InstructionMark im(this);
 7678   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7679   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7680   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7681   emit_int8((unsigned char)0xEB);
 7682   emit_operand(dst, src);
 7683 }
 7684 
 7685 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7686   assert(VM_Version::supports_evex(), "");
 7687   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7688   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7689   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7690 }
 7691 
 7692 
 7693 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7694   assert(VM_Version::supports_evex(), "");
 7695   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7696   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7697   attributes.set_is_evex_instruction();
 7698   attributes.set_embedded_opmask_register_specifier(mask);
 7699   if (merge) {
 7700     attributes.reset_is_clear_context();
 7701   }
 7702   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7703   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7704 }
 7705 
 7706 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7707   assert(VM_Version::supports_evex(), "");
 7708   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7709   InstructionMark im(this);
 7710   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7711   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7712   attributes.set_is_evex_instruction();
 7713   attributes.set_embedded_opmask_register_specifier(mask);
 7714   if (merge) {
 7715     attributes.reset_is_clear_context();
 7716   }
 7717   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7718   emit_int8((unsigned char)0xEB);
 7719   emit_operand(dst, src);
 7720 }
 7721 
 7722 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7725   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7726   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7727 }
 7728 
 7729 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7730   assert(UseAVX > 0, "requires some form of AVX");
 7731   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7732          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7733          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7734   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7735   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7736   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7737 }
 7738 
 7739 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7740   assert(UseAVX > 0, "requires some form of AVX");
 7741   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7742          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7743          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7744   InstructionMark im(this);
 7745   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7746   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7747   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7748   emit_int8((unsigned char)0xEF);
 7749   emit_operand(dst, src);
 7750 }
 7751 
 7752 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7753   assert(UseAVX > 2, "requires some form of EVEX");
 7754   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7755   attributes.set_rex_vex_w_reverted();
 7756   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7757   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7758 }
 7759 
 7760 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7761   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7762   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7763   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7764   attributes.set_is_evex_instruction();
 7765   attributes.set_embedded_opmask_register_specifier(mask);
 7766   if (merge) {
 7767     attributes.reset_is_clear_context();
 7768   }
 7769   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7770   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7771 }
 7772 
 7773 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7774   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7775   InstructionMark im(this);
 7776   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7777   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7778   attributes.set_is_evex_instruction();
 7779   attributes.set_embedded_opmask_register_specifier(mask);
 7780   if (merge) {
 7781     attributes.reset_is_clear_context();
 7782   }
 7783   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7784   emit_int8((unsigned char)0xEF);
 7785   emit_operand(dst, src);
 7786 }
 7787 
 7788 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7789   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7790   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7791   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7792   attributes.set_is_evex_instruction();
 7793   attributes.set_embedded_opmask_register_specifier(mask);
 7794   if (merge) {
 7795     attributes.reset_is_clear_context();
 7796   }
 7797   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7798   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7799 }
 7800 
 7801 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7802   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7803   InstructionMark im(this);
 7804   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7805   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7806   attributes.set_is_evex_instruction();
 7807   attributes.set_embedded_opmask_register_specifier(mask);
 7808   if (merge) {
 7809     attributes.reset_is_clear_context();
 7810   }
 7811   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7812   emit_int8((unsigned char)0xEF);
 7813   emit_operand(dst, src);
 7814 }
 7815 
 7816 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7817   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7818   InstructionMark im(this);
 7819   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7820   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7821   attributes.set_is_evex_instruction();
 7822   attributes.set_embedded_opmask_register_specifier(mask);
 7823   if (merge) {
 7824     attributes.reset_is_clear_context();
 7825   }
 7826   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7827   emit_int8((unsigned char)0xDB);
 7828   emit_operand(dst, src);
 7829 }
 7830 
 7831 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7832   assert(VM_Version::supports_evex(), "");
 7833   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7834   attributes.set_is_evex_instruction();
 7835   attributes.set_embedded_opmask_register_specifier(mask);
 7836   if (merge) {
 7837     attributes.reset_is_clear_context();
 7838   }
 7839   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7840   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7841 }
 7842 
 7843 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7844   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7845   InstructionMark im(this);
 7846   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7847   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7848   attributes.set_is_evex_instruction();
 7849   attributes.set_embedded_opmask_register_specifier(mask);
 7850   if (merge) {
 7851     attributes.reset_is_clear_context();
 7852   }
 7853   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7854   emit_int8((unsigned char)0xDB);
 7855   emit_operand(dst, src);
 7856 }
 7857 
 7858 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7859   assert(VM_Version::supports_evex(), "");
 7860   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7861   attributes.set_is_evex_instruction();
 7862   attributes.set_embedded_opmask_register_specifier(mask);
 7863   if (merge) {
 7864     attributes.reset_is_clear_context();
 7865   }
 7866   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7867   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7868 }
 7869 
 7870 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7871   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7872   InstructionMark im(this);
 7873   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7874   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7875   attributes.set_is_evex_instruction();
 7876   attributes.set_embedded_opmask_register_specifier(mask);
 7877   if (merge) {
 7878     attributes.reset_is_clear_context();
 7879   }
 7880   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7881   emit_int8((unsigned char)0xEB);
 7882   emit_operand(dst, src);
 7883 }
 7884 
 7885 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7886   assert(VM_Version::supports_evex(), "requires EVEX support");
 7887   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7888   attributes.set_is_evex_instruction();
 7889   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7890   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7891 }
 7892 
 7893 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7894   assert(VM_Version::supports_evex(), "requires EVEX support");
 7895   assert(dst != xnoreg, "sanity");
 7896   InstructionMark im(this);
 7897   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7898   attributes.set_is_evex_instruction();
 7899   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7900   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7901   emit_int8((unsigned char)0xEF);
 7902   emit_operand(dst, src);
 7903 }
 7904 
 7905 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7906   assert(VM_Version::supports_evex(), "requires EVEX support");
 7907   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7908   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7909   attributes.set_is_evex_instruction();
 7910   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7911   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7912 }
 7913 
 7914 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7915   assert(VM_Version::supports_evex(), "requires EVEX support");
 7916   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7917   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7918   attributes.set_is_evex_instruction();
 7919   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7920   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7921 }
 7922 
 7923 // Register is a class, but it would be assigned numerical value.
 7924 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7925 PRAGMA_DIAG_PUSH
 7926 PRAGMA_NONNULL_IGNORED
 7927 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7928   assert(VM_Version::supports_evex(), "requires EVEX support");
 7929   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7930   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7931   attributes.set_is_evex_instruction();
 7932   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7933   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7934 }
 7935 
 7936 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7937   assert(VM_Version::supports_evex(), "requires EVEX support");
 7938   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7939   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7940   attributes.set_is_evex_instruction();
 7941   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7942   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7943 }
 7944 PRAGMA_DIAG_POP
 7945 
 7946 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7947   assert(VM_Version::supports_evex(), "requires EVEX support");
 7948   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7949   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7950   attributes.set_is_evex_instruction();
 7951   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7952   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7953 }
 7954 
 7955 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7956   assert(VM_Version::supports_evex(), "requires EVEX support");
 7957   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7958   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7959   attributes.set_is_evex_instruction();
 7960   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7961   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7962 }
 7963 
 7964 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7965   assert(VM_Version::supports_evex(), "requires EVEX support");
 7966   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7967   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7968   attributes.set_is_evex_instruction();
 7969   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7970   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7971 }
 7972 
 7973 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7974   assert(VM_Version::supports_evex(), "requires EVEX support");
 7975   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7976   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7977   attributes.set_is_evex_instruction();
 7978   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7979   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7980 }
 7981 
 7982 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 7983   assert(VM_Version::supports_avx512cd() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 7984   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7985   attributes.set_is_evex_instruction();
 7986   attributes.set_embedded_opmask_register_specifier(mask);
 7987   if (merge) {
 7988     attributes.reset_is_clear_context();
 7989   }
 7990   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7991   emit_int16(0x44, (0xC0 | encode));
 7992 }
 7993 
 7994 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 7995   assert(VM_Version::supports_avx512cd() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 7996   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7997   attributes.set_is_evex_instruction();
 7998   attributes.set_embedded_opmask_register_specifier(mask);
 7999   if (merge) {
 8000     attributes.reset_is_clear_context();
 8001   }
 8002   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8003   emit_int16(0x44, (0xC0 | encode));
 8004 }
 8005 
 8006 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8007   assert(VM_Version::supports_evex(), "requires EVEX support");
 8008   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8009   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8010   attributes.set_is_evex_instruction();
 8011   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8012   emit_int8(0x25);
 8013   emit_int8((unsigned char)(0xC0 | encode));
 8014   emit_int8(imm8);
 8015 }
 8016 
 8017 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8018   assert(VM_Version::supports_evex(), "requires EVEX support");
 8019   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8020   assert(dst != xnoreg, "sanity");
 8021   InstructionMark im(this);
 8022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8023   attributes.set_is_evex_instruction();
 8024   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8025   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8026   emit_int8(0x25);
 8027   emit_operand(dst, src3);
 8028   emit_int8(imm8);
 8029 }
 8030 
 8031 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8032   assert(VM_Version::supports_evex(), "requires EVEX support");
 8033   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8034   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8035   attributes.set_is_evex_instruction();
 8036   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8037   emit_int8(0x25);
 8038   emit_int8((unsigned char)(0xC0 | encode));
 8039   emit_int8(imm8);
 8040 }
 8041 
 8042 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8043   assert(VM_Version::supports_evex(), "");
 8044   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8045   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8046   attributes.set_is_evex_instruction();
 8047   attributes.set_embedded_opmask_register_specifier(mask);
 8048   if (merge) {
 8049     attributes.reset_is_clear_context();
 8050   }
 8051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8052   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8053 }
 8054 
 8055 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8056   assert(VM_Version::supports_evex(), "");
 8057   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8058   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8059   attributes.set_is_evex_instruction();
 8060   attributes.set_embedded_opmask_register_specifier(mask);
 8061   if (merge) {
 8062     attributes.reset_is_clear_context();
 8063   }
 8064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8065   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8066 }
 8067 
 8068 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8069   assert(VM_Version::supports_avx512_vbmi2(), "");
 8070   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8072   attributes.set_is_evex_instruction();
 8073   attributes.set_embedded_opmask_register_specifier(mask);
 8074   if (merge) {
 8075     attributes.reset_is_clear_context();
 8076   }
 8077   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8078   emit_int16(0x62, (0xC0 | encode));
 8079 }
 8080 
 8081 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8082   assert(VM_Version::supports_avx512_vbmi2(), "");
 8083   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8084   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8085   attributes.set_is_evex_instruction();
 8086   attributes.set_embedded_opmask_register_specifier(mask);
 8087   if (merge) {
 8088     attributes.reset_is_clear_context();
 8089   }
 8090   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8091   emit_int16(0x62, (0xC0 | encode));
 8092 }
 8093 
 8094 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8095   assert(VM_Version::supports_evex(), "");
 8096   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8097   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8098   attributes.set_is_evex_instruction();
 8099   attributes.set_embedded_opmask_register_specifier(mask);
 8100   if (merge) {
 8101     attributes.reset_is_clear_context();
 8102   }
 8103   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8104   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8105 }
 8106 
 8107 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8108   assert(VM_Version::supports_evex(), "");
 8109   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8110   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8111   attributes.set_is_evex_instruction();
 8112   attributes.set_embedded_opmask_register_specifier(mask);
 8113   if (merge) {
 8114     attributes.reset_is_clear_context();
 8115   }
 8116   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8117   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8118 }
 8119 
 8120 // vinserti forms
 8121 
 8122 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8123   assert(VM_Version::supports_avx2(), "");
 8124   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8125   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8126   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8127   // last byte:
 8128   // 0x00 - insert into lower 128 bits
 8129   // 0x01 - insert into upper 128 bits
 8130   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8131 }
 8132 
 8133 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8134   assert(VM_Version::supports_avx2(), "");
 8135   assert(dst != xnoreg, "sanity");
 8136   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8137   InstructionMark im(this);
 8138   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8139   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8140   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8141   emit_int8(0x38);
 8142   emit_operand(dst, src);
 8143   // 0x00 - insert into lower 128 bits
 8144   // 0x01 - insert into upper 128 bits
 8145   emit_int8(imm8 & 0x01);
 8146 }
 8147 
 8148 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8149   assert(VM_Version::supports_evex(), "");
 8150   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8151   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8152   attributes.set_is_evex_instruction();
 8153   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8154   // imm8:
 8155   // 0x00 - insert into q0 128 bits (0..127)
 8156   // 0x01 - insert into q1 128 bits (128..255)
 8157   // 0x02 - insert into q2 128 bits (256..383)
 8158   // 0x03 - insert into q3 128 bits (384..511)
 8159   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8160 }
 8161 
 8162 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8163   assert(VM_Version::supports_evex(), "");
 8164   assert(dst != xnoreg, "sanity");
 8165   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8166   InstructionMark im(this);
 8167   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8168   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8169   attributes.set_is_evex_instruction();
 8170   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8171   emit_int8(0x18);
 8172   emit_operand(dst, src);
 8173   // 0x00 - insert into q0 128 bits (0..127)
 8174   // 0x01 - insert into q1 128 bits (128..255)
 8175   // 0x02 - insert into q2 128 bits (256..383)
 8176   // 0x03 - insert into q3 128 bits (384..511)
 8177   emit_int8(imm8 & 0x03);
 8178 }
 8179 
 8180 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8181   assert(VM_Version::supports_evex(), "");
 8182   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8183   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8184   attributes.set_is_evex_instruction();
 8185   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8186   //imm8:
 8187   // 0x00 - insert into lower 256 bits
 8188   // 0x01 - insert into upper 256 bits
 8189   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8190 }
 8191 
 8192 
 8193 // vinsertf forms
 8194 
 8195 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8196   assert(VM_Version::supports_avx(), "");
 8197   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8198   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8199   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8200   // imm8:
 8201   // 0x00 - insert into lower 128 bits
 8202   // 0x01 - insert into upper 128 bits
 8203   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8204 }
 8205 
 8206 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8207   assert(VM_Version::supports_avx(), "");
 8208   assert(dst != xnoreg, "sanity");
 8209   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8210   InstructionMark im(this);
 8211   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8212   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8213   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8214   emit_int8(0x18);
 8215   emit_operand(dst, src);
 8216   // 0x00 - insert into lower 128 bits
 8217   // 0x01 - insert into upper 128 bits
 8218   emit_int8(imm8 & 0x01);
 8219 }
 8220 
 8221 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8222   assert(VM_Version::supports_evex(), "");
 8223   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8224   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8225   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8226   // imm8:
 8227   // 0x00 - insert into q0 128 bits (0..127)
 8228   // 0x01 - insert into q1 128 bits (128..255)
 8229   // 0x02 - insert into q0 128 bits (256..383)
 8230   // 0x03 - insert into q1 128 bits (384..512)
 8231   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8232 }
 8233 
 8234 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8235   assert(VM_Version::supports_evex(), "");
 8236   assert(dst != xnoreg, "sanity");
 8237   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8238   InstructionMark im(this);
 8239   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8240   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8241   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8242   emit_int8(0x18);
 8243   emit_operand(dst, src);
 8244   // 0x00 - insert into q0 128 bits (0..127)
 8245   // 0x01 - insert into q1 128 bits (128..255)
 8246   // 0x02 - insert into q0 128 bits (256..383)
 8247   // 0x03 - insert into q1 128 bits (384..512)
 8248   emit_int8(imm8 & 0x03);
 8249 }
 8250 
 8251 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8252   assert(VM_Version::supports_evex(), "");
 8253   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8254   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8255   attributes.set_is_evex_instruction();
 8256   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8257   // imm8:
 8258   // 0x00 - insert into lower 256 bits
 8259   // 0x01 - insert into upper 256 bits
 8260   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8261 }
 8262 
 8263 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8264   assert(VM_Version::supports_evex(), "");
 8265   assert(dst != xnoreg, "sanity");
 8266   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8267   InstructionMark im(this);
 8268   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8269   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8270   attributes.set_is_evex_instruction();
 8271   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8272   emit_int8(0x1A);
 8273   emit_operand(dst, src);
 8274   // 0x00 - insert into lower 256 bits
 8275   // 0x01 - insert into upper 256 bits
 8276   emit_int8(imm8 & 0x01);
 8277 }
 8278 
 8279 
 8280 // vextracti forms
 8281 
 8282 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8283   assert(VM_Version::supports_avx2(), "");
 8284   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8285   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8286   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8287   // imm8:
 8288   // 0x00 - extract from lower 128 bits
 8289   // 0x01 - extract from upper 128 bits
 8290   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8291 }
 8292 
 8293 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8294   assert(VM_Version::supports_avx2(), "");
 8295   assert(src != xnoreg, "sanity");
 8296   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8297   InstructionMark im(this);
 8298   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8299   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8300   attributes.reset_is_clear_context();
 8301   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8302   emit_int8(0x39);
 8303   emit_operand(src, dst);
 8304   // 0x00 - extract from lower 128 bits
 8305   // 0x01 - extract from upper 128 bits
 8306   emit_int8(imm8 & 0x01);
 8307 }
 8308 
 8309 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8310   assert(VM_Version::supports_evex(), "");
 8311   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8312   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8313   attributes.set_is_evex_instruction();
 8314   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8315   // imm8:
 8316   // 0x00 - extract from bits 127:0
 8317   // 0x01 - extract from bits 255:128
 8318   // 0x02 - extract from bits 383:256
 8319   // 0x03 - extract from bits 511:384
 8320   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8321 }
 8322 
 8323 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8324   assert(VM_Version::supports_evex(), "");
 8325   assert(src != xnoreg, "sanity");
 8326   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8327   InstructionMark im(this);
 8328   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8329   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8330   attributes.reset_is_clear_context();
 8331   attributes.set_is_evex_instruction();
 8332   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8333   emit_int8(0x39);
 8334   emit_operand(src, dst);
 8335   // 0x00 - extract from bits 127:0
 8336   // 0x01 - extract from bits 255:128
 8337   // 0x02 - extract from bits 383:256
 8338   // 0x03 - extract from bits 511:384
 8339   emit_int8(imm8 & 0x03);
 8340 }
 8341 
 8342 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8343   assert(VM_Version::supports_avx512dq(), "");
 8344   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8345   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8346   attributes.set_is_evex_instruction();
 8347   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8348   // imm8:
 8349   // 0x00 - extract from bits 127:0
 8350   // 0x01 - extract from bits 255:128
 8351   // 0x02 - extract from bits 383:256
 8352   // 0x03 - extract from bits 511:384
 8353   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8354 }
 8355 
 8356 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8357   assert(VM_Version::supports_evex(), "");
 8358   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8359   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8360   attributes.set_is_evex_instruction();
 8361   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8362   // imm8:
 8363   // 0x00 - extract from lower 256 bits
 8364   // 0x01 - extract from upper 256 bits
 8365   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8366 }
 8367 
 8368 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8369   assert(VM_Version::supports_evex(), "");
 8370   assert(src != xnoreg, "sanity");
 8371   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8372   InstructionMark im(this);
 8373   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8374   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8375   attributes.reset_is_clear_context();
 8376   attributes.set_is_evex_instruction();
 8377   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8378   emit_int8(0x38);
 8379   emit_operand(src, dst);
 8380   // 0x00 - extract from lower 256 bits
 8381   // 0x01 - extract from upper 256 bits
 8382   emit_int8(imm8 & 0x01);
 8383 }
 8384 // vextractf forms
 8385 
 8386 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8387   assert(VM_Version::supports_avx(), "");
 8388   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8389   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8390   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8391   // imm8:
 8392   // 0x00 - extract from lower 128 bits
 8393   // 0x01 - extract from upper 128 bits
 8394   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8395 }
 8396 
 8397 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8398   assert(VM_Version::supports_avx(), "");
 8399   assert(src != xnoreg, "sanity");
 8400   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8401   InstructionMark im(this);
 8402   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8403   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8404   attributes.reset_is_clear_context();
 8405   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8406   emit_int8(0x19);
 8407   emit_operand(src, dst);
 8408   // 0x00 - extract from lower 128 bits
 8409   // 0x01 - extract from upper 128 bits
 8410   emit_int8(imm8 & 0x01);
 8411 }
 8412 
 8413 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8414   assert(VM_Version::supports_evex(), "");
 8415   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8416   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8417   attributes.set_is_evex_instruction();
 8418   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8419   // imm8:
 8420   // 0x00 - extract from bits 127:0
 8421   // 0x01 - extract from bits 255:128
 8422   // 0x02 - extract from bits 383:256
 8423   // 0x03 - extract from bits 511:384
 8424   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8425 }
 8426 
 8427 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8428   assert(VM_Version::supports_evex(), "");
 8429   assert(src != xnoreg, "sanity");
 8430   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8431   InstructionMark im(this);
 8432   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8433   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8434   attributes.reset_is_clear_context();
 8435   attributes.set_is_evex_instruction();
 8436   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8437   emit_int8(0x19);
 8438   emit_operand(src, dst);
 8439   // 0x00 - extract from bits 127:0
 8440   // 0x01 - extract from bits 255:128
 8441   // 0x02 - extract from bits 383:256
 8442   // 0x03 - extract from bits 511:384
 8443   emit_int8(imm8 & 0x03);
 8444 }
 8445 
 8446 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8447   assert(VM_Version::supports_avx512dq(), "");
 8448   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8449   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8450   attributes.set_is_evex_instruction();
 8451   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8452   // imm8:
 8453   // 0x00 - extract from bits 127:0
 8454   // 0x01 - extract from bits 255:128
 8455   // 0x02 - extract from bits 383:256
 8456   // 0x03 - extract from bits 511:384
 8457   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8458 }
 8459 
 8460 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8461   assert(VM_Version::supports_evex(), "");
 8462   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8463   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8464   attributes.set_is_evex_instruction();
 8465   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8466   // imm8:
 8467   // 0x00 - extract from lower 256 bits
 8468   // 0x01 - extract from upper 256 bits
 8469   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8470 }
 8471 
 8472 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8473   assert(VM_Version::supports_evex(), "");
 8474   assert(src != xnoreg, "sanity");
 8475   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8476   InstructionMark im(this);
 8477   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8478   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8479   attributes.reset_is_clear_context();
 8480   attributes.set_is_evex_instruction();
 8481   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8482   emit_int8(0x1B);
 8483   emit_operand(src, dst);
 8484   // 0x00 - extract from lower 256 bits
 8485   // 0x01 - extract from upper 256 bits
 8486   emit_int8(imm8 & 0x01);
 8487 }
 8488 
 8489 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8490 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8491   assert(VM_Version::supports_avx2(), "");
 8492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8493   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8494   emit_int16(0x78, (0xC0 | encode));
 8495 }
 8496 
 8497 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8498   assert(VM_Version::supports_avx2(), "");
 8499   assert(dst != xnoreg, "sanity");
 8500   InstructionMark im(this);
 8501   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8502   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8503   // swap src<->dst for encoding
 8504   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8505   emit_int8(0x78);
 8506   emit_operand(dst, src);
 8507 }
 8508 
 8509 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8510 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8511   assert(VM_Version::supports_avx2(), "");
 8512   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8513   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8514   emit_int16(0x79, (0xC0 | encode));
 8515 }
 8516 
 8517 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8518   assert(VM_Version::supports_avx2(), "");
 8519   assert(dst != xnoreg, "sanity");
 8520   InstructionMark im(this);
 8521   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8522   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8523   // swap src<->dst for encoding
 8524   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8525   emit_int8(0x79);
 8526   emit_operand(dst, src);
 8527 }
 8528 
 8529 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8530   assert(UseAVX > 0, "requires some form of AVX");
 8531   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8532   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8533   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8534 }
 8535 
 8536 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8537   assert(UseAVX > 0, "requires some form of AVX");
 8538   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8539   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8540   emit_int16(0x69, (0xC0 | encode));
 8541 }
 8542 
 8543 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8544   assert(UseAVX > 0, "requires some form of AVX");
 8545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8547   emit_int16(0x61, (0xC0 | encode));
 8548 }
 8549 
 8550 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8551   assert(UseAVX > 0, "requires some form of AVX");
 8552   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8553   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8554   emit_int16(0x6A, (0xC0 | encode));
 8555 }
 8556 
 8557 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8558   assert(UseAVX > 0, "requires some form of AVX");
 8559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8560   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8561   emit_int16(0x62, (0xC0 | encode));
 8562 }
 8563 
 8564 // xmm/mem sourced byte/word/dword/qword replicate
 8565 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8566   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8567   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8568   attributes.set_is_evex_instruction();
 8569   attributes.set_embedded_opmask_register_specifier(mask);
 8570   if (merge) {
 8571     attributes.reset_is_clear_context();
 8572   }
 8573   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8574   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8575 }
 8576 
 8577 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8578   InstructionMark im(this);
 8579   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8580   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8581   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8582   attributes.set_is_evex_instruction();
 8583   attributes.set_embedded_opmask_register_specifier(mask);
 8584   if (merge) {
 8585     attributes.reset_is_clear_context();
 8586   }
 8587   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8588   emit_int8((unsigned char)0xFC);
 8589   emit_operand(dst, src);
 8590 }
 8591 
 8592 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8593   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8594   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8595   attributes.set_is_evex_instruction();
 8596   attributes.set_embedded_opmask_register_specifier(mask);
 8597   if (merge) {
 8598     attributes.reset_is_clear_context();
 8599   }
 8600   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8601   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8602 }
 8603 
 8604 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8605   InstructionMark im(this);
 8606   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8607   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8608   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8609   attributes.set_is_evex_instruction();
 8610   attributes.set_embedded_opmask_register_specifier(mask);
 8611   if (merge) {
 8612     attributes.reset_is_clear_context();
 8613   }
 8614   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8615   emit_int8((unsigned char)0xFD);
 8616   emit_operand(dst, src);
 8617 }
 8618 
 8619 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8620   assert(VM_Version::supports_evex(), "");
 8621   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8622   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8623   attributes.set_is_evex_instruction();
 8624   attributes.set_embedded_opmask_register_specifier(mask);
 8625   if (merge) {
 8626     attributes.reset_is_clear_context();
 8627   }
 8628   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8629   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8630 }
 8631 
 8632 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8633   InstructionMark im(this);
 8634   assert(VM_Version::supports_evex(), "");
 8635   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8636   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8637   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8638   attributes.set_is_evex_instruction();
 8639   attributes.set_embedded_opmask_register_specifier(mask);
 8640   if (merge) {
 8641     attributes.reset_is_clear_context();
 8642   }
 8643   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8644   emit_int8((unsigned char)0xFE);
 8645   emit_operand(dst, src);
 8646 }
 8647 
 8648 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8649   assert(VM_Version::supports_evex(), "");
 8650   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8651   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8652   attributes.set_is_evex_instruction();
 8653   attributes.set_embedded_opmask_register_specifier(mask);
 8654   if (merge) {
 8655     attributes.reset_is_clear_context();
 8656   }
 8657   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8658   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8659 }
 8660 
 8661 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8662   InstructionMark im(this);
 8663   assert(VM_Version::supports_evex(), "");
 8664   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8665   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8666   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8667   attributes.set_is_evex_instruction();
 8668   attributes.set_embedded_opmask_register_specifier(mask);
 8669   if (merge) {
 8670     attributes.reset_is_clear_context();
 8671   }
 8672   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8673   emit_int8((unsigned char)0xD4);
 8674   emit_operand(dst, src);
 8675 }
 8676 
 8677 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8678   assert(VM_Version::supports_evex(), "");
 8679   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8680   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8681   attributes.set_is_evex_instruction();
 8682   attributes.set_embedded_opmask_register_specifier(mask);
 8683   if (merge) {
 8684     attributes.reset_is_clear_context();
 8685   }
 8686   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8687   emit_int16(0x58, (0xC0 | encode));
 8688 }
 8689 
 8690 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8691   InstructionMark im(this);
 8692   assert(VM_Version::supports_evex(), "");
 8693   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8694   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8695   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8696   attributes.set_is_evex_instruction();
 8697   attributes.set_embedded_opmask_register_specifier(mask);
 8698   if (merge) {
 8699     attributes.reset_is_clear_context();
 8700   }
 8701   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8702   emit_int8(0x58);
 8703   emit_operand(dst, src);
 8704 }
 8705 
 8706 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8707   assert(VM_Version::supports_evex(), "");
 8708   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8709   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8710   attributes.set_is_evex_instruction();
 8711   attributes.set_embedded_opmask_register_specifier(mask);
 8712   if (merge) {
 8713     attributes.reset_is_clear_context();
 8714   }
 8715   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8716   emit_int16(0x58, (0xC0 | encode));
 8717 }
 8718 
 8719 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8720   InstructionMark im(this);
 8721   assert(VM_Version::supports_evex(), "");
 8722   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8723   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8724   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8725   attributes.set_is_evex_instruction();
 8726   attributes.set_embedded_opmask_register_specifier(mask);
 8727   if (merge) {
 8728     attributes.reset_is_clear_context();
 8729   }
 8730   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8731   emit_int8(0x58);
 8732   emit_operand(dst, src);
 8733 }
 8734 
 8735 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8736   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8737   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8738   attributes.set_is_evex_instruction();
 8739   attributes.set_embedded_opmask_register_specifier(mask);
 8740   if (merge) {
 8741     attributes.reset_is_clear_context();
 8742   }
 8743   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8744   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8745 }
 8746 
 8747 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8748   InstructionMark im(this);
 8749   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8750   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8751   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8752   attributes.set_is_evex_instruction();
 8753   attributes.set_embedded_opmask_register_specifier(mask);
 8754   if (merge) {
 8755     attributes.reset_is_clear_context();
 8756   }
 8757   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8758   emit_int8((unsigned char)0xF8);
 8759   emit_operand(dst, src);
 8760 }
 8761 
 8762 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8763   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8764   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8765   attributes.set_is_evex_instruction();
 8766   attributes.set_embedded_opmask_register_specifier(mask);
 8767   if (merge) {
 8768     attributes.reset_is_clear_context();
 8769   }
 8770   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8771   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8772 }
 8773 
 8774 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8775   InstructionMark im(this);
 8776   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8777   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8778   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8779   attributes.set_is_evex_instruction();
 8780   attributes.set_embedded_opmask_register_specifier(mask);
 8781   if (merge) {
 8782     attributes.reset_is_clear_context();
 8783   }
 8784   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8785   emit_int8((unsigned char)0xF9);
 8786   emit_operand(dst, src);
 8787 }
 8788 
 8789 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8790   assert(VM_Version::supports_evex(), "");
 8791   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8792   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8793   attributes.set_is_evex_instruction();
 8794   attributes.set_embedded_opmask_register_specifier(mask);
 8795   if (merge) {
 8796     attributes.reset_is_clear_context();
 8797   }
 8798   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8799   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8800 }
 8801 
 8802 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8803   InstructionMark im(this);
 8804   assert(VM_Version::supports_evex(), "");
 8805   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8806   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8807   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8808   attributes.set_is_evex_instruction();
 8809   attributes.set_embedded_opmask_register_specifier(mask);
 8810   if (merge) {
 8811     attributes.reset_is_clear_context();
 8812   }
 8813   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8814   emit_int8((unsigned char)0xFA);
 8815   emit_operand(dst, src);
 8816 }
 8817 
 8818 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8819   assert(VM_Version::supports_evex(), "");
 8820   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8821   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8822   attributes.set_is_evex_instruction();
 8823   attributes.set_embedded_opmask_register_specifier(mask);
 8824   if (merge) {
 8825     attributes.reset_is_clear_context();
 8826   }
 8827   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8828   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 8829 }
 8830 
 8831 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8832   InstructionMark im(this);
 8833   assert(VM_Version::supports_evex(), "");
 8834   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8835   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8836   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8837   attributes.set_is_evex_instruction();
 8838   attributes.set_embedded_opmask_register_specifier(mask);
 8839   if (merge) {
 8840     attributes.reset_is_clear_context();
 8841   }
 8842   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8843   emit_int8((unsigned char)0xFB);
 8844   emit_operand(dst, src);
 8845 }
 8846 
 8847 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8848   assert(VM_Version::supports_evex(), "");
 8849   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8850   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8851   attributes.set_is_evex_instruction();
 8852   attributes.set_embedded_opmask_register_specifier(mask);
 8853   if (merge) {
 8854     attributes.reset_is_clear_context();
 8855   }
 8856   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8857   emit_int16(0x5C, (0xC0 | encode));
 8858 }
 8859 
 8860 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8861   InstructionMark im(this);
 8862   assert(VM_Version::supports_evex(), "");
 8863   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8864   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8865   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8866   attributes.set_is_evex_instruction();
 8867   attributes.set_embedded_opmask_register_specifier(mask);
 8868   if (merge) {
 8869     attributes.reset_is_clear_context();
 8870   }
 8871   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8872   emit_int8(0x5C);
 8873   emit_operand(dst, src);
 8874 }
 8875 
 8876 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8877   assert(VM_Version::supports_evex(), "");
 8878   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8879   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8880   attributes.set_is_evex_instruction();
 8881   attributes.set_embedded_opmask_register_specifier(mask);
 8882   if (merge) {
 8883     attributes.reset_is_clear_context();
 8884   }
 8885   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8886   emit_int16(0x5C, (0xC0 | encode));
 8887 }
 8888 
 8889 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8890   InstructionMark im(this);
 8891   assert(VM_Version::supports_evex(), "");
 8892   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8893   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8894   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8895   attributes.set_is_evex_instruction();
 8896   attributes.set_embedded_opmask_register_specifier(mask);
 8897   if (merge) {
 8898     attributes.reset_is_clear_context();
 8899   }
 8900   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8901   emit_int8(0x5C);
 8902   emit_operand(dst, src);
 8903 }
 8904 
 8905 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8906   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8907   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8908   attributes.set_is_evex_instruction();
 8909   attributes.set_embedded_opmask_register_specifier(mask);
 8910   if (merge) {
 8911     attributes.reset_is_clear_context();
 8912   }
 8913   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8914   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 8915 }
 8916 
 8917 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8918   InstructionMark im(this);
 8919   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8920   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8921   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8922   attributes.set_is_evex_instruction();
 8923   attributes.set_embedded_opmask_register_specifier(mask);
 8924   if (merge) {
 8925     attributes.reset_is_clear_context();
 8926   }
 8927   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8928   emit_int8((unsigned char)0xD5);
 8929   emit_operand(dst, src);
 8930 }
 8931 
 8932 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8933   assert(VM_Version::supports_evex(), "");
 8934   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8935   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8936   attributes.set_is_evex_instruction();
 8937   attributes.set_embedded_opmask_register_specifier(mask);
 8938   if (merge) {
 8939     attributes.reset_is_clear_context();
 8940   }
 8941   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8942   emit_int16(0x40, (0xC0 | encode));
 8943 }
 8944 
 8945 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8946   InstructionMark im(this);
 8947   assert(VM_Version::supports_evex(), "");
 8948   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8949   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8950   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8951   attributes.set_is_evex_instruction();
 8952   attributes.set_embedded_opmask_register_specifier(mask);
 8953   if (merge) {
 8954     attributes.reset_is_clear_context();
 8955   }
 8956   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8957   emit_int8(0x40);
 8958   emit_operand(dst, src);
 8959 }
 8960 
 8961 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8962   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8963   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8964   attributes.set_is_evex_instruction();
 8965   attributes.set_embedded_opmask_register_specifier(mask);
 8966   if (merge) {
 8967     attributes.reset_is_clear_context();
 8968   }
 8969   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8970   emit_int16(0x40, (0xC0 | encode));
 8971 }
 8972 
 8973 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8974   InstructionMark im(this);
 8975   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8976   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8977   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8978   attributes.set_is_evex_instruction();
 8979   attributes.set_embedded_opmask_register_specifier(mask);
 8980   if (merge) {
 8981     attributes.reset_is_clear_context();
 8982   }
 8983   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8984   emit_int8(0x40);
 8985   emit_operand(dst, src);
 8986 }
 8987 
 8988 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8989   assert(VM_Version::supports_evex(), "");
 8990   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8991   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8992   attributes.set_is_evex_instruction();
 8993   attributes.set_embedded_opmask_register_specifier(mask);
 8994   if (merge) {
 8995     attributes.reset_is_clear_context();
 8996   }
 8997   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8998   emit_int16(0x59, (0xC0 | encode));
 8999 }
 9000 
 9001 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9002   InstructionMark im(this);
 9003   assert(VM_Version::supports_evex(), "");
 9004   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9005   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9006   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9007   attributes.set_is_evex_instruction();
 9008   attributes.set_embedded_opmask_register_specifier(mask);
 9009   if (merge) {
 9010     attributes.reset_is_clear_context();
 9011   }
 9012   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9013   emit_int8(0x59);
 9014   emit_operand(dst, src);
 9015 }
 9016 
 9017 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9018   assert(VM_Version::supports_evex(), "");
 9019   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9020   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9021   attributes.set_is_evex_instruction();
 9022   attributes.set_embedded_opmask_register_specifier(mask);
 9023   if (merge) {
 9024     attributes.reset_is_clear_context();
 9025   }
 9026   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9027   emit_int16(0x59, (0xC0 | encode));
 9028 }
 9029 
 9030 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9031   InstructionMark im(this);
 9032   assert(VM_Version::supports_evex(), "");
 9033   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9034   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9035   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9036   attributes.set_is_evex_instruction();
 9037   attributes.set_embedded_opmask_register_specifier(mask);
 9038   if (merge) {
 9039     attributes.reset_is_clear_context();
 9040   }
 9041   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9042   emit_int8(0x59);
 9043   emit_operand(dst, src);
 9044 }
 9045 
 9046 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9047   assert(VM_Version::supports_evex(), "");
 9048   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9049   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9050   attributes.set_is_evex_instruction();
 9051   attributes.set_embedded_opmask_register_specifier(mask);
 9052   if (merge) {
 9053     attributes.reset_is_clear_context();
 9054   }
 9055   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9056   emit_int16(0x51, (0xC0 | encode));
 9057 }
 9058 
 9059 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9060   InstructionMark im(this);
 9061   assert(VM_Version::supports_evex(), "");
 9062   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9063   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9064   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9065   attributes.set_is_evex_instruction();
 9066   attributes.set_embedded_opmask_register_specifier(mask);
 9067   if (merge) {
 9068     attributes.reset_is_clear_context();
 9069   }
 9070   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9071   emit_int8(0x51);
 9072   emit_operand(dst, src);
 9073 }
 9074 
 9075 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9076   assert(VM_Version::supports_evex(), "");
 9077   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9078   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9079   attributes.set_is_evex_instruction();
 9080   attributes.set_embedded_opmask_register_specifier(mask);
 9081   if (merge) {
 9082     attributes.reset_is_clear_context();
 9083   }
 9084   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9085   emit_int16(0x51, (0xC0 | encode));
 9086 }
 9087 
 9088 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9089   InstructionMark im(this);
 9090   assert(VM_Version::supports_evex(), "");
 9091   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9092   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9093   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9094   attributes.set_is_evex_instruction();
 9095   attributes.set_embedded_opmask_register_specifier(mask);
 9096   if (merge) {
 9097     attributes.reset_is_clear_context();
 9098   }
 9099   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9100   emit_int8(0x51);
 9101   emit_operand(dst, src);
 9102 }
 9103 
 9104 
 9105 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9106   assert(VM_Version::supports_evex(), "");
 9107   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9108   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9109   attributes.set_is_evex_instruction();
 9110   attributes.set_embedded_opmask_register_specifier(mask);
 9111   if (merge) {
 9112     attributes.reset_is_clear_context();
 9113   }
 9114   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9115   emit_int16(0x5E, (0xC0 | encode));
 9116 }
 9117 
 9118 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9119   InstructionMark im(this);
 9120   assert(VM_Version::supports_evex(), "");
 9121   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9122   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9123   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9124   attributes.set_is_evex_instruction();
 9125   attributes.set_embedded_opmask_register_specifier(mask);
 9126   if (merge) {
 9127     attributes.reset_is_clear_context();
 9128   }
 9129   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9130   emit_int8(0x5E);
 9131   emit_operand(dst, src);
 9132 }
 9133 
 9134 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9135   assert(VM_Version::supports_evex(), "");
 9136   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9137   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9138   attributes.set_is_evex_instruction();
 9139   attributes.set_embedded_opmask_register_specifier(mask);
 9140   if (merge) {
 9141     attributes.reset_is_clear_context();
 9142   }
 9143   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9144   emit_int16(0x5E, (0xC0 | encode));
 9145 }
 9146 
 9147 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9148   InstructionMark im(this);
 9149   assert(VM_Version::supports_evex(), "");
 9150   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9151   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9152   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9153   attributes.set_is_evex_instruction();
 9154   attributes.set_embedded_opmask_register_specifier(mask);
 9155   if (merge) {
 9156     attributes.reset_is_clear_context();
 9157   }
 9158   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9159   emit_int8(0x5E);
 9160   emit_operand(dst, src);
 9161 }
 9162 
 9163 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9164   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9165   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9166   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9167   attributes.set_is_evex_instruction();
 9168   attributes.set_embedded_opmask_register_specifier(mask);
 9169   if (merge) {
 9170     attributes.reset_is_clear_context();
 9171   }
 9172   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9173   emit_int16(0x1C, (0xC0 | encode));
 9174 }
 9175 
 9176 
 9177 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9178   InstructionMark im(this);
 9179   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9180   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9181   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9182   attributes.set_is_evex_instruction();
 9183   attributes.set_embedded_opmask_register_specifier(mask);
 9184   if (merge) {
 9185     attributes.reset_is_clear_context();
 9186   }
 9187   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9188   emit_int8(0x1C);
 9189   emit_operand(dst, src);
 9190 }
 9191 
 9192 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9193   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9194   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9195   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9196   attributes.set_is_evex_instruction();
 9197   attributes.set_embedded_opmask_register_specifier(mask);
 9198   if (merge) {
 9199     attributes.reset_is_clear_context();
 9200   }
 9201   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9202   emit_int16(0x1D, (0xC0 | encode));
 9203 }
 9204 
 9205 
 9206 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9207   InstructionMark im(this);
 9208   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9209   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9210   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9211   attributes.set_is_evex_instruction();
 9212   attributes.set_embedded_opmask_register_specifier(mask);
 9213   if (merge) {
 9214     attributes.reset_is_clear_context();
 9215   }
 9216   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9217   emit_int8(0x1D);
 9218   emit_operand(dst, src);
 9219 }
 9220 
 9221 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9222   assert(VM_Version::supports_evex(), "");
 9223   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9224   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9225   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9226   attributes.set_is_evex_instruction();
 9227   attributes.set_embedded_opmask_register_specifier(mask);
 9228   if (merge) {
 9229     attributes.reset_is_clear_context();
 9230   }
 9231   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9232   emit_int16(0x1E, (0xC0 | encode));
 9233 }
 9234 
 9235 
 9236 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9237   InstructionMark im(this);
 9238   assert(VM_Version::supports_evex(), "");
 9239   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9240   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9241   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9242   attributes.set_is_evex_instruction();
 9243   attributes.set_embedded_opmask_register_specifier(mask);
 9244   if (merge) {
 9245     attributes.reset_is_clear_context();
 9246   }
 9247   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9248   emit_int8(0x1E);
 9249   emit_operand(dst, src);
 9250 }
 9251 
 9252 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9253   assert(VM_Version::supports_evex(), "");
 9254   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9255   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9256   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9257   attributes.set_is_evex_instruction();
 9258   attributes.set_embedded_opmask_register_specifier(mask);
 9259   if (merge) {
 9260     attributes.reset_is_clear_context();
 9261   }
 9262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9263   emit_int16(0x1F, (0xC0 | encode));
 9264 }
 9265 
 9266 
 9267 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9268   InstructionMark im(this);
 9269   assert(VM_Version::supports_evex(), "");
 9270   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9271   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9272   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9273   attributes.set_is_evex_instruction();
 9274   attributes.set_embedded_opmask_register_specifier(mask);
 9275   if (merge) {
 9276     attributes.reset_is_clear_context();
 9277   }
 9278   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9279   emit_int8(0x1F);
 9280   emit_operand(dst, src);
 9281 }
 9282 
 9283 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9284   assert(VM_Version::supports_evex(), "");
 9285   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9286   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9287   attributes.set_is_evex_instruction();
 9288   attributes.set_embedded_opmask_register_specifier(mask);
 9289   if (merge) {
 9290     attributes.reset_is_clear_context();
 9291   }
 9292   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9293   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9294 }
 9295 
 9296 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9297   InstructionMark im(this);
 9298   assert(VM_Version::supports_evex(), "");
 9299   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9300   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9301   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9302   attributes.set_is_evex_instruction();
 9303   attributes.set_embedded_opmask_register_specifier(mask);
 9304   if (merge) {
 9305     attributes.reset_is_clear_context();
 9306   }
 9307   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9308   emit_int8((unsigned char)0xA8);
 9309   emit_operand(dst, src);
 9310 }
 9311 
 9312 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9313   assert(VM_Version::supports_evex(), "");
 9314   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9315   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9316   attributes.set_is_evex_instruction();
 9317   attributes.set_embedded_opmask_register_specifier(mask);
 9318   if (merge) {
 9319     attributes.reset_is_clear_context();
 9320   }
 9321   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9322   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9323 }
 9324 
 9325 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9326   InstructionMark im(this);
 9327   assert(VM_Version::supports_evex(), "");
 9328   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9329   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9330   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9331   attributes.set_is_evex_instruction();
 9332   attributes.set_embedded_opmask_register_specifier(mask);
 9333   if (merge) {
 9334     attributes.reset_is_clear_context();
 9335   }
 9336   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9337   emit_int8((unsigned char)0xA8);
 9338   emit_operand(dst, src);
 9339 }
 9340 
 9341 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9342   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9343   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9344   attributes.set_is_evex_instruction();
 9345   attributes.set_embedded_opmask_register_specifier(mask);
 9346   if (merge) {
 9347     attributes.reset_is_clear_context();
 9348   }
 9349   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9350   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9351 }
 9352 
 9353 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9354   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9355   InstructionMark im(this);
 9356   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9357   attributes.set_is_evex_instruction();
 9358   attributes.set_embedded_opmask_register_specifier(mask);
 9359   if (merge) {
 9360     attributes.reset_is_clear_context();
 9361   }
 9362   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9363   emit_int8((unsigned char)0x8D);
 9364   emit_operand(dst, src);
 9365 }
 9366 
 9367 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9368   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9369   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9370   attributes.set_is_evex_instruction();
 9371   attributes.set_embedded_opmask_register_specifier(mask);
 9372   if (merge) {
 9373     attributes.reset_is_clear_context();
 9374   }
 9375   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9376   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9377 }
 9378 
 9379 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9380   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9381   InstructionMark im(this);
 9382   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9383   attributes.set_is_evex_instruction();
 9384   attributes.set_embedded_opmask_register_specifier(mask);
 9385   if (merge) {
 9386     attributes.reset_is_clear_context();
 9387   }
 9388   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9389   emit_int8((unsigned char)0x8D);
 9390   emit_operand(dst, src);
 9391 }
 9392 
 9393 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9394   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9395   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9396   attributes.set_is_evex_instruction();
 9397   attributes.set_embedded_opmask_register_specifier(mask);
 9398   if (merge) {
 9399     attributes.reset_is_clear_context();
 9400   }
 9401   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9402   emit_int16(0x36, (0xC0 | encode));
 9403 }
 9404 
 9405 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9406   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9407   InstructionMark im(this);
 9408   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9409   attributes.set_is_evex_instruction();
 9410   attributes.set_embedded_opmask_register_specifier(mask);
 9411   if (merge) {
 9412     attributes.reset_is_clear_context();
 9413   }
 9414   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9415   emit_int8(0x36);
 9416   emit_operand(dst, src);
 9417 }
 9418 
 9419 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9420   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9421   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9422   attributes.set_is_evex_instruction();
 9423   attributes.set_embedded_opmask_register_specifier(mask);
 9424   if (merge) {
 9425     attributes.reset_is_clear_context();
 9426   }
 9427   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9428   emit_int16(0x36, (0xC0 | encode));
 9429 }
 9430 
 9431 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9432   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9433   InstructionMark im(this);
 9434   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9435   attributes.set_is_evex_instruction();
 9436   attributes.set_embedded_opmask_register_specifier(mask);
 9437   if (merge) {
 9438     attributes.reset_is_clear_context();
 9439   }
 9440   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9441   emit_int8(0x36);
 9442   emit_operand(dst, src);
 9443 }
 9444 
 9445 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9446   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9447   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9448   attributes.set_is_evex_instruction();
 9449   attributes.set_embedded_opmask_register_specifier(mask);
 9450   if (merge) {
 9451     attributes.reset_is_clear_context();
 9452   }
 9453   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9454   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9455 }
 9456 
 9457 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9458   assert(VM_Version::supports_evex(), "");
 9459   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9460   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9461   attributes.set_is_evex_instruction();
 9462   attributes.set_embedded_opmask_register_specifier(mask);
 9463   if (merge) {
 9464     attributes.reset_is_clear_context();
 9465   }
 9466   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9467   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9468 }
 9469 
 9470 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9471   assert(VM_Version::supports_evex(), "");
 9472   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9473   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9474   attributes.set_is_evex_instruction();
 9475   attributes.set_embedded_opmask_register_specifier(mask);
 9476   if (merge) {
 9477     attributes.reset_is_clear_context();
 9478   }
 9479   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9480   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9481 }
 9482 
 9483 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9484   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9486   attributes.set_is_evex_instruction();
 9487   attributes.set_embedded_opmask_register_specifier(mask);
 9488   if (merge) {
 9489     attributes.reset_is_clear_context();
 9490   }
 9491   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9492   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9493 }
 9494 
 9495 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9496   assert(VM_Version::supports_evex(), "");
 9497   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9499   attributes.set_is_evex_instruction();
 9500   attributes.set_embedded_opmask_register_specifier(mask);
 9501   if (merge) {
 9502     attributes.reset_is_clear_context();
 9503   }
 9504   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9505   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9506 }
 9507 
 9508 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9509   assert(VM_Version::supports_evex(), "");
 9510   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9511   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9512   attributes.set_is_evex_instruction();
 9513   attributes.set_embedded_opmask_register_specifier(mask);
 9514   if (merge) {
 9515     attributes.reset_is_clear_context();
 9516   }
 9517   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9518   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9519 }
 9520 
 9521 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9522   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9523   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9524   attributes.set_is_evex_instruction();
 9525   attributes.set_embedded_opmask_register_specifier(mask);
 9526   if (merge) {
 9527     attributes.reset_is_clear_context();
 9528   }
 9529   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9530   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9531 }
 9532 
 9533 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9534   assert(VM_Version::supports_evex(), "");
 9535   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9536   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9537   attributes.set_is_evex_instruction();
 9538   attributes.set_embedded_opmask_register_specifier(mask);
 9539   if (merge) {
 9540     attributes.reset_is_clear_context();
 9541   }
 9542   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9543   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9544 }
 9545 
 9546 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9547   assert(VM_Version::supports_evex(), "");
 9548   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9549   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9550   attributes.set_is_evex_instruction();
 9551   attributes.set_embedded_opmask_register_specifier(mask);
 9552   if (merge) {
 9553     attributes.reset_is_clear_context();
 9554   }
 9555   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9556   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9557 }
 9558 
 9559 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9560   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9561   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9562   attributes.set_is_evex_instruction();
 9563   attributes.set_embedded_opmask_register_specifier(mask);
 9564   if (merge) {
 9565     attributes.reset_is_clear_context();
 9566   }
 9567   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9568   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9569 }
 9570 
 9571 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9572   assert(VM_Version::supports_evex(), "");
 9573   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9574   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9575   attributes.set_is_evex_instruction();
 9576   attributes.set_embedded_opmask_register_specifier(mask);
 9577   if (merge) {
 9578     attributes.reset_is_clear_context();
 9579   }
 9580   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9581   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9582 }
 9583 
 9584 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9585   assert(VM_Version::supports_evex(), "");
 9586   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9587   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9588   attributes.set_is_evex_instruction();
 9589   attributes.set_embedded_opmask_register_specifier(mask);
 9590   if (merge) {
 9591     attributes.reset_is_clear_context();
 9592   }
 9593   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9594   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9595 }
 9596 
 9597 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9598   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9599   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9600   attributes.set_is_evex_instruction();
 9601   attributes.set_embedded_opmask_register_specifier(mask);
 9602   if (merge) {
 9603     attributes.reset_is_clear_context();
 9604   }
 9605   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9606   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9607 }
 9608 
 9609 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9610   assert(VM_Version::supports_evex(), "");
 9611   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9612   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9613   attributes.set_is_evex_instruction();
 9614   attributes.set_embedded_opmask_register_specifier(mask);
 9615   if (merge) {
 9616     attributes.reset_is_clear_context();
 9617   }
 9618   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9619   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9620 }
 9621 
 9622 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9623   assert(VM_Version::supports_evex(), "");
 9624   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9625   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9626   attributes.set_is_evex_instruction();
 9627   attributes.set_embedded_opmask_register_specifier(mask);
 9628   if (merge) {
 9629     attributes.reset_is_clear_context();
 9630   }
 9631   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9632   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9633 }
 9634 
 9635 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9636   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9637   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9638   attributes.set_is_evex_instruction();
 9639   attributes.set_embedded_opmask_register_specifier(mask);
 9640   if (merge) {
 9641     attributes.reset_is_clear_context();
 9642   }
 9643   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9644   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9645 }
 9646 
 9647 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9648   assert(VM_Version::supports_evex(), "");
 9649   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9651   attributes.set_is_evex_instruction();
 9652   attributes.set_embedded_opmask_register_specifier(mask);
 9653   if (merge) {
 9654     attributes.reset_is_clear_context();
 9655   }
 9656   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9657   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9658 }
 9659 
 9660 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9661   assert(VM_Version::supports_evex(), "");
 9662   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9663   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9664   attributes.set_is_evex_instruction();
 9665   attributes.set_embedded_opmask_register_specifier(mask);
 9666   if (merge) {
 9667     attributes.reset_is_clear_context();
 9668   }
 9669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9670   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9671 }
 9672 
 9673 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9674   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9675   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9676   attributes.set_is_evex_instruction();
 9677   attributes.set_embedded_opmask_register_specifier(mask);
 9678   if (merge) {
 9679     attributes.reset_is_clear_context();
 9680   }
 9681   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9682   emit_int16(0x12, (0xC0 | encode));
 9683 }
 9684 
 9685 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9686   assert(VM_Version::supports_evex(), "");
 9687   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9689   attributes.set_is_evex_instruction();
 9690   attributes.set_embedded_opmask_register_specifier(mask);
 9691   if (merge) {
 9692     attributes.reset_is_clear_context();
 9693   }
 9694   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9695   emit_int16(0x47, (0xC0 | encode));
 9696 }
 9697 
 9698 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9699   assert(VM_Version::supports_evex(), "");
 9700   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9701   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9702   attributes.set_is_evex_instruction();
 9703   attributes.set_embedded_opmask_register_specifier(mask);
 9704   if (merge) {
 9705     attributes.reset_is_clear_context();
 9706   }
 9707   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9708   emit_int16(0x47, (0xC0 | encode));
 9709 }
 9710 
 9711 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9712   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9713   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9714   attributes.set_is_evex_instruction();
 9715   attributes.set_embedded_opmask_register_specifier(mask);
 9716   if (merge) {
 9717     attributes.reset_is_clear_context();
 9718   }
 9719   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9720   emit_int16(0x10, (0xC0 | encode));
 9721 }
 9722 
 9723 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9724   assert(VM_Version::supports_evex(), "");
 9725   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9726   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9727   attributes.set_is_evex_instruction();
 9728   attributes.set_embedded_opmask_register_specifier(mask);
 9729   if (merge) {
 9730     attributes.reset_is_clear_context();
 9731   }
 9732   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9733   emit_int16(0x45, (0xC0 | encode));
 9734 }
 9735 
 9736 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9737   assert(VM_Version::supports_evex(), "");
 9738   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9739   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9740   attributes.set_is_evex_instruction();
 9741   attributes.set_embedded_opmask_register_specifier(mask);
 9742   if (merge) {
 9743     attributes.reset_is_clear_context();
 9744   }
 9745   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9746   emit_int16(0x45, (0xC0 | encode));
 9747 }
 9748 
 9749 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9750   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9751   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9752   attributes.set_is_evex_instruction();
 9753   attributes.set_embedded_opmask_register_specifier(mask);
 9754   if (merge) {
 9755     attributes.reset_is_clear_context();
 9756   }
 9757   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9758   emit_int16(0x11, (0xC0 | encode));
 9759 }
 9760 
 9761 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9762   assert(VM_Version::supports_evex(), "");
 9763   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9764   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9765   attributes.set_is_evex_instruction();
 9766   attributes.set_embedded_opmask_register_specifier(mask);
 9767   if (merge) {
 9768     attributes.reset_is_clear_context();
 9769   }
 9770   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9771   emit_int16(0x46, (0xC0 | encode));
 9772 }
 9773 
 9774 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9775   assert(VM_Version::supports_evex(), "");
 9776   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9777   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9778   attributes.set_is_evex_instruction();
 9779   attributes.set_embedded_opmask_register_specifier(mask);
 9780   if (merge) {
 9781     attributes.reset_is_clear_context();
 9782   }
 9783   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9784   emit_int16(0x46, (0xC0 | encode));
 9785 }
 9786 
 9787 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9788   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9789   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9790   attributes.set_is_evex_instruction();
 9791   attributes.set_embedded_opmask_register_specifier(mask);
 9792   if (merge) {
 9793     attributes.reset_is_clear_context();
 9794   }
 9795   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9796   emit_int16(0x38, (0xC0 | encode));
 9797 }
 9798 
 9799 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9800   assert(VM_Version::supports_avx512bw(), "");
 9801   InstructionMark im(this);
 9802   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9803   attributes.set_is_evex_instruction();
 9804   attributes.set_embedded_opmask_register_specifier(mask);
 9805   if (merge) {
 9806     attributes.reset_is_clear_context();
 9807   }
 9808   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9809   emit_int8(0x38);
 9810   emit_operand(dst, src);
 9811 }
 9812 
 9813 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9814   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9815   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9816   attributes.set_is_evex_instruction();
 9817   attributes.set_embedded_opmask_register_specifier(mask);
 9818   if (merge) {
 9819     attributes.reset_is_clear_context();
 9820   }
 9821   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9822   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9823 }
 9824 
 9825 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9826   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9827   InstructionMark im(this);
 9828   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9829   attributes.set_is_evex_instruction();
 9830   attributes.set_embedded_opmask_register_specifier(mask);
 9831   if (merge) {
 9832     attributes.reset_is_clear_context();
 9833   }
 9834   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9835   emit_int8((unsigned char)0xEA);
 9836   emit_operand(dst, src);
 9837 }
 9838 
 9839 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9840   assert(VM_Version::supports_evex(), "");
 9841   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9842   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9843   attributes.set_is_evex_instruction();
 9844   attributes.set_embedded_opmask_register_specifier(mask);
 9845   if (merge) {
 9846     attributes.reset_is_clear_context();
 9847   }
 9848   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9849   emit_int16(0x39, (0xC0 | encode));
 9850 }
 9851 
 9852 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9853   assert(VM_Version::supports_evex(), "");
 9854   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9855   InstructionMark im(this);
 9856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9857   attributes.set_is_evex_instruction();
 9858   attributes.set_embedded_opmask_register_specifier(mask);
 9859   if (merge) {
 9860     attributes.reset_is_clear_context();
 9861   }
 9862   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9863   emit_int8(0x39);
 9864   emit_operand(dst, src);
 9865 }
 9866 
 9867 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9868   assert(VM_Version::supports_evex(), "");
 9869   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9870   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9871   attributes.set_is_evex_instruction();
 9872   attributes.set_embedded_opmask_register_specifier(mask);
 9873   if (merge) {
 9874     attributes.reset_is_clear_context();
 9875   }
 9876   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9877   emit_int16(0x39, (0xC0 | encode));
 9878 }
 9879 
 9880 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9881   assert(VM_Version::supports_evex(), "");
 9882   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9883   InstructionMark im(this);
 9884   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9885   attributes.set_is_evex_instruction();
 9886   attributes.set_embedded_opmask_register_specifier(mask);
 9887   if (merge) {
 9888     attributes.reset_is_clear_context();
 9889   }
 9890   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9891   emit_int8(0x39);
 9892   emit_operand(dst, src);
 9893 }
 9894 
 9895 
 9896 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9897   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9898   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9899   attributes.set_is_evex_instruction();
 9900   attributes.set_embedded_opmask_register_specifier(mask);
 9901   if (merge) {
 9902     attributes.reset_is_clear_context();
 9903   }
 9904   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9905   emit_int16(0x3C, (0xC0 | encode));
 9906 }
 9907 
 9908 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9909   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9910   InstructionMark im(this);
 9911   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9912   attributes.set_is_evex_instruction();
 9913   attributes.set_embedded_opmask_register_specifier(mask);
 9914   if (merge) {
 9915     attributes.reset_is_clear_context();
 9916   }
 9917   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9918   emit_int8(0x3C);
 9919   emit_operand(dst, src);
 9920 }
 9921 
 9922 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9923   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9924   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9925   attributes.set_is_evex_instruction();
 9926   attributes.set_embedded_opmask_register_specifier(mask);
 9927   if (merge) {
 9928     attributes.reset_is_clear_context();
 9929   }
 9930   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9931   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 9932 }
 9933 
 9934 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9935   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9936   InstructionMark im(this);
 9937   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9938   attributes.set_is_evex_instruction();
 9939   attributes.set_embedded_opmask_register_specifier(mask);
 9940   if (merge) {
 9941     attributes.reset_is_clear_context();
 9942   }
 9943   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9944   emit_int8((unsigned char)0xEE);
 9945   emit_operand(dst, src);
 9946 }
 9947 
 9948 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9949   assert(VM_Version::supports_evex(), "");
 9950   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9951   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9952   attributes.set_is_evex_instruction();
 9953   attributes.set_embedded_opmask_register_specifier(mask);
 9954   if (merge) {
 9955     attributes.reset_is_clear_context();
 9956   }
 9957   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9958   emit_int16(0x3D, (0xC0 | encode));
 9959 }
 9960 
 9961 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9962   assert(VM_Version::supports_evex(), "");
 9963   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9964   InstructionMark im(this);
 9965   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9966   attributes.set_is_evex_instruction();
 9967   attributes.set_embedded_opmask_register_specifier(mask);
 9968   if (merge) {
 9969     attributes.reset_is_clear_context();
 9970   }
 9971   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9972   emit_int8(0x3D);
 9973   emit_operand(dst, src);
 9974 }
 9975 
 9976 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9977   assert(VM_Version::supports_evex(), "");
 9978   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9979   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9980   attributes.set_is_evex_instruction();
 9981   attributes.set_embedded_opmask_register_specifier(mask);
 9982   if (merge) {
 9983     attributes.reset_is_clear_context();
 9984   }
 9985   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9986   emit_int16(0x3D, (0xC0 | encode));
 9987 }
 9988 
 9989 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9990   assert(VM_Version::supports_evex(), "");
 9991   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9992   InstructionMark im(this);
 9993   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9994   attributes.set_is_evex_instruction();
 9995   attributes.set_embedded_opmask_register_specifier(mask);
 9996   if (merge) {
 9997     attributes.reset_is_clear_context();
 9998   }
 9999   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10000   emit_int8(0x3D);
10001   emit_operand(dst, src);
10002 }
10003 
10004 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10005   assert(VM_Version::supports_evex(), "requires EVEX support");
10006   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10008   attributes.set_is_evex_instruction();
10009   attributes.set_embedded_opmask_register_specifier(mask);
10010   if (merge) {
10011     attributes.reset_is_clear_context();
10012   }
10013   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10014   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10015 }
10016 
10017 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10018   assert(VM_Version::supports_evex(), "requires EVEX support");
10019   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10020   assert(dst != xnoreg, "sanity");
10021   InstructionMark im(this);
10022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10023   attributes.set_is_evex_instruction();
10024   attributes.set_embedded_opmask_register_specifier(mask);
10025   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10026   if (merge) {
10027     attributes.reset_is_clear_context();
10028   }
10029   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10030   emit_int8(0x25);
10031   emit_operand(dst, src3);
10032   emit_int8(imm8);
10033 }
10034 
10035 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10036   assert(VM_Version::supports_evex(), "requires EVEX support");
10037   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10038   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10039   attributes.set_is_evex_instruction();
10040   attributes.set_embedded_opmask_register_specifier(mask);
10041   if (merge) {
10042     attributes.reset_is_clear_context();
10043   }
10044   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10045   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10046 }
10047 
10048 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10049   assert(VM_Version::supports_evex(), "requires EVEX support");
10050   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10051   assert(dst != xnoreg, "sanity");
10052   InstructionMark im(this);
10053   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10054   attributes.set_is_evex_instruction();
10055   attributes.set_embedded_opmask_register_specifier(mask);
10056   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10057   if (merge) {
10058     attributes.reset_is_clear_context();
10059   }
10060   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10061   emit_int8(0x25);
10062   emit_operand(dst, src3);
10063   emit_int8(imm8);
10064 }
10065 
10066 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10067   assert(VM_Version::supports_gfni(), "requires GFNI support");
10068   assert(VM_Version::supports_sse(), "");
10069   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10070   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10071   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10072 }
10073 
10074 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10075 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10076   assert(UseAVX >= 2, "");
10077   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10078   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10079   emit_int16(0x58, (0xC0 | encode));
10080 }
10081 
10082 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10083   assert(VM_Version::supports_avx2(), "");
10084   assert(dst != xnoreg, "sanity");
10085   InstructionMark im(this);
10086   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10087   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10088   // swap src<->dst for encoding
10089   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10090   emit_int8(0x58);
10091   emit_operand(dst, src);
10092 }
10093 
10094 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10095 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10096   assert(VM_Version::supports_avx2(), "");
10097   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10098   attributes.set_rex_vex_w_reverted();
10099   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10100   emit_int16(0x59, (0xC0 | encode));
10101 }
10102 
10103 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10104   assert(VM_Version::supports_avx2(), "");
10105   assert(dst != xnoreg, "sanity");
10106   InstructionMark im(this);
10107   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10108   attributes.set_rex_vex_w_reverted();
10109   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10110   // swap src<->dst for encoding
10111   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10112   emit_int8(0x59);
10113   emit_operand(dst, src);
10114 }
10115 
10116 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10117   assert(vector_len != Assembler::AVX_128bit, "");
10118   assert(VM_Version::supports_evex(), "");
10119   assert(dst != xnoreg, "sanity");
10120   InstructionMark im(this);
10121   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10122   attributes.set_rex_vex_w_reverted();
10123   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10124   // swap src<->dst for encoding
10125   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10126   emit_int8(0x5A);
10127   emit_operand(dst, src);
10128 }
10129 
10130 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10131   assert(vector_len != Assembler::AVX_128bit, "");
10132   assert(VM_Version::supports_avx512dq(), "");
10133   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10134   attributes.set_rex_vex_w_reverted();
10135   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10136   emit_int16(0x5A, (0xC0 | encode));
10137 }
10138 
10139 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10140   assert(vector_len != Assembler::AVX_128bit, "");
10141   assert(VM_Version::supports_avx512dq(), "");
10142   assert(dst != xnoreg, "sanity");
10143   InstructionMark im(this);
10144   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10145   attributes.set_rex_vex_w_reverted();
10146   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10147   // swap src<->dst for encoding
10148   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10149   emit_int8(0x5A);
10150   emit_operand(dst, src);
10151 }
10152 
10153 // scalar single/double precision replicate
10154 
10155 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10156 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10157   assert(VM_Version::supports_avx2(), "");
10158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10159   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10160   emit_int16(0x18, (0xC0 | encode));
10161 }
10162 
10163 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10164   assert(VM_Version::supports_avx(), "");
10165   assert(dst != xnoreg, "sanity");
10166   InstructionMark im(this);
10167   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10168   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10169   // swap src<->dst for encoding
10170   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10171   emit_int8(0x18);
10172   emit_operand(dst, src);
10173 }
10174 
10175 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10176 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10177   assert(VM_Version::supports_avx2(), "");
10178   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10179   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10180   attributes.set_rex_vex_w_reverted();
10181   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10182   emit_int16(0x19, (0xC0 | encode));
10183 }
10184 
10185 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10186   assert(VM_Version::supports_avx(), "");
10187   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10188   assert(dst != xnoreg, "sanity");
10189   InstructionMark im(this);
10190   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10191   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10192   attributes.set_rex_vex_w_reverted();
10193   // swap src<->dst for encoding
10194   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10195   emit_int8(0x19);
10196   emit_operand(dst, src);
10197 }
10198 
10199 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10200   assert(VM_Version::supports_avx(), "");
10201   assert(vector_len == AVX_256bit, "");
10202   assert(dst != xnoreg, "sanity");
10203   InstructionMark im(this);
10204   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10205   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10206   // swap src<->dst for encoding
10207   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10208   emit_int8(0x1A);
10209   emit_operand(dst, src);
10210 }
10211 
10212 // gpr source broadcast forms
10213 
10214 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10215 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10216   assert(VM_Version::supports_avx512bw(), "");
10217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10218   attributes.set_is_evex_instruction();
10219   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10220   emit_int16(0x7A, (0xC0 | encode));
10221 }
10222 
10223 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10224 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10225   assert(VM_Version::supports_avx512bw(), "");
10226   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10227   attributes.set_is_evex_instruction();
10228   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10229   emit_int16(0x7B, (0xC0 | encode));
10230 }
10231 
10232 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10233 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10234   assert(VM_Version::supports_evex(), "");
10235   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10236   attributes.set_is_evex_instruction();
10237   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10238   emit_int16(0x7C, (0xC0 | encode));
10239 }
10240 
10241 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10242 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10243   assert(VM_Version::supports_evex(), "");
10244   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10245   attributes.set_is_evex_instruction();
10246   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10247   emit_int16(0x7C, (0xC0 | encode));
10248 }
10249 
10250 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10251   assert(VM_Version::supports_avx2(), "");
10252   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10253   assert(dst != xnoreg, "sanity");
10254   assert(src.isxmmindex(),"expected to be xmm index");
10255   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10256   InstructionMark im(this);
10257   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10258   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10259   emit_int8((unsigned char)0x90);
10260   emit_operand(dst, src);
10261 }
10262 
10263 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10264   assert(VM_Version::supports_avx2(), "");
10265   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10266   assert(dst != xnoreg, "sanity");
10267   assert(src.isxmmindex(),"expected to be xmm index");
10268   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10269   InstructionMark im(this);
10270   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10271   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10272   emit_int8((unsigned char)0x90);
10273   emit_operand(dst, src);
10274 }
10275 
10276 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10277   assert(VM_Version::supports_avx2(), "");
10278   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10279   assert(dst != xnoreg, "sanity");
10280   assert(src.isxmmindex(),"expected to be xmm index");
10281   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10282   InstructionMark im(this);
10283   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10284   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10285   emit_int8((unsigned char)0x92);
10286   emit_operand(dst, src);
10287 }
10288 
10289 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10290   assert(VM_Version::supports_avx2(), "");
10291   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10292   assert(dst != xnoreg, "sanity");
10293   assert(src.isxmmindex(),"expected to be xmm index");
10294   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10295   InstructionMark im(this);
10296   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10297   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10298   emit_int8((unsigned char)0x92);
10299   emit_operand(dst, src);
10300 }
10301 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10302   assert(VM_Version::supports_evex(), "");
10303   assert(dst != xnoreg, "sanity");
10304   assert(src.isxmmindex(),"expected to be xmm index");
10305   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10306   assert(mask != k0, "instruction will #UD if mask is in k0");
10307   InstructionMark im(this);
10308   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10309   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10310   attributes.reset_is_clear_context();
10311   attributes.set_embedded_opmask_register_specifier(mask);
10312   attributes.set_is_evex_instruction();
10313   // swap src<->dst for encoding
10314   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10315   emit_int8((unsigned char)0x90);
10316   emit_operand(dst, src);
10317 }
10318 
10319 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10320   assert(VM_Version::supports_evex(), "");
10321   assert(dst != xnoreg, "sanity");
10322   assert(src.isxmmindex(),"expected to be xmm index");
10323   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10324   assert(mask != k0, "instruction will #UD if mask is in k0");
10325   InstructionMark im(this);
10326   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10327   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10328   attributes.reset_is_clear_context();
10329   attributes.set_embedded_opmask_register_specifier(mask);
10330   attributes.set_is_evex_instruction();
10331   // swap src<->dst for encoding
10332   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10333   emit_int8((unsigned char)0x90);
10334   emit_operand(dst, src);
10335 }
10336 
10337 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10338   assert(VM_Version::supports_evex(), "");
10339   assert(dst != xnoreg, "sanity");
10340   assert(src.isxmmindex(),"expected to be xmm index");
10341   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10342   assert(mask != k0, "instruction will #UD if mask is in k0");
10343   InstructionMark im(this);
10344   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10345   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10346   attributes.reset_is_clear_context();
10347   attributes.set_embedded_opmask_register_specifier(mask);
10348   attributes.set_is_evex_instruction();
10349   // swap src<->dst for encoding
10350   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10351   emit_int8((unsigned char)0x92);
10352   emit_operand(dst, src);
10353 }
10354 
10355 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10356   assert(VM_Version::supports_evex(), "");
10357   assert(dst != xnoreg, "sanity");
10358   assert(src.isxmmindex(),"expected to be xmm index");
10359   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10360   assert(mask != k0, "instruction will #UD if mask is in k0");
10361   InstructionMark im(this);
10362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10363   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10364   attributes.reset_is_clear_context();
10365   attributes.set_embedded_opmask_register_specifier(mask);
10366   attributes.set_is_evex_instruction();
10367   // swap src<->dst for encoding
10368   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10369   emit_int8((unsigned char)0x92);
10370   emit_operand(dst, src);
10371 }
10372 
10373 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10374   assert(VM_Version::supports_evex(), "");
10375   assert(mask != k0, "instruction will #UD if mask is in k0");
10376   InstructionMark im(this);
10377   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10378   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10379   attributes.reset_is_clear_context();
10380   attributes.set_embedded_opmask_register_specifier(mask);
10381   attributes.set_is_evex_instruction();
10382   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10383   emit_int8((unsigned char)0xA0);
10384   emit_operand(src, dst);
10385 }
10386 
10387 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10388   assert(VM_Version::supports_evex(), "");
10389   assert(mask != k0, "instruction will #UD if mask is in k0");
10390   InstructionMark im(this);
10391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10392   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10393   attributes.reset_is_clear_context();
10394   attributes.set_embedded_opmask_register_specifier(mask);
10395   attributes.set_is_evex_instruction();
10396   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10397   emit_int8((unsigned char)0xA0);
10398   emit_operand(src, dst);
10399 }
10400 
10401 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10402   assert(VM_Version::supports_evex(), "");
10403   assert(mask != k0, "instruction will #UD if mask is in k0");
10404   InstructionMark im(this);
10405   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10406   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10407   attributes.reset_is_clear_context();
10408   attributes.set_embedded_opmask_register_specifier(mask);
10409   attributes.set_is_evex_instruction();
10410   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10411   emit_int8((unsigned char)0xA2);
10412   emit_operand(src, dst);
10413 }
10414 
10415 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10416   assert(VM_Version::supports_evex(), "");
10417   assert(mask != k0, "instruction will #UD if mask is in k0");
10418   InstructionMark im(this);
10419   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10420   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10421   attributes.reset_is_clear_context();
10422   attributes.set_embedded_opmask_register_specifier(mask);
10423   attributes.set_is_evex_instruction();
10424   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10425   emit_int8((unsigned char)0xA2);
10426   emit_operand(src, dst);
10427 }
10428 // Carry-Less Multiplication Quadword
10429 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10430   assert(VM_Version::supports_clmul(), "");
10431   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10432   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10433   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10434 }
10435 
10436 // Carry-Less Multiplication Quadword
10437 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10438   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10439   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10440   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10441   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10442 }
10443 
10444 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10445   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10446   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10447   attributes.set_is_evex_instruction();
10448   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10449   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10450 }
10451 
10452 void Assembler::vzeroupper_uncached() {
10453   if (VM_Version::supports_vzeroupper()) {
10454     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10455     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10456     emit_int8(0x77);
10457   }
10458 }
10459 
10460 void Assembler::fld_x(Address adr) {
10461   InstructionMark im(this);
10462   emit_int8((unsigned char)0xDB);
10463   emit_operand32(rbp, adr);
10464 }
10465 
10466 void Assembler::fstp_x(Address adr) {
10467   InstructionMark im(this);
10468   emit_int8((unsigned char)0xDB);
10469   emit_operand32(rdi, adr);
10470 }
10471 
10472 void Assembler::emit_operand32(Register reg, Address adr) {
10473   assert(reg->encoding() < 8, "no extended registers");
10474   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10475   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
10476                adr._rspec);
10477 }
10478 
10479 #ifndef _LP64
10480 // 32bit only pieces of the assembler
10481 
10482 void Assembler::emms() {
10483   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10484   emit_int16(0x0F, 0x77);
10485 }
10486 
10487 void Assembler::vzeroupper() {
10488   vzeroupper_uncached();
10489 }
10490 
10491 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10492   // NO PREFIX AS NEVER 64BIT
10493   InstructionMark im(this);
10494   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10495   emit_data(imm32, rspec, 0);
10496 }
10497 
10498 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10499   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10500   InstructionMark im(this);
10501   emit_int8((unsigned char)0x81);
10502   emit_operand(rdi, src1);
10503   emit_data(imm32, rspec, 0);
10504 }
10505 
10506 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10507 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10508 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10509 void Assembler::cmpxchg8(Address adr) {
10510   InstructionMark im(this);
10511   emit_int16(0x0F, (unsigned char)0xC7);
10512   emit_operand(rcx, adr);
10513 }
10514 
10515 void Assembler::decl(Register dst) {
10516   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10517  emit_int8(0x48 | dst->encoding());
10518 }
10519 
10520 // 64bit doesn't use the x87
10521 
10522 void Assembler::emit_farith(int b1, int b2, int i) {
10523   assert(isByte(b1) && isByte(b2), "wrong opcode");
10524   assert(0 <= i &&  i < 8, "illegal stack offset");
10525   emit_int16(b1, b2 + i);
10526 }
10527 
10528 void Assembler::fabs() {
10529   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10530 }
10531 
10532 void Assembler::fadd(int i) {
10533   emit_farith(0xD8, 0xC0, i);
10534 }
10535 
10536 void Assembler::fadd_d(Address src) {
10537   InstructionMark im(this);
10538   emit_int8((unsigned char)0xDC);
10539   emit_operand32(rax, src);
10540 }
10541 
10542 void Assembler::fadd_s(Address src) {
10543   InstructionMark im(this);
10544   emit_int8((unsigned char)0xD8);
10545   emit_operand32(rax, src);
10546 }
10547 
10548 void Assembler::fadda(int i) {
10549   emit_farith(0xDC, 0xC0, i);
10550 }
10551 
10552 void Assembler::faddp(int i) {
10553   emit_farith(0xDE, 0xC0, i);
10554 }
10555 
10556 void Assembler::fchs() {
10557   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10558 }
10559 
10560 void Assembler::fcom(int i) {
10561   emit_farith(0xD8, 0xD0, i);
10562 }
10563 
10564 void Assembler::fcomp(int i) {
10565   emit_farith(0xD8, 0xD8, i);
10566 }
10567 
10568 void Assembler::fcomp_d(Address src) {
10569   InstructionMark im(this);
10570   emit_int8((unsigned char)0xDC);
10571   emit_operand32(rbx, src);
10572 }
10573 
10574 void Assembler::fcomp_s(Address src) {
10575   InstructionMark im(this);
10576   emit_int8((unsigned char)0xD8);
10577   emit_operand32(rbx, src);
10578 }
10579 
10580 void Assembler::fcompp() {
10581   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10582 }
10583 
10584 void Assembler::fcos() {
10585   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10586 }
10587 
10588 void Assembler::fdecstp() {
10589   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10590 }
10591 
10592 void Assembler::fdiv(int i) {
10593   emit_farith(0xD8, 0xF0, i);
10594 }
10595 
10596 void Assembler::fdiv_d(Address src) {
10597   InstructionMark im(this);
10598   emit_int8((unsigned char)0xDC);
10599   emit_operand32(rsi, src);
10600 }
10601 
10602 void Assembler::fdiv_s(Address src) {
10603   InstructionMark im(this);
10604   emit_int8((unsigned char)0xD8);
10605   emit_operand32(rsi, src);
10606 }
10607 
10608 void Assembler::fdiva(int i) {
10609   emit_farith(0xDC, 0xF8, i);
10610 }
10611 
10612 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10613 //       is erroneous for some of the floating-point instructions below.
10614 
10615 void Assembler::fdivp(int i) {
10616   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10617 }
10618 
10619 void Assembler::fdivr(int i) {
10620   emit_farith(0xD8, 0xF8, i);
10621 }
10622 
10623 void Assembler::fdivr_d(Address src) {
10624   InstructionMark im(this);
10625   emit_int8((unsigned char)0xDC);
10626   emit_operand32(rdi, src);
10627 }
10628 
10629 void Assembler::fdivr_s(Address src) {
10630   InstructionMark im(this);
10631   emit_int8((unsigned char)0xD8);
10632   emit_operand32(rdi, src);
10633 }
10634 
10635 void Assembler::fdivra(int i) {
10636   emit_farith(0xDC, 0xF0, i);
10637 }
10638 
10639 void Assembler::fdivrp(int i) {
10640   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10641 }
10642 
10643 void Assembler::ffree(int i) {
10644   emit_farith(0xDD, 0xC0, i);
10645 }
10646 
10647 void Assembler::fild_d(Address adr) {
10648   InstructionMark im(this);
10649   emit_int8((unsigned char)0xDF);
10650   emit_operand32(rbp, adr);
10651 }
10652 
10653 void Assembler::fild_s(Address adr) {
10654   InstructionMark im(this);
10655   emit_int8((unsigned char)0xDB);
10656   emit_operand32(rax, adr);
10657 }
10658 
10659 void Assembler::fincstp() {
10660   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10661 }
10662 
10663 void Assembler::finit() {
10664   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10665 }
10666 
10667 void Assembler::fist_s(Address adr) {
10668   InstructionMark im(this);
10669   emit_int8((unsigned char)0xDB);
10670   emit_operand32(rdx, adr);
10671 }
10672 
10673 void Assembler::fistp_d(Address adr) {
10674   InstructionMark im(this);
10675   emit_int8((unsigned char)0xDF);
10676   emit_operand32(rdi, adr);
10677 }
10678 
10679 void Assembler::fistp_s(Address adr) {
10680   InstructionMark im(this);
10681   emit_int8((unsigned char)0xDB);
10682   emit_operand32(rbx, adr);
10683 }
10684 
10685 void Assembler::fld1() {
10686   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10687 }
10688 
10689 void Assembler::fld_d(Address adr) {
10690   InstructionMark im(this);
10691   emit_int8((unsigned char)0xDD);
10692   emit_operand32(rax, adr);
10693 }
10694 
10695 void Assembler::fld_s(Address adr) {
10696   InstructionMark im(this);
10697   emit_int8((unsigned char)0xD9);
10698   emit_operand32(rax, adr);
10699 }
10700 
10701 
10702 void Assembler::fld_s(int index) {
10703   emit_farith(0xD9, 0xC0, index);
10704 }
10705 
10706 void Assembler::fldcw(Address src) {
10707   InstructionMark im(this);
10708   emit_int8((unsigned char)0xD9);
10709   emit_operand32(rbp, src);
10710 }
10711 
10712 void Assembler::fldenv(Address src) {
10713   InstructionMark im(this);
10714   emit_int8((unsigned char)0xD9);
10715   emit_operand32(rsp, src);
10716 }
10717 
10718 void Assembler::fldlg2() {
10719   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10720 }
10721 
10722 void Assembler::fldln2() {
10723   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10724 }
10725 
10726 void Assembler::fldz() {
10727   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10728 }
10729 
10730 void Assembler::flog() {
10731   fldln2();
10732   fxch();
10733   fyl2x();
10734 }
10735 
10736 void Assembler::flog10() {
10737   fldlg2();
10738   fxch();
10739   fyl2x();
10740 }
10741 
10742 void Assembler::fmul(int i) {
10743   emit_farith(0xD8, 0xC8, i);
10744 }
10745 
10746 void Assembler::fmul_d(Address src) {
10747   InstructionMark im(this);
10748   emit_int8((unsigned char)0xDC);
10749   emit_operand32(rcx, src);
10750 }
10751 
10752 void Assembler::fmul_s(Address src) {
10753   InstructionMark im(this);
10754   emit_int8((unsigned char)0xD8);
10755   emit_operand32(rcx, src);
10756 }
10757 
10758 void Assembler::fmula(int i) {
10759   emit_farith(0xDC, 0xC8, i);
10760 }
10761 
10762 void Assembler::fmulp(int i) {
10763   emit_farith(0xDE, 0xC8, i);
10764 }
10765 
10766 void Assembler::fnsave(Address dst) {
10767   InstructionMark im(this);
10768   emit_int8((unsigned char)0xDD);
10769   emit_operand32(rsi, dst);
10770 }
10771 
10772 void Assembler::fnstcw(Address src) {
10773   InstructionMark im(this);
10774   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10775   emit_operand32(rdi, src);
10776 }
10777 
10778 void Assembler::fnstsw_ax() {
10779   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10780 }
10781 
10782 void Assembler::fprem() {
10783   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10784 }
10785 
10786 void Assembler::fprem1() {
10787   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10788 }
10789 
10790 void Assembler::frstor(Address src) {
10791   InstructionMark im(this);
10792   emit_int8((unsigned char)0xDD);
10793   emit_operand32(rsp, src);
10794 }
10795 
10796 void Assembler::fsin() {
10797   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10798 }
10799 
10800 void Assembler::fsqrt() {
10801   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
10802 }
10803 
10804 void Assembler::fst_d(Address adr) {
10805   InstructionMark im(this);
10806   emit_int8((unsigned char)0xDD);
10807   emit_operand32(rdx, adr);
10808 }
10809 
10810 void Assembler::fst_s(Address adr) {
10811   InstructionMark im(this);
10812   emit_int8((unsigned char)0xD9);
10813   emit_operand32(rdx, adr);
10814 }
10815 
10816 void Assembler::fstp_d(Address adr) {
10817   InstructionMark im(this);
10818   emit_int8((unsigned char)0xDD);
10819   emit_operand32(rbx, adr);
10820 }
10821 
10822 void Assembler::fstp_d(int index) {
10823   emit_farith(0xDD, 0xD8, index);
10824 }
10825 
10826 void Assembler::fstp_s(Address adr) {
10827   InstructionMark im(this);
10828   emit_int8((unsigned char)0xD9);
10829   emit_operand32(rbx, adr);
10830 }
10831 
10832 void Assembler::fsub(int i) {
10833   emit_farith(0xD8, 0xE0, i);
10834 }
10835 
10836 void Assembler::fsub_d(Address src) {
10837   InstructionMark im(this);
10838   emit_int8((unsigned char)0xDC);
10839   emit_operand32(rsp, src);
10840 }
10841 
10842 void Assembler::fsub_s(Address src) {
10843   InstructionMark im(this);
10844   emit_int8((unsigned char)0xD8);
10845   emit_operand32(rsp, src);
10846 }
10847 
10848 void Assembler::fsuba(int i) {
10849   emit_farith(0xDC, 0xE8, i);
10850 }
10851 
10852 void Assembler::fsubp(int i) {
10853   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
10854 }
10855 
10856 void Assembler::fsubr(int i) {
10857   emit_farith(0xD8, 0xE8, i);
10858 }
10859 
10860 void Assembler::fsubr_d(Address src) {
10861   InstructionMark im(this);
10862   emit_int8((unsigned char)0xDC);
10863   emit_operand32(rbp, src);
10864 }
10865 
10866 void Assembler::fsubr_s(Address src) {
10867   InstructionMark im(this);
10868   emit_int8((unsigned char)0xD8);
10869   emit_operand32(rbp, src);
10870 }
10871 
10872 void Assembler::fsubra(int i) {
10873   emit_farith(0xDC, 0xE0, i);
10874 }
10875 
10876 void Assembler::fsubrp(int i) {
10877   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
10878 }
10879 
10880 void Assembler::ftan() {
10881   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
10882 }
10883 
10884 void Assembler::ftst() {
10885   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
10886 }
10887 
10888 void Assembler::fucomi(int i) {
10889   // make sure the instruction is supported (introduced for P6, together with cmov)
10890   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10891   emit_farith(0xDB, 0xE8, i);
10892 }
10893 
10894 void Assembler::fucomip(int i) {
10895   // make sure the instruction is supported (introduced for P6, together with cmov)
10896   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10897   emit_farith(0xDF, 0xE8, i);
10898 }
10899 
10900 void Assembler::fwait() {
10901   emit_int8((unsigned char)0x9B);
10902 }
10903 
10904 void Assembler::fxch(int i) {
10905   emit_farith(0xD9, 0xC8, i);
10906 }
10907 
10908 void Assembler::fyl2x() {
10909   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
10910 }
10911 
10912 void Assembler::frndint() {
10913   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
10914 }
10915 
10916 void Assembler::f2xm1() {
10917   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
10918 }
10919 
10920 void Assembler::fldl2e() {
10921   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
10922 }
10923 #endif // !_LP64
10924 
10925 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
10926 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
10927 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
10928 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
10929 
10930 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
10931 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10932   if (pre > 0) {
10933     emit_int8(simd_pre[pre]);
10934   }
10935   if (rex_w) {
10936     prefixq(adr, xreg);
10937   } else {
10938     prefix(adr, xreg);
10939   }
10940   if (opc > 0) {
10941     emit_int8(0x0F);
10942     int opc2 = simd_opc[opc];
10943     if (opc2 > 0) {
10944       emit_int8(opc2);
10945     }
10946   }
10947 }
10948 
10949 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10950   if (pre > 0) {
10951     emit_int8(simd_pre[pre]);
10952   }
10953   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
10954   if (opc > 0) {
10955     emit_int8(0x0F);
10956     int opc2 = simd_opc[opc];
10957     if (opc2 > 0) {
10958       emit_int8(opc2);
10959     }
10960   }
10961   return encode;
10962 }
10963 
10964 
10965 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
10966   int vector_len = _attributes->get_vector_len();
10967   bool vex_w = _attributes->is_rex_vex_w();
10968   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
10969     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
10970     byte1 = (~byte1) & 0xE0;
10971     byte1 |= opc;
10972 
10973     int byte2 = ((~nds_enc) & 0xf) << 3;
10974     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
10975 
10976     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
10977   } else {
10978     int byte1 = vex_r ? VEX_R : 0;
10979     byte1 = (~byte1) & 0x80;
10980     byte1 |= ((~nds_enc) & 0xf) << 3;
10981     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
10982     emit_int16((unsigned char)VEX_2bytes, byte1);
10983   }
10984 }
10985 
10986 // This is a 4 byte encoding
10987 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
10988   // EVEX 0x62 prefix
10989   // byte1 = EVEX_4bytes;
10990 
10991   bool vex_w = _attributes->is_rex_vex_w();
10992   int evex_encoding = (vex_w ? VEX_W : 0);
10993   // EVEX.b is not currently used for broadcast of single element or data rounding modes
10994   _attributes->set_evex_encoding(evex_encoding);
10995 
10996   // P0: byte 2, initialized to RXBR`00mm
10997   // instead of not'd
10998   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
10999   byte2 = (~byte2) & 0xF0;
11000   // confine opc opcode extensions in mm bits to lower two bits
11001   // of form {0F, 0F_38, 0F_3A}
11002   byte2 |= opc;
11003 
11004   // P1: byte 3 as Wvvvv1pp
11005   int byte3 = ((~nds_enc) & 0xf) << 3;
11006   // p[10] is always 1
11007   byte3 |= EVEX_F;
11008   byte3 |= (vex_w & 1) << 7;
11009   // confine pre opcode extensions in pp bits to lower two bits
11010   // of form {66, F3, F2}
11011   byte3 |= pre;
11012 
11013   // P2: byte 4 as zL'Lbv'aaa
11014   // kregs are implemented in the low 3 bits as aaa
11015   int byte4 = (_attributes->is_no_reg_mask()) ?
11016               0 :
11017               _attributes->get_embedded_opmask_register_specifier();
11018   // EVEX.v` for extending EVEX.vvvv or VIDX
11019   byte4 |= (evex_v ? 0: EVEX_V);
11020   // third EXEC.b for broadcast actions
11021   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11022   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11023   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11024   // last is EVEX.z for zero/merge actions
11025   if (_attributes->is_no_reg_mask() == false &&
11026       _attributes->get_embedded_opmask_register_specifier() != 0) {
11027     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11028   }
11029 
11030   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11031 }
11032 
11033 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11034   bool vex_r = (xreg_enc & 8) == 8;
11035   bool vex_b = adr.base_needs_rex();
11036   bool vex_x;
11037   if (adr.isxmmindex()) {
11038     vex_x = adr.xmmindex_needs_rex();
11039   } else {
11040     vex_x = adr.index_needs_rex();
11041   }
11042   set_attributes(attributes);
11043   attributes->set_current_assembler(this);
11044 
11045   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11046   // is allowed in legacy mode and has resources which will fit in it.
11047   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11048   if (!attributes->is_legacy_mode()) {
11049     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11050       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11051           attributes->set_is_legacy_mode();
11052       }
11053     }
11054   }
11055 
11056   if (UseAVX > 2) {
11057     assert(((!attributes->uses_vl()) ||
11058             (attributes->get_vector_len() == AVX_512bit) ||
11059             (!_legacy_mode_vl) ||
11060             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11061     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11062   }
11063 
11064   clear_managed();
11065   if (UseAVX > 2 && !attributes->is_legacy_mode())
11066   {
11067     bool evex_r = (xreg_enc >= 16);
11068     bool evex_v;
11069     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11070     if (adr.isxmmindex())  {
11071       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11072     } else {
11073       evex_v = (nds_enc >= 16);
11074     }
11075     attributes->set_is_evex_instruction();
11076     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11077   } else {
11078     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11079       attributes->set_rex_vex_w(false);
11080     }
11081     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11082   }
11083 }
11084 
11085 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11086   bool vex_r = (dst_enc & 8) == 8;
11087   bool vex_b = (src_enc & 8) == 8;
11088   bool vex_x = false;
11089   set_attributes(attributes);
11090   attributes->set_current_assembler(this);
11091 
11092   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11093   // is allowed in legacy mode and has resources which will fit in it.
11094   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11095   if (!attributes->is_legacy_mode()) {
11096     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11097       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11098           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11099           attributes->set_is_legacy_mode();
11100       }
11101     }
11102   }
11103 
11104   if (UseAVX > 2) {
11105     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11106     // Instruction with uses_vl true are vector instructions
11107     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11108     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11109     // Rest all should have legacy_mode set as true
11110     assert(((!attributes->uses_vl()) ||
11111             (attributes->get_vector_len() == AVX_512bit) ||
11112             (!_legacy_mode_vl) ||
11113             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11114     // Instruction with legacy_mode true should have dst, nds and src < 15
11115     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11116   }
11117 
11118   clear_managed();
11119   if (UseAVX > 2 && !attributes->is_legacy_mode())
11120   {
11121     bool evex_r = (dst_enc >= 16);
11122     bool evex_v = (nds_enc >= 16);
11123     // can use vex_x as bank extender on rm encoding
11124     vex_x = (src_enc >= 16);
11125     attributes->set_is_evex_instruction();
11126     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11127   } else {
11128     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11129       attributes->set_rex_vex_w(false);
11130     }
11131     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11132   }
11133 
11134   // return modrm byte components for operands
11135   return (((dst_enc & 7) << 3) | (src_enc & 7));
11136 }
11137 
11138 
11139 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11140                             VexOpcode opc, InstructionAttr *attributes) {
11141   if (UseAVX > 0) {
11142     int xreg_enc = xreg->encoding();
11143     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11144     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11145   } else {
11146     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11147     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11148   }
11149 }
11150 
11151 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11152                                       VexOpcode opc, InstructionAttr *attributes) {
11153   int dst_enc = dst->encoding();
11154   int src_enc = src->encoding();
11155   if (UseAVX > 0) {
11156     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11157     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11158   } else {
11159     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11160     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11161   }
11162 }
11163 
11164 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11165   assert(VM_Version::supports_avx(), "");
11166   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11167   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11168   emit_int16(0x5F, (0xC0 | encode));
11169 }
11170 
11171 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11172   assert(VM_Version::supports_avx(), "");
11173   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11174   attributes.set_rex_vex_w_reverted();
11175   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11176   emit_int16(0x5F, (0xC0 | encode));
11177 }
11178 
11179 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11180   assert(VM_Version::supports_avx(), "");
11181   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11182   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11183   emit_int16(0x5D, (0xC0 | encode));
11184 }
11185 
11186 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11187   assert(VM_Version::supports_avx(), "");
11188   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11189   attributes.set_rex_vex_w_reverted();
11190   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11191   emit_int16(0x5D, (0xC0 | encode));
11192 }
11193 
11194 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11195   assert(VM_Version::supports_avx(), "");
11196   assert(vector_len <= AVX_256bit, "");
11197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11198   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11199   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11200 }
11201 
11202 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11203   assert(VM_Version::supports_avx(), "");
11204   assert(vector_len <= AVX_256bit, "");
11205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11206   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11207   int src2_enc = src2->encoding();
11208   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11209 }
11210 
11211 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11212   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11213   assert(vector_len <= AVX_256bit, "");
11214   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11216   int src2_enc = src2->encoding();
11217   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11218 }
11219 
11220 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11221   assert(VM_Version::supports_avx2(), "");
11222   assert(vector_len <= AVX_256bit, "");
11223   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11224   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11225   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11226 }
11227 
11228 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11229   assert(VM_Version::supports_avx(), "");
11230   assert(vector_len <= AVX_256bit, "");
11231   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11232   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11233   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11234 }
11235 
11236 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11237                         ComparisonPredicateFP comparison, int vector_len) {
11238   assert(VM_Version::supports_evex(), "");
11239   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11240   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11241   attributes.set_is_evex_instruction();
11242   attributes.set_embedded_opmask_register_specifier(mask);
11243   attributes.reset_is_clear_context();
11244   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11245   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11246 }
11247 
11248 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11249                         ComparisonPredicateFP comparison, int vector_len) {
11250   assert(VM_Version::supports_evex(), "");
11251   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11252   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11253   attributes.set_is_evex_instruction();
11254   attributes.set_embedded_opmask_register_specifier(mask);
11255   attributes.reset_is_clear_context();
11256   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11257   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11258 }
11259 
11260 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11261   assert(VM_Version::supports_sse4_1(), "");
11262   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11263   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11264   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11265   emit_int16(0x14, (0xC0 | encode));
11266 }
11267 
11268 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11269   assert(VM_Version::supports_sse4_1(), "");
11270   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11272   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11273   emit_int16(0x15, (0xC0 | encode));
11274 }
11275 
11276 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11277   assert(VM_Version::supports_sse4_1(), "");
11278   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11280   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11281   emit_int16(0x10, (0xC0 | encode));
11282 }
11283 
11284 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11285   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11286   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11287   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11288   int src2_enc = src2->encoding();
11289   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11290 }
11291 
11292 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11293   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11294   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11295   emit_int24(0x0C, (0xC0 | encode), imm8);
11296 }
11297 
11298 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11299   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11300   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11303   emit_int16(0x64, (0xC0 | encode));
11304 }
11305 
11306 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11307   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11308   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11309   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11311   emit_int16(0x65, (0xC0 | encode));
11312 }
11313 
11314 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11315   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11316   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11317   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11319   emit_int16(0x66, (0xC0 | encode));
11320 }
11321 
11322 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11323   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11324   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11325   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11326   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11327   emit_int16(0x37, (0xC0 | encode));
11328 }
11329 
11330 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11331                         int comparison, bool is_signed, int vector_len) {
11332   assert(VM_Version::supports_evex(), "");
11333   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11334   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11335   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11336   attributes.set_is_evex_instruction();
11337   attributes.set_embedded_opmask_register_specifier(mask);
11338   attributes.reset_is_clear_context();
11339   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11340   int opcode = is_signed ? 0x1F : 0x1E;
11341   emit_int24(opcode, (0xC0 | encode), comparison);
11342 }
11343 
11344 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11345                         int comparison, bool is_signed, int vector_len) {
11346   assert(VM_Version::supports_evex(), "");
11347   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11348   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11349   InstructionMark im(this);
11350   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11351   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11352   attributes.set_is_evex_instruction();
11353   attributes.set_embedded_opmask_register_specifier(mask);
11354   attributes.reset_is_clear_context();
11355   int dst_enc = kdst->encoding();
11356   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11357   int opcode = is_signed ? 0x1F : 0x1E;
11358   emit_int8((unsigned char)opcode);
11359   emit_operand(as_Register(dst_enc), src);
11360   emit_int8((unsigned char)comparison);
11361 }
11362 
11363 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11364                         int comparison, bool is_signed, int vector_len) {
11365   assert(VM_Version::supports_evex(), "");
11366   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11367   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11368   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11369   attributes.set_is_evex_instruction();
11370   attributes.set_embedded_opmask_register_specifier(mask);
11371   attributes.reset_is_clear_context();
11372   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11373   int opcode = is_signed ? 0x1F : 0x1E;
11374   emit_int24(opcode, (0xC0 | encode), comparison);
11375 }
11376 
11377 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11378                         int comparison, bool is_signed, int vector_len) {
11379   assert(VM_Version::supports_evex(), "");
11380   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11381   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11382   InstructionMark im(this);
11383   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11384   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11385   attributes.set_is_evex_instruction();
11386   attributes.set_embedded_opmask_register_specifier(mask);
11387   attributes.reset_is_clear_context();
11388   int dst_enc = kdst->encoding();
11389   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11390   int opcode = is_signed ? 0x1F : 0x1E;
11391   emit_int8((unsigned char)opcode);
11392   emit_operand(as_Register(dst_enc), src);
11393   emit_int8((unsigned char)comparison);
11394 }
11395 
11396 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11397                         int comparison, bool is_signed, int vector_len) {
11398   assert(VM_Version::supports_evex(), "");
11399   assert(VM_Version::supports_avx512bw(), "");
11400   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11401   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11402   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11403   attributes.set_is_evex_instruction();
11404   attributes.set_embedded_opmask_register_specifier(mask);
11405   attributes.reset_is_clear_context();
11406   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11407   int opcode = is_signed ? 0x3F : 0x3E;
11408   emit_int24(opcode, (0xC0 | encode), comparison);
11409 }
11410 
11411 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11412                         int comparison, bool is_signed, int vector_len) {
11413   assert(VM_Version::supports_evex(), "");
11414   assert(VM_Version::supports_avx512bw(), "");
11415   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11416   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11417   InstructionMark im(this);
11418   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11419   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11420   attributes.set_is_evex_instruction();
11421   attributes.set_embedded_opmask_register_specifier(mask);
11422   attributes.reset_is_clear_context();
11423   int dst_enc = kdst->encoding();
11424   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11425   int opcode = is_signed ? 0x3F : 0x3E;
11426   emit_int8((unsigned char)opcode);
11427   emit_operand(as_Register(dst_enc), src);
11428   emit_int8((unsigned char)comparison);
11429 }
11430 
11431 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11432                         int comparison, bool is_signed, int vector_len) {
11433   assert(VM_Version::supports_evex(), "");
11434   assert(VM_Version::supports_avx512bw(), "");
11435   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11436   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11437   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11438   attributes.set_is_evex_instruction();
11439   attributes.set_embedded_opmask_register_specifier(mask);
11440   attributes.reset_is_clear_context();
11441   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11442   int opcode = is_signed ? 0x3F : 0x3E;
11443   emit_int24(opcode, (0xC0 | encode), comparison);
11444 }
11445 
11446 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11447                         int comparison, bool is_signed, int vector_len) {
11448   assert(VM_Version::supports_evex(), "");
11449   assert(VM_Version::supports_avx512bw(), "");
11450   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11451   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11452   InstructionMark im(this);
11453   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11454   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11455   attributes.set_is_evex_instruction();
11456   attributes.set_embedded_opmask_register_specifier(mask);
11457   attributes.reset_is_clear_context();
11458   int dst_enc = kdst->encoding();
11459   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11460   int opcode = is_signed ? 0x3F : 0x3E;
11461   emit_int8((unsigned char)opcode);
11462   emit_operand(as_Register(dst_enc), src);
11463   emit_int8((unsigned char)comparison);
11464 }
11465 
11466 // Register is a class, but it would be assigned numerical value.
11467 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
11468 PRAGMA_DIAG_PUSH
11469 PRAGMA_NONNULL_IGNORED
11470 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11471   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11472   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11473   attributes.set_is_evex_instruction();
11474   attributes.set_embedded_opmask_register_specifier(mask);
11475   if (merge) {
11476     attributes.reset_is_clear_context();
11477   }
11478   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11479   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11480 }
11481 
11482 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11483   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11484   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11485   attributes.set_is_evex_instruction();
11486   attributes.set_embedded_opmask_register_specifier(mask);
11487   if (merge) {
11488     attributes.reset_is_clear_context();
11489   }
11490   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11491   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11492 }
11493 PRAGMA_DIAG_POP
11494 
11495 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11496   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11497   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11498   attributes.set_is_evex_instruction();
11499   attributes.set_embedded_opmask_register_specifier(mask);
11500   if (merge) {
11501     attributes.reset_is_clear_context();
11502   }
11503   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11504   emit_int16(0x14, (0xC0 | encode));
11505 }
11506 
11507 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11508   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11509   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11510   attributes.set_is_evex_instruction();
11511   attributes.set_embedded_opmask_register_specifier(mask);
11512   if (merge) {
11513     attributes.reset_is_clear_context();
11514   }
11515   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11516   emit_int16(0x14, (0xC0 | encode));
11517 }
11518 
11519 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11520   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11521   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11522   attributes.set_is_evex_instruction();
11523   attributes.set_embedded_opmask_register_specifier(mask);
11524   if (merge) {
11525     attributes.reset_is_clear_context();
11526   }
11527   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11528   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11529 }
11530 
11531 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11532   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11533   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11534   attributes.set_is_evex_instruction();
11535   attributes.set_embedded_opmask_register_specifier(mask);
11536   if (merge) {
11537     attributes.reset_is_clear_context();
11538   }
11539   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11540   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11541 }
11542 
11543 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11544   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11546   attributes.set_is_evex_instruction();
11547   attributes.set_embedded_opmask_register_specifier(mask);
11548   if (merge) {
11549     attributes.reset_is_clear_context();
11550   }
11551   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11552   emit_int16(0x15, (0xC0 | encode));
11553 }
11554 
11555 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11556   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11557   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11558   attributes.set_is_evex_instruction();
11559   attributes.set_embedded_opmask_register_specifier(mask);
11560   if (merge) {
11561     attributes.reset_is_clear_context();
11562   }
11563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11564   emit_int16(0x15, (0xC0 | encode));
11565 }
11566 
11567 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11568   assert(VM_Version::supports_avx(), "");
11569   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11570   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11571   int mask_enc = mask->encoding();
11572   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11573 }
11574 
11575 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11576   assert(VM_Version::supports_evex(), "");
11577   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11578   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11579   attributes.set_is_evex_instruction();
11580   attributes.set_embedded_opmask_register_specifier(mask);
11581   if (merge) {
11582     attributes.reset_is_clear_context();
11583   }
11584   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11585   emit_int16(0x65, (0xC0 | encode));
11586 }
11587 
11588 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11589   assert(VM_Version::supports_evex(), "");
11590   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11591   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11592   attributes.set_is_evex_instruction();
11593   attributes.set_embedded_opmask_register_specifier(mask);
11594   if (merge) {
11595     attributes.reset_is_clear_context();
11596   }
11597   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11598   emit_int16(0x65, (0xC0 | encode));
11599 }
11600 
11601 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11602   assert(VM_Version::supports_evex(), "");
11603   assert(VM_Version::supports_avx512bw(), "");
11604   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11606   attributes.set_is_evex_instruction();
11607   attributes.set_embedded_opmask_register_specifier(mask);
11608   if (merge) {
11609     attributes.reset_is_clear_context();
11610   }
11611   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11612   emit_int16(0x66, (0xC0 | encode));
11613 }
11614 
11615 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11616   assert(VM_Version::supports_evex(), "");
11617   assert(VM_Version::supports_avx512bw(), "");
11618   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11620   attributes.set_is_evex_instruction();
11621   attributes.set_embedded_opmask_register_specifier(mask);
11622   if (merge) {
11623     attributes.reset_is_clear_context();
11624   }
11625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11626   emit_int16(0x66, (0xC0 | encode));
11627 }
11628 
11629 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11630   assert(VM_Version::supports_evex(), "");
11631   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11633   attributes.set_is_evex_instruction();
11634   attributes.set_embedded_opmask_register_specifier(mask);
11635   if (merge) {
11636     attributes.reset_is_clear_context();
11637   }
11638   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11639   emit_int16(0x64, (0xC0 | encode));
11640 }
11641 
11642 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11643   assert(VM_Version::supports_evex(), "");
11644   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11645   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11646   attributes.set_is_evex_instruction();
11647   attributes.set_embedded_opmask_register_specifier(mask);
11648   if (merge) {
11649     attributes.reset_is_clear_context();
11650   }
11651   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11652   emit_int16(0x64, (0xC0 | encode));
11653 }
11654 
11655 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11656   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11657   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11658   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11659   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11660 }
11661 
11662 void Assembler::pext(Register dst, Register src1, Register src2) {
11663   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11664   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11665   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11666   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11667 }
11668 
11669 void Assembler::pdep(Register dst, Register src1, Register src2) {
11670   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11671   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11672   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11673   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11674 }
11675 
11676 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11677   assert(VM_Version::supports_bmi2(), "");
11678   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11679   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11680   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11681 }
11682 
11683 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11684   assert(VM_Version::supports_bmi2(), "");
11685   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11686   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11687   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11688 }
11689 
11690 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11691   assert(VM_Version::supports_bmi2(), "");
11692   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11693   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11694   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11695 }
11696 
11697 void Assembler::shrxq(Register dst, Register src1, Register src2) {
11698   assert(VM_Version::supports_bmi2(), "");
11699   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11700   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11701   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11702 }
11703 
11704 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
11705   assert(VM_Version::supports_avx512vldq(), "");
11706   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11707   attributes.set_is_evex_instruction();
11708   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11709   emit_int16(0x39, (0xC0 | encode));
11710 }
11711 
11712 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
11713   assert(VM_Version::supports_avx512vldq(), "");
11714   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11715   attributes.set_is_evex_instruction();
11716   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11717   emit_int16(0x39, (0xC0 | encode));
11718 }
11719 
11720 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
11721   assert(VM_Version::supports_avx512vlbw(), "");
11722   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11723   attributes.set_is_evex_instruction();
11724   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11725   emit_int16(0x29, (0xC0 | encode));
11726 }
11727 
11728 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
11729   assert(VM_Version::supports_avx512vlbw(), "");
11730   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11731   attributes.set_is_evex_instruction();
11732   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11733   emit_int16(0x29, (0xC0 | encode));
11734 }
11735 
11736 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
11737   assert(VM_Version::supports_avx512vldq(), "");
11738   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11739   attributes.set_is_evex_instruction();
11740   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11741   emit_int16(0x38, (0xC0 | encode));
11742 }
11743 
11744 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
11745   assert(VM_Version::supports_avx512vldq(), "");
11746   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11747   attributes.set_is_evex_instruction();
11748   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11749   emit_int16(0x38, (0xC0 | encode));
11750 }
11751 
11752 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
11753   assert(VM_Version::supports_avx512vlbw(), "");
11754   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11755   attributes.set_is_evex_instruction();
11756   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11757   emit_int16(0x28, (0xC0 | encode));
11758 }
11759 
11760 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
11761   assert(VM_Version::supports_avx512vlbw(), "");
11762   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11763   attributes.set_is_evex_instruction();
11764   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11765   emit_int16(0x28, (0xC0 | encode));
11766 }
11767 
11768 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11769   assert(VM_Version::supports_avx512_vbmi2() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11770   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11771   attributes.set_embedded_opmask_register_specifier(mask);
11772   attributes.set_is_evex_instruction();
11773   if (merge) {
11774     attributes.reset_is_clear_context();
11775   }
11776   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11777   emit_int16((unsigned char)0x63, (0xC0 | encode));
11778 }
11779 
11780 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11781   assert(VM_Version::supports_avx512_vbmi2() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11782   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11783   attributes.set_embedded_opmask_register_specifier(mask);
11784   attributes.set_is_evex_instruction();
11785   if (merge) {
11786     attributes.reset_is_clear_context();
11787   }
11788   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11789   emit_int16((unsigned char)0x63, (0xC0 | encode));
11790 }
11791 
11792 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11793   assert(VM_Version::supports_evex() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11794   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11795   attributes.set_embedded_opmask_register_specifier(mask);
11796   attributes.set_is_evex_instruction();
11797   if (merge) {
11798     attributes.reset_is_clear_context();
11799   }
11800   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11801   emit_int16((unsigned char)0x8B, (0xC0 | encode));
11802 }
11803 
11804 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11805   assert(VM_Version::supports_evex() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11806   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11807   attributes.set_embedded_opmask_register_specifier(mask);
11808   attributes.set_is_evex_instruction();
11809   if (merge) {
11810     attributes.reset_is_clear_context();
11811   }
11812   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11813   emit_int16((unsigned char)0x8B, (0xC0 | encode));
11814 }
11815 
11816 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11817   assert(VM_Version::supports_evex() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11818   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11819   attributes.set_embedded_opmask_register_specifier(mask);
11820   attributes.set_is_evex_instruction();
11821   if (merge) {
11822     attributes.reset_is_clear_context();
11823   }
11824   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11825   emit_int16((unsigned char)0x8A, (0xC0 | encode));
11826 }
11827 
11828 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11829   assert(VM_Version::supports_evex() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11830   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11831   attributes.set_embedded_opmask_register_specifier(mask);
11832   attributes.set_is_evex_instruction();
11833   if (merge) {
11834     attributes.reset_is_clear_context();
11835   }
11836   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11837   emit_int16((unsigned char)0x8A, (0xC0 | encode));
11838 }
11839 
11840 #ifndef _LP64
11841 
11842 void Assembler::incl(Register dst) {
11843   // Don't use it directly. Use MacroAssembler::incrementl() instead.
11844   emit_int8(0x40 | dst->encoding());
11845 }
11846 
11847 void Assembler::lea(Register dst, Address src) {
11848   leal(dst, src);
11849 }
11850 
11851 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
11852   InstructionMark im(this);
11853   emit_int8((unsigned char)0xC7);
11854   emit_operand(rax, dst);
11855   emit_data((int)imm32, rspec, 0);
11856 }
11857 
11858 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
11859   InstructionMark im(this);
11860   int encode = prefix_and_encode(dst->encoding());
11861   emit_int8((0xB8 | encode));
11862   emit_data((int)imm32, rspec, 0);
11863 }
11864 
11865 void Assembler::popa() { // 32bit
11866   emit_int8(0x61);
11867 }
11868 
11869 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
11870   InstructionMark im(this);
11871   emit_int8(0x68);
11872   emit_data(imm32, rspec, 0);
11873 }
11874 
11875 void Assembler::pusha() { // 32bit
11876   emit_int8(0x60);
11877 }
11878 
11879 void Assembler::set_byte_if_not_zero(Register dst) {
11880   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
11881 }
11882 
11883 #else // LP64
11884 
11885 void Assembler::set_byte_if_not_zero(Register dst) {
11886   int enc = prefix_and_encode(dst->encoding(), true);
11887   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
11888 }
11889 
11890 // 64bit only pieces of the assembler
11891 // This should only be used by 64bit instructions that can use rip-relative
11892 // it cannot be used by instructions that want an immediate value.
11893 
11894 bool Assembler::reachable(AddressLiteral adr) {
11895   int64_t disp;
11896   relocInfo::relocType relocType = adr.reloc();
11897 
11898   // None will force a 64bit literal to the code stream. Likely a placeholder
11899   // for something that will be patched later and we need to certain it will
11900   // always be reachable.
11901   if (relocType == relocInfo::none) {
11902     return false;
11903   }
11904   if (relocType == relocInfo::internal_word_type) {
11905     // This should be rip relative and easily reachable.
11906     return true;
11907   }
11908   if (relocType == relocInfo::virtual_call_type ||
11909       relocType == relocInfo::opt_virtual_call_type ||
11910       relocType == relocInfo::static_call_type ||
11911       relocType == relocInfo::static_stub_type ) {
11912     // This should be rip relative within the code cache and easily
11913     // reachable until we get huge code caches. (At which point
11914     // ic code is going to have issues).
11915     return true;
11916   }
11917   if (relocType != relocInfo::external_word_type &&
11918       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
11919       relocType != relocInfo::poll_type &&         // relocs to identify them
11920       relocType != relocInfo::runtime_call_type ) {
11921     return false;
11922   }
11923 
11924   // Stress the correction code
11925   if (ForceUnreachable) {
11926     // Must be runtimecall reloc, see if it is in the codecache
11927     // Flipping stuff in the codecache to be unreachable causes issues
11928     // with things like inline caches where the additional instructions
11929     // are not handled.
11930     if (CodeCache::find_blob(adr._target) == NULL) {
11931       return false;
11932     }
11933   }
11934   // For external_word_type/runtime_call_type if it is reachable from where we
11935   // are now (possibly a temp buffer) and where we might end up
11936   // anywhere in the codeCache then we are always reachable.
11937   // This would have to change if we ever save/restore shared code
11938   // to be more pessimistic.
11939   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
11940   if (!is_simm32(disp)) return false;
11941   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
11942   if (!is_simm32(disp)) return false;
11943 
11944   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
11945 
11946   // Because rip relative is a disp + address_of_next_instruction and we
11947   // don't know the value of address_of_next_instruction we apply a fudge factor
11948   // to make sure we will be ok no matter the size of the instruction we get placed into.
11949   // We don't have to fudge the checks above here because they are already worst case.
11950 
11951   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
11952   // + 4 because better safe than sorry.
11953   const int fudge = 12 + 4;
11954   if (disp < 0) {
11955     disp -= fudge;
11956   } else {
11957     disp += fudge;
11958   }
11959   return is_simm32(disp);
11960 }
11961 
11962 void Assembler::emit_data64(jlong data,
11963                             relocInfo::relocType rtype,
11964                             int format) {
11965   if (rtype == relocInfo::none) {
11966     emit_int64(data);
11967   } else {
11968     emit_data64(data, Relocation::spec_simple(rtype), format);
11969   }
11970 }
11971 
11972 void Assembler::emit_data64(jlong data,
11973                             RelocationHolder const& rspec,
11974                             int format) {
11975   assert(imm_operand == 0, "default format must be immediate in this file");
11976   assert(imm_operand == format, "must be immediate");
11977   assert(inst_mark() != NULL, "must be inside InstructionMark");
11978   // Do not use AbstractAssembler::relocate, which is not intended for
11979   // embedded words.  Instead, relocate to the enclosing instruction.
11980   code_section()->relocate(inst_mark(), rspec, format);
11981 #ifdef ASSERT
11982   check_relocation(rspec, format);
11983 #endif
11984   emit_int64(data);
11985 }
11986 
11987 void Assembler::prefix(Register reg) {
11988   if (reg->encoding() >= 8) {
11989     prefix(REX_B);
11990   }
11991 }
11992 
11993 void Assembler::prefix(Register dst, Register src, Prefix p) {
11994   if (src->encoding() >= 8) {
11995     p = (Prefix)(p | REX_B);
11996   }
11997   if (dst->encoding() >= 8) {
11998     p = (Prefix)(p | REX_R);
11999   }
12000   if (p != Prefix_EMPTY) {
12001     // do not generate an empty prefix
12002     prefix(p);
12003   }
12004 }
12005 
12006 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12007   if (adr.base_needs_rex()) {
12008     if (adr.index_needs_rex()) {
12009       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12010     } else {
12011       prefix(REX_B);
12012     }
12013   } else {
12014     if (adr.index_needs_rex()) {
12015       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12016     }
12017   }
12018   if (dst->encoding() >= 8) {
12019     p = (Prefix)(p | REX_R);
12020   }
12021   if (p != Prefix_EMPTY) {
12022     // do not generate an empty prefix
12023     prefix(p);
12024   }
12025 }
12026 
12027 void Assembler::prefix(Address adr) {
12028   if (adr.base_needs_rex()) {
12029     if (adr.index_needs_rex()) {
12030       prefix(REX_XB);
12031     } else {
12032       prefix(REX_B);
12033     }
12034   } else {
12035     if (adr.index_needs_rex()) {
12036       prefix(REX_X);
12037     }
12038   }
12039 }
12040 
12041 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12042   if (reg->encoding() < 8) {
12043     if (adr.base_needs_rex()) {
12044       if (adr.index_needs_rex()) {
12045         prefix(REX_XB);
12046       } else {
12047         prefix(REX_B);
12048       }
12049     } else {
12050       if (adr.index_needs_rex()) {
12051         prefix(REX_X);
12052       } else if (byteinst && reg->encoding() >= 4) {
12053         prefix(REX);
12054       }
12055     }
12056   } else {
12057     if (adr.base_needs_rex()) {
12058       if (adr.index_needs_rex()) {
12059         prefix(REX_RXB);
12060       } else {
12061         prefix(REX_RB);
12062       }
12063     } else {
12064       if (adr.index_needs_rex()) {
12065         prefix(REX_RX);
12066       } else {
12067         prefix(REX_R);
12068       }
12069     }
12070   }
12071 }
12072 
12073 void Assembler::prefix(Address adr, XMMRegister reg) {
12074   if (reg->encoding() < 8) {
12075     if (adr.base_needs_rex()) {
12076       if (adr.index_needs_rex()) {
12077         prefix(REX_XB);
12078       } else {
12079         prefix(REX_B);
12080       }
12081     } else {
12082       if (adr.index_needs_rex()) {
12083         prefix(REX_X);
12084       }
12085     }
12086   } else {
12087     if (adr.base_needs_rex()) {
12088       if (adr.index_needs_rex()) {
12089         prefix(REX_RXB);
12090       } else {
12091         prefix(REX_RB);
12092       }
12093     } else {
12094       if (adr.index_needs_rex()) {
12095         prefix(REX_RX);
12096       } else {
12097         prefix(REX_R);
12098       }
12099     }
12100   }
12101 }
12102 
12103 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12104   if (reg_enc >= 8) {
12105     prefix(REX_B);
12106     reg_enc -= 8;
12107   } else if (byteinst && reg_enc >= 4) {
12108     prefix(REX);
12109   }
12110   return reg_enc;
12111 }
12112 
12113 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12114   if (dst_enc < 8) {
12115     if (src_enc >= 8) {
12116       prefix(REX_B);
12117       src_enc -= 8;
12118     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12119       prefix(REX);
12120     }
12121   } else {
12122     if (src_enc < 8) {
12123       prefix(REX_R);
12124     } else {
12125       prefix(REX_RB);
12126       src_enc -= 8;
12127     }
12128     dst_enc -= 8;
12129   }
12130   return dst_enc << 3 | src_enc;
12131 }
12132 
12133 int8_t Assembler::get_prefixq(Address adr) {
12134   int8_t prfx = get_prefixq(adr, rax);
12135   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12136   return prfx;
12137 }
12138 
12139 int8_t Assembler::get_prefixq(Address adr, Register src) {
12140   int8_t prfx = (int8_t)(REX_W +
12141                          ((int)adr.base_needs_rex()) +
12142                          ((int)adr.index_needs_rex() << 1) +
12143                          ((int)(src->encoding() >= 8) << 2));
12144 #ifdef ASSERT
12145   if (src->encoding() < 8) {
12146     if (adr.base_needs_rex()) {
12147       if (adr.index_needs_rex()) {
12148         assert(prfx == REX_WXB, "must be");
12149       } else {
12150         assert(prfx == REX_WB, "must be");
12151       }
12152     } else {
12153       if (adr.index_needs_rex()) {
12154         assert(prfx == REX_WX, "must be");
12155       } else {
12156         assert(prfx == REX_W, "must be");
12157       }
12158     }
12159   } else {
12160     if (adr.base_needs_rex()) {
12161       if (adr.index_needs_rex()) {
12162         assert(prfx == REX_WRXB, "must be");
12163       } else {
12164         assert(prfx == REX_WRB, "must be");
12165       }
12166     } else {
12167       if (adr.index_needs_rex()) {
12168         assert(prfx == REX_WRX, "must be");
12169       } else {
12170         assert(prfx == REX_WR, "must be");
12171       }
12172     }
12173   }
12174 #endif
12175   return prfx;
12176 }
12177 
12178 void Assembler::prefixq(Address adr) {
12179   emit_int8(get_prefixq(adr));
12180 }
12181 
12182 void Assembler::prefixq(Address adr, Register src) {
12183   emit_int8(get_prefixq(adr, src));
12184 }
12185 
12186 void Assembler::prefixq(Address adr, XMMRegister src) {
12187   if (src->encoding() < 8) {
12188     if (adr.base_needs_rex()) {
12189       if (adr.index_needs_rex()) {
12190         prefix(REX_WXB);
12191       } else {
12192         prefix(REX_WB);
12193       }
12194     } else {
12195       if (adr.index_needs_rex()) {
12196         prefix(REX_WX);
12197       } else {
12198         prefix(REX_W);
12199       }
12200     }
12201   } else {
12202     if (adr.base_needs_rex()) {
12203       if (adr.index_needs_rex()) {
12204         prefix(REX_WRXB);
12205       } else {
12206         prefix(REX_WRB);
12207       }
12208     } else {
12209       if (adr.index_needs_rex()) {
12210         prefix(REX_WRX);
12211       } else {
12212         prefix(REX_WR);
12213       }
12214     }
12215   }
12216 }
12217 
12218 int Assembler::prefixq_and_encode(int reg_enc) {
12219   if (reg_enc < 8) {
12220     prefix(REX_W);
12221   } else {
12222     prefix(REX_WB);
12223     reg_enc -= 8;
12224   }
12225   return reg_enc;
12226 }
12227 
12228 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12229   if (dst_enc < 8) {
12230     if (src_enc < 8) {
12231       prefix(REX_W);
12232     } else {
12233       prefix(REX_WB);
12234       src_enc -= 8;
12235     }
12236   } else {
12237     if (src_enc < 8) {
12238       prefix(REX_WR);
12239     } else {
12240       prefix(REX_WRB);
12241       src_enc -= 8;
12242     }
12243     dst_enc -= 8;
12244   }
12245   return dst_enc << 3 | src_enc;
12246 }
12247 
12248 void Assembler::adcq(Register dst, int32_t imm32) {
12249   (void) prefixq_and_encode(dst->encoding());
12250   emit_arith(0x81, 0xD0, dst, imm32);
12251 }
12252 
12253 void Assembler::adcq(Register dst, Address src) {
12254   InstructionMark im(this);
12255   emit_int16(get_prefixq(src, dst), 0x13);
12256   emit_operand(dst, src);
12257 }
12258 
12259 void Assembler::adcq(Register dst, Register src) {
12260   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12261   emit_arith(0x13, 0xC0, dst, src);
12262 }
12263 
12264 void Assembler::addq(Address dst, int32_t imm32) {
12265   InstructionMark im(this);
12266   prefixq(dst);
12267   emit_arith_operand(0x81, rax, dst, imm32);
12268 }
12269 
12270 void Assembler::addq(Address dst, Register src) {
12271   InstructionMark im(this);
12272   emit_int16(get_prefixq(dst, src), 0x01);
12273   emit_operand(src, dst);
12274 }
12275 
12276 void Assembler::addq(Register dst, int32_t imm32) {
12277   (void) prefixq_and_encode(dst->encoding());
12278   emit_arith(0x81, 0xC0, dst, imm32);
12279 }
12280 
12281 void Assembler::addq(Register dst, Address src) {
12282   InstructionMark im(this);
12283   emit_int16(get_prefixq(src, dst), 0x03);
12284   emit_operand(dst, src);
12285 }
12286 
12287 void Assembler::addq(Register dst, Register src) {
12288   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12289   emit_arith(0x03, 0xC0, dst, src);
12290 }
12291 
12292 void Assembler::adcxq(Register dst, Register src) {
12293   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12294   emit_int8(0x66);
12295   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12296   emit_int32(0x0F,
12297              0x38,
12298              (unsigned char)0xF6,
12299              (0xC0 | encode));
12300 }
12301 
12302 void Assembler::adoxq(Register dst, Register src) {
12303   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12304   emit_int8((unsigned char)0xF3);
12305   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12306   emit_int32(0x0F,
12307              0x38,
12308              (unsigned char)0xF6,
12309              (0xC0 | encode));
12310 }
12311 
12312 void Assembler::andq(Address dst, int32_t imm32) {
12313   InstructionMark im(this);
12314   prefixq(dst);
12315   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12316 }
12317 
12318 void Assembler::andq(Register dst, int32_t imm32) {
12319   (void) prefixq_and_encode(dst->encoding());
12320   emit_arith(0x81, 0xE0, dst, imm32);
12321 }
12322 
12323 void Assembler::andq(Register dst, Address src) {
12324   InstructionMark im(this);
12325   emit_int16(get_prefixq(src, dst), 0x23);
12326   emit_operand(dst, src);
12327 }
12328 
12329 void Assembler::andq(Register dst, Register src) {
12330   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12331   emit_arith(0x23, 0xC0, dst, src);
12332 }
12333 
12334 void Assembler::andq(Address dst, Register src) {
12335   InstructionMark im(this);
12336   emit_int16(get_prefixq(dst, src), 0x21);
12337   emit_operand(src, dst);
12338 }
12339 
12340 void Assembler::andnq(Register dst, Register src1, Register src2) {
12341   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12342   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12343   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12344   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12345 }
12346 
12347 void Assembler::andnq(Register dst, Register src1, Address src2) {
12348   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12349   InstructionMark im(this);
12350   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12351   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12352   emit_int8((unsigned char)0xF2);
12353   emit_operand(dst, src2);
12354 }
12355 
12356 void Assembler::bsfq(Register dst, Register src) {
12357   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12358   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12359 }
12360 
12361 void Assembler::bsrq(Register dst, Register src) {
12362   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12363   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12364 }
12365 
12366 void Assembler::bswapq(Register reg) {
12367   int encode = prefixq_and_encode(reg->encoding());
12368   emit_int16(0x0F, (0xC8 | encode));
12369 }
12370 
12371 void Assembler::blsiq(Register dst, Register src) {
12372   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12373   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12374   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12375   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12376 }
12377 
12378 void Assembler::blsiq(Register dst, Address src) {
12379   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12380   InstructionMark im(this);
12381   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12382   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12383   emit_int8((unsigned char)0xF3);
12384   emit_operand(rbx, src);
12385 }
12386 
12387 void Assembler::blsmskq(Register dst, Register src) {
12388   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12389   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12390   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12391   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12392 }
12393 
12394 void Assembler::blsmskq(Register dst, Address src) {
12395   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12396   InstructionMark im(this);
12397   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12398   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12399   emit_int8((unsigned char)0xF3);
12400   emit_operand(rdx, src);
12401 }
12402 
12403 void Assembler::blsrq(Register dst, Register src) {
12404   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12405   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12406   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12407   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12408 }
12409 
12410 void Assembler::blsrq(Register dst, Address src) {
12411   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12412   InstructionMark im(this);
12413   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12414   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12415   emit_int8((unsigned char)0xF3);
12416   emit_operand(rcx, src);
12417 }
12418 
12419 void Assembler::cdqq() {
12420   emit_int16(REX_W, (unsigned char)0x99);
12421 }
12422 
12423 void Assembler::clflush(Address adr) {
12424   assert(VM_Version::supports_clflush(), "should do");
12425   prefix(adr);
12426   emit_int16(0x0F, (unsigned char)0xAE);
12427   emit_operand(rdi, adr);
12428 }
12429 
12430 void Assembler::clflushopt(Address adr) {
12431   assert(VM_Version::supports_clflushopt(), "should do!");
12432   // adr should be base reg only with no index or offset
12433   assert(adr.index() == noreg, "index should be noreg");
12434   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12435   assert(adr.disp() == 0, "displacement should be 0");
12436   // instruction prefix is 0x66
12437   emit_int8(0x66);
12438   prefix(adr);
12439   // opcode family is 0x0F 0xAE
12440   emit_int16(0x0F, (unsigned char)0xAE);
12441   // extended opcode byte is 7 == rdi
12442   emit_operand(rdi, adr);
12443 }
12444 
12445 void Assembler::clwb(Address adr) {
12446   assert(VM_Version::supports_clwb(), "should do!");
12447   // adr should be base reg only with no index or offset
12448   assert(adr.index() == noreg, "index should be noreg");
12449   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12450   assert(adr.disp() == 0, "displacement should be 0");
12451   // instruction prefix is 0x66
12452   emit_int8(0x66);
12453   prefix(adr);
12454   // opcode family is 0x0f 0xAE
12455   emit_int16(0x0F, (unsigned char)0xAE);
12456   // extended opcode byte is 6 == rsi
12457   emit_operand(rsi, adr);
12458 }
12459 
12460 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12461   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12462   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12463 }
12464 
12465 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12466   InstructionMark im(this);
12467   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12468   emit_operand(dst, src);
12469 }
12470 
12471 void Assembler::cmpq(Address dst, int32_t imm32) {
12472   InstructionMark im(this);
12473   emit_int16(get_prefixq(dst), (unsigned char)0x81);
12474   emit_operand(rdi, dst, 4);
12475   emit_int32(imm32);
12476 }
12477 
12478 void Assembler::cmpq(Register dst, int32_t imm32) {
12479   (void) prefixq_and_encode(dst->encoding());
12480   emit_arith(0x81, 0xF8, dst, imm32);
12481 }
12482 
12483 void Assembler::cmpq(Address dst, Register src) {
12484   InstructionMark im(this);
12485   emit_int16(get_prefixq(dst, src), 0x39);
12486   emit_operand(src, dst);
12487 }
12488 
12489 void Assembler::cmpq(Register dst, Register src) {
12490   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12491   emit_arith(0x3B, 0xC0, dst, src);
12492 }
12493 
12494 void Assembler::cmpq(Register dst, Address src) {
12495   InstructionMark im(this);
12496   emit_int16(get_prefixq(src, dst), 0x3B);
12497   emit_operand(dst, src);
12498 }
12499 
12500 void Assembler::cmpxchgq(Register reg, Address adr) {
12501   InstructionMark im(this);
12502   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12503   emit_operand(reg, adr);
12504 }
12505 
12506 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12507   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12508   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12509   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12510   emit_int16(0x2A, (0xC0 | encode));
12511 }
12512 
12513 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12515   InstructionMark im(this);
12516   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12517   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12518   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12519   emit_int8(0x2A);
12520   emit_operand(dst, src);
12521 }
12522 
12523 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12524   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12525   InstructionMark im(this);
12526   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12527   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12528   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12529   emit_int8(0x2A);
12530   emit_operand(dst, src);
12531 }
12532 
12533 void Assembler::cvttsd2siq(Register dst, Address src) {
12534   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12535   // F2 REX.W 0F 2C /r
12536   // CVTTSD2SI r64, xmm1/m64
12537   InstructionMark im(this);
12538   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12539   emit_operand(dst, src);
12540 }
12541 
12542 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12543   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12544   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12545   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12546   emit_int16(0x2C, (0xC0 | encode));
12547 }
12548 
12549 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12550   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12551   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12552   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12553   emit_int16(0x2D, (0xC0 | encode));
12554 }
12555 
12556 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12557   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12558   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12559   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12560   emit_int16(0x2C, (0xC0 | encode));
12561 }
12562 
12563 void Assembler::decl(Register dst) {
12564   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12565   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12566   int encode = prefix_and_encode(dst->encoding());
12567   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12568 }
12569 
12570 void Assembler::decq(Register dst) {
12571   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12572   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12573   int encode = prefixq_and_encode(dst->encoding());
12574   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12575 }
12576 
12577 void Assembler::decq(Address dst) {
12578   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12579   InstructionMark im(this);
12580   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12581   emit_operand(rcx, dst);
12582 }
12583 
12584 void Assembler::fxrstor(Address src) {
12585   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12586   emit_operand(as_Register(1), src);
12587 }
12588 
12589 void Assembler::xrstor(Address src) {
12590   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12591   emit_operand(as_Register(5), src);
12592 }
12593 
12594 void Assembler::fxsave(Address dst) {
12595   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12596   emit_operand(as_Register(0), dst);
12597 }
12598 
12599 void Assembler::xsave(Address dst) {
12600   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12601   emit_operand(as_Register(4), dst);
12602 }
12603 
12604 void Assembler::idivq(Register src) {
12605   int encode = prefixq_and_encode(src->encoding());
12606   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12607 }
12608 
12609 void Assembler::idivq(Address src) {
12610   InstructionMark im(this);
12611   prefixq(src);
12612   emit_int8((unsigned char)0xF7);
12613   emit_operand(as_Register(7), src);
12614 }
12615 
12616 void Assembler::divq(Register src) {
12617   int encode = prefixq_and_encode(src->encoding());
12618   emit_int16((unsigned char)0xF7, (0xF0 | encode));
12619 }
12620 
12621 void Assembler::imulq(Register dst, Register src) {
12622   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12623   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12624 }
12625 
12626 void Assembler::imulq(Register src) {
12627   int encode = prefixq_and_encode(src->encoding());
12628   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12629 }
12630 
12631 void Assembler::imulq(Register dst, Address src, int32_t value) {
12632   InstructionMark im(this);
12633   prefixq(src, dst);
12634   if (is8bit(value)) {
12635     emit_int8((unsigned char)0x6B);
12636     emit_operand(dst, src);
12637     emit_int8(value);
12638   } else {
12639     emit_int8((unsigned char)0x69);
12640     emit_operand(dst, src);
12641     emit_int32(value);
12642   }
12643 }
12644 
12645 void Assembler::imulq(Register dst, Register src, int value) {
12646   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12647   if (is8bit(value)) {
12648     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12649   } else {
12650     emit_int16(0x69, (0xC0 | encode));
12651     emit_int32(value);
12652   }
12653 }
12654 
12655 void Assembler::imulq(Register dst, Address src) {
12656   InstructionMark im(this);
12657   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12658   emit_operand(dst, src);
12659 }
12660 
12661 void Assembler::incl(Register dst) {
12662   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12663   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12664   int encode = prefix_and_encode(dst->encoding());
12665   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12666 }
12667 
12668 void Assembler::incq(Register dst) {
12669   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12670   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12671   int encode = prefixq_and_encode(dst->encoding());
12672   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12673 }
12674 
12675 void Assembler::incq(Address dst) {
12676   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12677   InstructionMark im(this);
12678   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12679   emit_operand(rax, dst);
12680 }
12681 
12682 void Assembler::lea(Register dst, Address src) {
12683   leaq(dst, src);
12684 }
12685 
12686 void Assembler::leaq(Register dst, Address src) {
12687   InstructionMark im(this);
12688   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
12689   emit_operand(dst, src);
12690 }
12691 
12692 void Assembler::mov64(Register dst, int64_t imm64) {
12693   InstructionMark im(this);
12694   int encode = prefixq_and_encode(dst->encoding());
12695   emit_int8(0xB8 | encode);
12696   emit_int64(imm64);
12697 }
12698 
12699 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
12700   InstructionMark im(this);
12701   int encode = prefixq_and_encode(dst->encoding());
12702   emit_int8(0xB8 | encode);
12703   emit_data64(imm64, rtype, format);
12704 }
12705 
12706 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
12707   InstructionMark im(this);
12708   int encode = prefixq_and_encode(dst->encoding());
12709   emit_int8(0xB8 | encode);
12710   emit_data64(imm64, rspec);
12711 }
12712 
12713 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12714   InstructionMark im(this);
12715   int encode = prefix_and_encode(dst->encoding());
12716   emit_int8(0xB8 | encode);
12717   emit_data((int)imm32, rspec, narrow_oop_operand);
12718 }
12719 
12720 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
12721   InstructionMark im(this);
12722   prefix(dst);
12723   emit_int8((unsigned char)0xC7);
12724   emit_operand(rax, dst, 4);
12725   emit_data((int)imm32, rspec, narrow_oop_operand);
12726 }
12727 
12728 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
12729   InstructionMark im(this);
12730   int encode = prefix_and_encode(src1->encoding());
12731   emit_int16((unsigned char)0x81, (0xF8 | encode));
12732   emit_data((int)imm32, rspec, narrow_oop_operand);
12733 }
12734 
12735 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
12736   InstructionMark im(this);
12737   prefix(src1);
12738   emit_int8((unsigned char)0x81);
12739   emit_operand(rax, src1, 4);
12740   emit_data((int)imm32, rspec, narrow_oop_operand);
12741 }
12742 
12743 void Assembler::lzcntq(Register dst, Register src) {
12744   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12745   emit_int8((unsigned char)0xF3);
12746   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12747   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12748 }
12749 
12750 void Assembler::movdq(XMMRegister dst, Register src) {
12751   // table D-1 says MMX/SSE2
12752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12753   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12754   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12755   emit_int16(0x6E, (0xC0 | encode));
12756 }
12757 
12758 void Assembler::movdq(Register dst, XMMRegister src) {
12759   // table D-1 says MMX/SSE2
12760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12761   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12762   // swap src/dst to get correct prefix
12763   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12764   emit_int16(0x7E,
12765              (0xC0 | encode));
12766 }
12767 
12768 void Assembler::movq(Register dst, Register src) {
12769   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12770   emit_int16((unsigned char)0x8B,
12771              (0xC0 | encode));
12772 }
12773 
12774 void Assembler::movq(Register dst, Address src) {
12775   InstructionMark im(this);
12776   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
12777   emit_operand(dst, src);
12778 }
12779 
12780 void Assembler::movq(Address dst, Register src) {
12781   InstructionMark im(this);
12782   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
12783   emit_operand(src, dst);
12784 }
12785 
12786 void Assembler::movq(Address dst, int32_t imm32) {
12787   InstructionMark im(this);
12788   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12789   emit_operand(as_Register(0), dst);
12790   emit_int32(imm32);
12791 }
12792 
12793 void Assembler::movq(Register dst, int32_t imm32) {
12794   int encode = prefixq_and_encode(dst->encoding());
12795   emit_int16((unsigned char)0xC7, (0xC0 | encode));
12796   emit_int32(imm32);
12797 }
12798 
12799 void Assembler::movsbq(Register dst, Address src) {
12800   InstructionMark im(this);
12801   emit_int24(get_prefixq(src, dst),
12802              0x0F,
12803              (unsigned char)0xBE);
12804   emit_operand(dst, src);
12805 }
12806 
12807 void Assembler::movsbq(Register dst, Register src) {
12808   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12809   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
12810 }
12811 
12812 void Assembler::movslq(Register dst, int32_t imm32) {
12813   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
12814   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
12815   // as a result we shouldn't use until tested at runtime...
12816   ShouldNotReachHere();
12817   InstructionMark im(this);
12818   int encode = prefixq_and_encode(dst->encoding());
12819   emit_int8(0xC7 | encode);
12820   emit_int32(imm32);
12821 }
12822 
12823 void Assembler::movslq(Address dst, int32_t imm32) {
12824   assert(is_simm32(imm32), "lost bits");
12825   InstructionMark im(this);
12826   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12827   emit_operand(rax, dst, 4);
12828   emit_int32(imm32);
12829 }
12830 
12831 void Assembler::movslq(Register dst, Address src) {
12832   InstructionMark im(this);
12833   emit_int16(get_prefixq(src, dst), 0x63);
12834   emit_operand(dst, src);
12835 }
12836 
12837 void Assembler::movslq(Register dst, Register src) {
12838   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12839   emit_int16(0x63, (0xC0 | encode));
12840 }
12841 
12842 void Assembler::movswq(Register dst, Address src) {
12843   InstructionMark im(this);
12844   emit_int24(get_prefixq(src, dst),
12845              0x0F,
12846              (unsigned char)0xBF);
12847   emit_operand(dst, src);
12848 }
12849 
12850 void Assembler::movswq(Register dst, Register src) {
12851   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12852   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
12853 }
12854 
12855 void Assembler::movzbq(Register dst, Address src) {
12856   InstructionMark im(this);
12857   emit_int24(get_prefixq(src, dst),
12858              0x0F,
12859              (unsigned char)0xB6);
12860   emit_operand(dst, src);
12861 }
12862 
12863 void Assembler::movzbq(Register dst, Register src) {
12864   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12865   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
12866 }
12867 
12868 void Assembler::movzwq(Register dst, Address src) {
12869   InstructionMark im(this);
12870   emit_int24(get_prefixq(src, dst),
12871              0x0F,
12872              (unsigned char)0xB7);
12873   emit_operand(dst, src);
12874 }
12875 
12876 void Assembler::movzwq(Register dst, Register src) {
12877   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12878   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
12879 }
12880 
12881 void Assembler::mulq(Address src) {
12882   InstructionMark im(this);
12883   emit_int16(get_prefixq(src), (unsigned char)0xF7);
12884   emit_operand(rsp, src);
12885 }
12886 
12887 void Assembler::mulq(Register src) {
12888   int encode = prefixq_and_encode(src->encoding());
12889   emit_int16((unsigned char)0xF7, (0xE0 | encode));
12890 }
12891 
12892 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
12893   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12894   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12895   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12896   emit_int16((unsigned char)0xF6, (0xC0 | encode));
12897 }
12898 
12899 void Assembler::negq(Register dst) {
12900   int encode = prefixq_and_encode(dst->encoding());
12901   emit_int16((unsigned char)0xF7, (0xD8 | encode));
12902 }
12903 
12904 void Assembler::negq(Address dst) {
12905   InstructionMark im(this);
12906   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
12907   emit_operand(as_Register(3), dst);
12908 }
12909 
12910 void Assembler::notq(Register dst) {
12911   int encode = prefixq_and_encode(dst->encoding());
12912   emit_int16((unsigned char)0xF7, (0xD0 | encode));
12913 }
12914 
12915 void Assembler::btsq(Address dst, int imm8) {
12916   assert(isByte(imm8), "not a byte");
12917   InstructionMark im(this);
12918   emit_int24(get_prefixq(dst),
12919              0x0F,
12920              (unsigned char)0xBA);
12921   emit_operand(rbp /* 5 */, dst, 1);
12922   emit_int8(imm8);
12923 }
12924 
12925 void Assembler::btrq(Address dst, int imm8) {
12926   assert(isByte(imm8), "not a byte");
12927   InstructionMark im(this);
12928   emit_int24(get_prefixq(dst),
12929              0x0F,
12930              (unsigned char)0xBA);
12931   emit_operand(rsi /* 6 */, dst, 1);
12932   emit_int8(imm8);
12933 }
12934 
12935 void Assembler::orq(Address dst, int32_t imm32) {
12936   InstructionMark im(this);
12937   prefixq(dst);
12938   emit_arith_operand(0x81, as_Register(1), dst, imm32);
12939 }
12940 
12941 void Assembler::orq(Address dst, Register src) {
12942   InstructionMark im(this);
12943   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
12944   emit_operand(src, dst);
12945 }
12946 
12947 void Assembler::orq(Register dst, int32_t imm32) {
12948   (void) prefixq_and_encode(dst->encoding());
12949   emit_arith(0x81, 0xC8, dst, imm32);
12950 }
12951 
12952 void Assembler::orq(Register dst, Address src) {
12953   InstructionMark im(this);
12954   emit_int16(get_prefixq(src, dst), 0x0B);
12955   emit_operand(dst, src);
12956 }
12957 
12958 void Assembler::orq(Register dst, Register src) {
12959   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12960   emit_arith(0x0B, 0xC0, dst, src);
12961 }
12962 
12963 void Assembler::popcntq(Register dst, Address src) {
12964   assert(VM_Version::supports_popcnt(), "must support");
12965   InstructionMark im(this);
12966   emit_int32((unsigned char)0xF3,
12967              get_prefixq(src, dst),
12968              0x0F,
12969              (unsigned char)0xB8);
12970   emit_operand(dst, src);
12971 }
12972 
12973 void Assembler::popcntq(Register dst, Register src) {
12974   assert(VM_Version::supports_popcnt(), "must support");
12975   emit_int8((unsigned char)0xF3);
12976   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12977   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
12978 }
12979 
12980 void Assembler::popq(Address dst) {
12981   InstructionMark im(this);
12982   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
12983   emit_operand(rax, dst);
12984 }
12985 
12986 void Assembler::popq(Register dst) {
12987   emit_int8((unsigned char)0x58 | dst->encoding());
12988 }
12989 
12990 // Precomputable: popa, pusha, vzeroupper
12991 
12992 // The result of these routines are invariant from one invocation to another
12993 // invocation for the duration of a run. Caching the result on bootstrap
12994 // and copying it out on subsequent invocations can thus be beneficial
12995 static bool     precomputed = false;
12996 
12997 static u_char* popa_code  = NULL;
12998 static int     popa_len   = 0;
12999 
13000 static u_char* pusha_code = NULL;
13001 static int     pusha_len  = 0;
13002 
13003 static u_char* vzup_code  = NULL;
13004 static int     vzup_len   = 0;
13005 
13006 void Assembler::precompute_instructions() {
13007   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13008   guarantee(!precomputed, "only once");
13009   precomputed = true;
13010   ResourceMark rm;
13011 
13012   // Make a temporary buffer big enough for the routines we're capturing
13013   int size = 256;
13014   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13015   CodeBuffer buffer((address)tmp_code, size);
13016   MacroAssembler masm(&buffer);
13017 
13018   address begin_popa  = masm.code_section()->end();
13019   masm.popa_uncached();
13020   address end_popa    = masm.code_section()->end();
13021   masm.pusha_uncached();
13022   address end_pusha   = masm.code_section()->end();
13023   masm.vzeroupper_uncached();
13024   address end_vzup    = masm.code_section()->end();
13025 
13026   // Save the instructions to permanent buffers.
13027   popa_len = (int)(end_popa - begin_popa);
13028   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13029   memcpy(popa_code, begin_popa, popa_len);
13030 
13031   pusha_len = (int)(end_pusha - end_popa);
13032   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13033   memcpy(pusha_code, end_popa, pusha_len);
13034 
13035   vzup_len = (int)(end_vzup - end_pusha);
13036   if (vzup_len > 0) {
13037     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13038     memcpy(vzup_code, end_pusha, vzup_len);
13039   } else {
13040     vzup_code = pusha_code; // dummy
13041   }
13042 
13043   assert(masm.code()->total_oop_size() == 0 &&
13044          masm.code()->total_metadata_size() == 0 &&
13045          masm.code()->total_relocation_size() == 0,
13046          "pre-computed code can't reference oops, metadata or contain relocations");
13047 }
13048 
13049 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13050   assert(src != NULL, "code to copy must have been pre-computed");
13051   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13052   address end = code_section->end();
13053   memcpy(end, src, src_len);
13054   code_section->set_end(end + src_len);
13055 }
13056 
13057 void Assembler::popa() { // 64bit
13058   emit_copy(code_section(), popa_code, popa_len);
13059 }
13060 
13061 void Assembler::popa_uncached() { // 64bit
13062   movq(r15, Address(rsp, 0));
13063   movq(r14, Address(rsp, wordSize));
13064   movq(r13, Address(rsp, 2 * wordSize));
13065   movq(r12, Address(rsp, 3 * wordSize));
13066   movq(r11, Address(rsp, 4 * wordSize));
13067   movq(r10, Address(rsp, 5 * wordSize));
13068   movq(r9,  Address(rsp, 6 * wordSize));
13069   movq(r8,  Address(rsp, 7 * wordSize));
13070   movq(rdi, Address(rsp, 8 * wordSize));
13071   movq(rsi, Address(rsp, 9 * wordSize));
13072   movq(rbp, Address(rsp, 10 * wordSize));
13073   // Skip rsp as it is restored automatically to the value
13074   // before the corresponding pusha when popa is done.
13075   movq(rbx, Address(rsp, 12 * wordSize));
13076   movq(rdx, Address(rsp, 13 * wordSize));
13077   movq(rcx, Address(rsp, 14 * wordSize));
13078   movq(rax, Address(rsp, 15 * wordSize));
13079 
13080   addq(rsp, 16 * wordSize);
13081 }
13082 
13083 // Does not actually store the value of rsp on the stack.
13084 // The slot for rsp just contains an arbitrary value.
13085 void Assembler::pusha() { // 64bit
13086   emit_copy(code_section(), pusha_code, pusha_len);
13087 }
13088 
13089 // Does not actually store the value of rsp on the stack.
13090 // The slot for rsp just contains an arbitrary value.
13091 void Assembler::pusha_uncached() { // 64bit
13092   subq(rsp, 16 * wordSize);
13093 
13094   movq(Address(rsp, 15 * wordSize), rax);
13095   movq(Address(rsp, 14 * wordSize), rcx);
13096   movq(Address(rsp, 13 * wordSize), rdx);
13097   movq(Address(rsp, 12 * wordSize), rbx);
13098   // Skip rsp as the value is normally not used. There are a few places where
13099   // the original value of rsp needs to be known but that can be computed
13100   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13101   movq(Address(rsp, 10 * wordSize), rbp);
13102   movq(Address(rsp, 9 * wordSize), rsi);
13103   movq(Address(rsp, 8 * wordSize), rdi);
13104   movq(Address(rsp, 7 * wordSize), r8);
13105   movq(Address(rsp, 6 * wordSize), r9);
13106   movq(Address(rsp, 5 * wordSize), r10);
13107   movq(Address(rsp, 4 * wordSize), r11);
13108   movq(Address(rsp, 3 * wordSize), r12);
13109   movq(Address(rsp, 2 * wordSize), r13);
13110   movq(Address(rsp, wordSize), r14);
13111   movq(Address(rsp, 0), r15);
13112 }
13113 
13114 void Assembler::vzeroupper() {
13115   emit_copy(code_section(), vzup_code, vzup_len);
13116 }
13117 
13118 void Assembler::pushq(Address src) {
13119   InstructionMark im(this);
13120   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13121   emit_operand(rsi, src);
13122 }
13123 
13124 void Assembler::rclq(Register dst, int imm8) {
13125   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13126   int encode = prefixq_and_encode(dst->encoding());
13127   if (imm8 == 1) {
13128     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13129   } else {
13130     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13131   }
13132 }
13133 
13134 void Assembler::rcrq(Register dst, int imm8) {
13135   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13136   int encode = prefixq_and_encode(dst->encoding());
13137   if (imm8 == 1) {
13138     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13139   } else {
13140     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13141   }
13142 }
13143 
13144 
13145 void Assembler::rorxq(Register dst, Register src, int imm8) {
13146   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13147   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13148   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13149   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13150 }
13151 
13152 void Assembler::rorxd(Register dst, Register src, int imm8) {
13153   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13154   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13155   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13156   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13157 }
13158 
13159 #ifdef _LP64
13160 void Assembler::salq(Address dst, int imm8) {
13161   InstructionMark im(this);
13162   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13163   if (imm8 == 1) {
13164     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13165     emit_operand(as_Register(4), dst);
13166   }
13167   else {
13168     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13169     emit_operand(as_Register(4), dst);
13170     emit_int8(imm8);
13171   }
13172 }
13173 
13174 void Assembler::salq(Address dst) {
13175   InstructionMark im(this);
13176   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13177   emit_operand(as_Register(4), dst);
13178 }
13179 
13180 void Assembler::salq(Register dst, int imm8) {
13181   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13182   int encode = prefixq_and_encode(dst->encoding());
13183   if (imm8 == 1) {
13184     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13185   } else {
13186     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13187   }
13188 }
13189 
13190 void Assembler::salq(Register dst) {
13191   int encode = prefixq_and_encode(dst->encoding());
13192   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13193 }
13194 
13195 void Assembler::sarq(Address dst, int imm8) {
13196   InstructionMark im(this);
13197   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13198   if (imm8 == 1) {
13199     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13200     emit_operand(as_Register(7), dst);
13201   }
13202   else {
13203     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13204     emit_operand(as_Register(7), dst);
13205     emit_int8(imm8);
13206   }
13207 }
13208 
13209 void Assembler::sarq(Address dst) {
13210   InstructionMark im(this);
13211   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13212   emit_operand(as_Register(7), dst);
13213 }
13214 
13215 void Assembler::sarq(Register dst, int imm8) {
13216   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13217   int encode = prefixq_and_encode(dst->encoding());
13218   if (imm8 == 1) {
13219     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13220   } else {
13221     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13222   }
13223 }
13224 
13225 void Assembler::sarq(Register dst) {
13226   int encode = prefixq_and_encode(dst->encoding());
13227   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13228 }
13229 #endif
13230 
13231 void Assembler::sbbq(Address dst, int32_t imm32) {
13232   InstructionMark im(this);
13233   prefixq(dst);
13234   emit_arith_operand(0x81, rbx, dst, imm32);
13235 }
13236 
13237 void Assembler::sbbq(Register dst, int32_t imm32) {
13238   (void) prefixq_and_encode(dst->encoding());
13239   emit_arith(0x81, 0xD8, dst, imm32);
13240 }
13241 
13242 void Assembler::sbbq(Register dst, Address src) {
13243   InstructionMark im(this);
13244   emit_int16(get_prefixq(src, dst), 0x1B);
13245   emit_operand(dst, src);
13246 }
13247 
13248 void Assembler::sbbq(Register dst, Register src) {
13249   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13250   emit_arith(0x1B, 0xC0, dst, src);
13251 }
13252 
13253 void Assembler::shlq(Register dst, int imm8) {
13254   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13255   int encode = prefixq_and_encode(dst->encoding());
13256   if (imm8 == 1) {
13257     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13258   } else {
13259     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13260   }
13261 }
13262 
13263 void Assembler::shlq(Register dst) {
13264   int encode = prefixq_and_encode(dst->encoding());
13265   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13266 }
13267 
13268 void Assembler::shrq(Register dst, int imm8) {
13269   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13270   int encode = prefixq_and_encode(dst->encoding());
13271   if (imm8 == 1) {
13272     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13273   }
13274   else {
13275     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13276   }
13277 }
13278 
13279 void Assembler::shrq(Register dst) {
13280   int encode = prefixq_and_encode(dst->encoding());
13281   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13282 }
13283 
13284 void Assembler::shrq(Address dst) {
13285   InstructionMark im(this);
13286   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13287   emit_operand(as_Register(5), dst);
13288 }
13289 
13290 void Assembler::shrq(Address dst, int imm8) {
13291   InstructionMark im(this);
13292   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13293   if (imm8 == 1) {
13294     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13295     emit_operand(as_Register(5), dst);
13296   }
13297   else {
13298     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13299     emit_operand(as_Register(5), dst);
13300     emit_int8(imm8);
13301   }
13302 }
13303 
13304 void Assembler::subq(Address dst, int32_t imm32) {
13305   InstructionMark im(this);
13306   prefixq(dst);
13307   emit_arith_operand(0x81, rbp, dst, imm32);
13308 }
13309 
13310 void Assembler::subq(Address dst, Register src) {
13311   InstructionMark im(this);
13312   emit_int16(get_prefixq(dst, src), 0x29);
13313   emit_operand(src, dst);
13314 }
13315 
13316 void Assembler::subq(Register dst, int32_t imm32) {
13317   (void) prefixq_and_encode(dst->encoding());
13318   emit_arith(0x81, 0xE8, dst, imm32);
13319 }
13320 
13321 // Force generation of a 4 byte immediate value even if it fits into 8bit
13322 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13323   (void) prefixq_and_encode(dst->encoding());
13324   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13325 }
13326 
13327 void Assembler::subq(Register dst, Address src) {
13328   InstructionMark im(this);
13329   emit_int16(get_prefixq(src, dst), 0x2B);
13330   emit_operand(dst, src);
13331 }
13332 
13333 void Assembler::subq(Register dst, Register src) {
13334   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13335   emit_arith(0x2B, 0xC0, dst, src);
13336 }
13337 
13338 void Assembler::testq(Address dst, int32_t imm32) {
13339   if (imm32 >= 0) {
13340     testl(dst, imm32);
13341     return;
13342   }
13343   InstructionMark im(this);
13344   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13345   emit_operand(as_Register(0), dst);
13346   emit_int32(imm32);
13347 }
13348 
13349 void Assembler::testq(Register dst, int32_t imm32) {
13350   if (imm32 >= 0) {
13351     testl(dst, imm32);
13352     return;
13353   }
13354   // not using emit_arith because test
13355   // doesn't support sign-extension of
13356   // 8bit operands
13357   if (dst == rax) {
13358     prefix(REX_W);
13359     emit_int8((unsigned char)0xA9);
13360     emit_int32(imm32);
13361   } else {
13362     int encode = dst->encoding();
13363     encode = prefixq_and_encode(encode);
13364     emit_int16((unsigned char)0xF7, (0xC0 | encode));
13365     emit_int32(imm32);
13366   }
13367 }
13368 
13369 void Assembler::testq(Register dst, Register src) {
13370   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13371   emit_arith(0x85, 0xC0, dst, src);
13372 }
13373 
13374 void Assembler::testq(Register dst, Address src) {
13375   InstructionMark im(this);
13376   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13377   emit_operand(dst, src);
13378 }
13379 
13380 void Assembler::xaddq(Address dst, Register src) {
13381   InstructionMark im(this);
13382   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13383   emit_operand(src, dst);
13384 }
13385 
13386 void Assembler::xchgq(Register dst, Address src) {
13387   InstructionMark im(this);
13388   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13389   emit_operand(dst, src);
13390 }
13391 
13392 void Assembler::xchgq(Register dst, Register src) {
13393   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13394   emit_int16((unsigned char)0x87, (0xc0 | encode));
13395 }
13396 
13397 void Assembler::xorq(Register dst, Register src) {
13398   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13399   emit_arith(0x33, 0xC0, dst, src);
13400 }
13401 
13402 void Assembler::xorq(Register dst, Address src) {
13403   InstructionMark im(this);
13404   emit_int16(get_prefixq(src, dst), 0x33);
13405   emit_operand(dst, src);
13406 }
13407 
13408 void Assembler::xorq(Register dst, int32_t imm32) {
13409   (void) prefixq_and_encode(dst->encoding());
13410   emit_arith(0x81, 0xF0, dst, imm32);
13411 }
13412 
13413 void Assembler::xorq(Address dst, int32_t imm32) {
13414   InstructionMark im(this);
13415   prefixq(dst);
13416   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13417 }
13418 
13419 void Assembler::xorq(Address dst, Register src) {
13420   InstructionMark im(this);
13421   emit_int16(get_prefixq(dst, src), 0x31);
13422   emit_operand(src, dst);
13423 }
13424 
13425 #endif // !LP64
13426 
13427 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13428   if (VM_Version::supports_evex()) {
13429     _tuple_type = tuple_type;
13430     _input_size_in_bits = input_size_in_bits;
13431   }
13432 }