1 /*
    2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec = RelocationHolder::none;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert(op1 == 0x81, "Unexpected opcode");
  304   if (is8bit(imm32)) {
  305     emit_int24(op1 | 0x02,        // set sign bit
  306                op2 | encode(dst),
  307                imm32 & 0xFF);
  308   } else if (dst == rax) {
  309     switch (op2) {
  310       case 0xD0: emit_int8(0x15); break; // adc
  311       case 0xC0: emit_int8(0x05); break; // add
  312       case 0xE0: emit_int8(0x25); break; // and
  313       case 0xF8: emit_int8(0x3D); break; // cmp
  314       case 0xC8: emit_int8(0x0D); break; // or
  315       case 0xD8: emit_int8(0x1D); break; // sbb
  316       case 0xE8: emit_int8(0x2D); break; // sub
  317       case 0xF0: emit_int8(0x35); break; // xor
  318       default: ShouldNotReachHere();
  319     }
  320     emit_int32(imm32);
  321   } else {
  322     emit_int16(op1, (op2 | encode(dst)));
  323     emit_int32(imm32);
  324   }
  325 }
  326 
  327 // Force generation of a 4 byte immediate value even if it fits into 8bit
  328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  329   assert(isByte(op1) && isByte(op2), "wrong opcode");
  330   assert((op1 & 0x01) == 1, "should be 32bit operation");
  331   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  332   emit_int16(op1, (op2 | encode(dst)));
  333   emit_int32(imm32);
  334 }
  335 
  336 // immediate-to-memory forms
  337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  338   assert((op1 & 0x01) == 1, "should be 32bit operation");
  339   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  340   if (is8bit(imm32)) {
  341     emit_int8(op1 | 0x02); // set sign bit
  342     emit_operand(rm, adr, 1);
  343     emit_int8(imm32 & 0xFF);
  344   } else {
  345     emit_int8(op1);
  346     emit_operand(rm, adr, 4);
  347     emit_int32(imm32);
  348   }
  349 }
  350 
  351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
  352   assert(op1 == 0x81, "unexpected opcode");
  353   emit_int8(op1);
  354   emit_operand(rm, adr, 4);
  355   emit_int32(imm32);
  356 }
  357 
  358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  359   assert(isByte(op1) && isByte(op2), "wrong opcode");
  360   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  361 }
  362 
  363 
  364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  365                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  366   int mod_idx = 0;
  367   // We will test if the displacement fits the compressed format and if so
  368   // apply the compression to the displacement iff the result is8bit.
  369   if (VM_Version::supports_evex() && is_evex_inst) {
  370     switch (cur_tuple_type) {
  371     case EVEX_FV:
  372       if ((cur_encoding & VEX_W) == VEX_W) {
  373         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  374       } else {
  375         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  376       }
  377       break;
  378 
  379     case EVEX_HV:
  380       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  381       break;
  382 
  383     case EVEX_FVM:
  384       break;
  385 
  386     case EVEX_T1S:
  387       switch (in_size_in_bits) {
  388       case EVEX_8bit:
  389         break;
  390 
  391       case EVEX_16bit:
  392         mod_idx = 1;
  393         break;
  394 
  395       case EVEX_32bit:
  396         mod_idx = 2;
  397         break;
  398 
  399       case EVEX_64bit:
  400         mod_idx = 3;
  401         break;
  402       }
  403       break;
  404 
  405     case EVEX_T1F:
  406     case EVEX_T2:
  407     case EVEX_T4:
  408       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  409       break;
  410 
  411     case EVEX_T8:
  412       break;
  413 
  414     case EVEX_HVM:
  415       break;
  416 
  417     case EVEX_QVM:
  418       break;
  419 
  420     case EVEX_OVM:
  421       break;
  422 
  423     case EVEX_M128:
  424       break;
  425 
  426     case EVEX_DUP:
  427       break;
  428 
  429     default:
  430       assert(0, "no valid evex tuple_table entry");
  431       break;
  432     }
  433 
  434     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  435       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  436       if ((disp % disp_factor) == 0) {
  437         int new_disp = disp / disp_factor;
  438         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  439           disp = new_disp;
  440         }
  441       } else {
  442         return false;
  443       }
  444     }
  445   }
  446   return (-0x80 <= disp && disp < 0x80);
  447 }
  448 
  449 
  450 bool Assembler::emit_compressed_disp_byte(int &disp) {
  451   int mod_idx = 0;
  452   // We will test if the displacement fits the compressed format and if so
  453   // apply the compression to the displacement iff the result is8bit.
  454   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  455     int evex_encoding = _attributes->get_evex_encoding();
  456     int tuple_type = _attributes->get_tuple_type();
  457     switch (tuple_type) {
  458     case EVEX_FV:
  459       if ((evex_encoding & VEX_W) == VEX_W) {
  460         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  461       } else {
  462         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  463       }
  464       break;
  465 
  466     case EVEX_HV:
  467       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  468       break;
  469 
  470     case EVEX_FVM:
  471       break;
  472 
  473     case EVEX_T1S:
  474       switch (_attributes->get_input_size()) {
  475       case EVEX_8bit:
  476         break;
  477 
  478       case EVEX_16bit:
  479         mod_idx = 1;
  480         break;
  481 
  482       case EVEX_32bit:
  483         mod_idx = 2;
  484         break;
  485 
  486       case EVEX_64bit:
  487         mod_idx = 3;
  488         break;
  489       }
  490       break;
  491 
  492     case EVEX_T1F:
  493     case EVEX_T2:
  494     case EVEX_T4:
  495       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  496       break;
  497 
  498     case EVEX_T8:
  499       break;
  500 
  501     case EVEX_HVM:
  502       break;
  503 
  504     case EVEX_QVM:
  505       break;
  506 
  507     case EVEX_OVM:
  508       break;
  509 
  510     case EVEX_M128:
  511       break;
  512 
  513     case EVEX_DUP:
  514       break;
  515 
  516     default:
  517       assert(0, "no valid evex tuple_table entry");
  518       break;
  519     }
  520 
  521     int vector_len = _attributes->get_vector_len();
  522     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  523       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  524       if ((disp % disp_factor) == 0) {
  525         int new_disp = disp / disp_factor;
  526         if (is8bit(new_disp)) {
  527           disp = new_disp;
  528         }
  529       } else {
  530         return false;
  531       }
  532     }
  533   }
  534   return is8bit(disp);
  535 }
  536 
  537 static bool is_valid_encoding(int reg_enc) {
  538   return reg_enc >= 0;
  539 }
  540 
  541 static int raw_encode(Register reg) {
  542   assert(reg == noreg || reg->is_valid(), "sanity");
  543   int reg_enc = reg->raw_encoding();
  544   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  545   return reg_enc;
  546 }
  547 
  548 static int raw_encode(XMMRegister xmmreg) {
  549   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  550   int xmmreg_enc = xmmreg->raw_encoding();
  551   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  552   return xmmreg_enc;
  553 }
  554 
  555 static int raw_encode(KRegister kreg) {
  556   assert(kreg == knoreg || kreg->is_valid(), "sanity");
  557   int kreg_enc = kreg->raw_encoding();
  558   assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
  559   return kreg_enc;
  560 }
  561 
  562 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  563   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  564 }
  565 
  566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  567   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  568 }
  569 
  570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  571   assert((mod & 3) != 0b11, "forbidden");
  572   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  573   emit_int8(modrm);
  574 }
  575 
  576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  577                                         int disp) {
  578   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  579   emit_int16(modrm, disp & 0xFF);
  580 }
  581 
  582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  583                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  584   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  585   int sib = sib_encoding(scale, index_enc, base_enc);
  586   emit_int16(modrm, sib);
  587 }
  588 
  589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  590                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  591                                             int disp) {
  592   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  593   int sib = sib_encoding(scale, index_enc, base_enc);
  594   emit_int24(modrm, sib, disp & 0xFF);
  595 }
  596 
  597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  598                                     Address::ScaleFactor scale, int disp,
  599                                     RelocationHolder const& rspec,
  600                                     int post_addr_length) {
  601   bool no_relocation = (rspec.type() == relocInfo::none);
  602 
  603   if (is_valid_encoding(base_enc)) {
  604     if (is_valid_encoding(index_enc)) {
  605       assert(scale != Address::no_scale, "inconsistent address");
  606       // [base + index*scale + disp]
  607       if (disp == 0 && no_relocation &&
  608           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  609         // [base + index*scale]
  610         // [00 reg 100][ss index base]
  611         emit_modrm_sib(0b00, reg_enc, 0b100,
  612                        scale, index_enc, base_enc);
  613       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  614         // [base + index*scale + imm8]
  615         // [01 reg 100][ss index base] imm8
  616         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  617                              scale, index_enc, base_enc,
  618                              disp);
  619       } else {
  620         // [base + index*scale + disp32]
  621         // [10 reg 100][ss index base] disp32
  622         emit_modrm_sib(0b10, reg_enc, 0b100,
  623                        scale, index_enc, base_enc);
  624         emit_data(disp, rspec, disp32_operand);
  625       }
  626     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  627       // [rsp + disp]
  628       if (disp == 0 && no_relocation) {
  629         // [rsp]
  630         // [00 reg 100][00 100 100]
  631         emit_modrm_sib(0b00, reg_enc, 0b100,
  632                        Address::times_1, 0b100, 0b100);
  633       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  634         // [rsp + imm8]
  635         // [01 reg 100][00 100 100] disp8
  636         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  637                              Address::times_1, 0b100, 0b100,
  638                              disp);
  639       } else {
  640         // [rsp + imm32]
  641         // [10 reg 100][00 100 100] disp32
  642         emit_modrm_sib(0b10, reg_enc, 0b100,
  643                        Address::times_1, 0b100, 0b100);
  644         emit_data(disp, rspec, disp32_operand);
  645       }
  646     } else {
  647       // [base + disp]
  648       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  649       if (disp == 0 && no_relocation &&
  650           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  651         // [base]
  652         // [00 reg base]
  653         emit_modrm(0, reg_enc, base_enc);
  654       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  655         // [base + disp8]
  656         // [01 reg base] disp8
  657         emit_modrm_disp8(0b01, reg_enc, base_enc,
  658                          disp);
  659       } else {
  660         // [base + disp32]
  661         // [10 reg base] disp32
  662         emit_modrm(0b10, reg_enc, base_enc);
  663         emit_data(disp, rspec, disp32_operand);
  664       }
  665     }
  666   } else {
  667     if (is_valid_encoding(index_enc)) {
  668       assert(scale != Address::no_scale, "inconsistent address");
  669       // base == noreg
  670       // [index*scale + disp]
  671       // [00 reg 100][ss index 101] disp32
  672       emit_modrm_sib(0b00, reg_enc, 0b100,
  673                      scale, index_enc, 0b101 /* no base */);
  674       emit_data(disp, rspec, disp32_operand);
  675     } else if (!no_relocation) {
  676       // base == noreg, index == noreg
  677       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  678       // [00 reg 101] disp32
  679 
  680       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  681       // Note that the RIP-rel. correction applies to the generated
  682       // disp field, but _not_ to the target address in the rspec.
  683 
  684       // disp was created by converting the target address minus the pc
  685       // at the start of the instruction. That needs more correction here.
  686       // intptr_t disp = target - next_ip;
  687       assert(inst_mark() != NULL, "must be inside InstructionMark");
  688       address next_ip = pc() + sizeof(int32_t) + post_addr_length;
  689       int64_t adjusted = disp;
  690       // Do rip-rel adjustment for 64bit
  691       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  692       assert(is_simm32(adjusted),
  693              "must be 32bit offset (RIP relative address)");
  694       emit_data((int32_t) adjusted, rspec, disp32_operand);
  695 
  696     } else {
  697       // base == noreg, index == noreg, no_relocation == true
  698       // 32bit never did this, did everything as the rip-rel/disp code above
  699       // [disp] ABSOLUTE
  700       // [00 reg 100][00 100 101] disp32
  701       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  702                      Address::times_1, 0b100, 0b101);
  703       emit_data(disp, rspec, disp32_operand);
  704     }
  705   }
  706 }
  707 
  708 void Assembler::emit_operand(Register reg, Register base, Register index,
  709                              Address::ScaleFactor scale, int disp,
  710                              RelocationHolder const& rspec,
  711                              int post_addr_length) {
  712   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  713   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  714                       scale, disp, rspec, post_addr_length);
  715 
  716 }
  717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  718                              Address::ScaleFactor scale, int disp,
  719                              RelocationHolder const& rspec,
  720                              int post_addr_length) {
  721   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  722   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  723   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  724                       scale, disp, rspec, post_addr_length);
  725 }
  726 
  727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  728                              Address::ScaleFactor scale, int disp,
  729                              RelocationHolder const& rspec,
  730                              int post_addr_length) {
  731   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  732   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  733   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  734                       scale, disp, rspec, post_addr_length);
  735 }
  736 
  737 void Assembler::emit_operand(KRegister kreg, Address adr,
  738                              int post_addr_length) {
  739   emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
  740                adr._rspec,
  741                post_addr_length);
  742 }
  743 
  744 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
  745                              Address::ScaleFactor scale, int disp,
  746                              RelocationHolder const& rspec,
  747                              int post_addr_length) {
  748   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  749   emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
  750                       scale, disp, rspec, post_addr_length);
  751 }
  752 
  753 // Secret local extension to Assembler::WhichOperand:
  754 #define end_pc_operand (_WhichOperand_limit)
  755 
  756 address Assembler::locate_operand(address inst, WhichOperand which) {
  757   // Decode the given instruction, and return the address of
  758   // an embedded 32-bit operand word.
  759 
  760   // If "which" is disp32_operand, selects the displacement portion
  761   // of an effective address specifier.
  762   // If "which" is imm64_operand, selects the trailing immediate constant.
  763   // If "which" is call32_operand, selects the displacement of a call or jump.
  764   // Caller is responsible for ensuring that there is such an operand,
  765   // and that it is 32/64 bits wide.
  766 
  767   // If "which" is end_pc_operand, find the end of the instruction.
  768 
  769   address ip = inst;
  770   bool is_64bit = false;
  771 
  772   debug_only(bool has_disp32 = false);
  773   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  774 
  775   again_after_prefix:
  776   switch (0xFF & *ip++) {
  777 
  778   // These convenience macros generate groups of "case" labels for the switch.
  779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  781              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  782 #define REP16(x) REP8((x)+0): \
  783               case REP8((x)+8)
  784 
  785   case CS_segment:
  786   case SS_segment:
  787   case DS_segment:
  788   case ES_segment:
  789   case FS_segment:
  790   case GS_segment:
  791     // Seems dubious
  792     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  793     assert(ip == inst+1, "only one prefix allowed");
  794     goto again_after_prefix;
  795 
  796   case 0x67:
  797   case REX:
  798   case REX_B:
  799   case REX_X:
  800   case REX_XB:
  801   case REX_R:
  802   case REX_RB:
  803   case REX_RX:
  804   case REX_RXB:
  805     NOT_LP64(assert(false, "64bit prefixes"));
  806     goto again_after_prefix;
  807 
  808   case REX_W:
  809   case REX_WB:
  810   case REX_WX:
  811   case REX_WXB:
  812   case REX_WR:
  813   case REX_WRB:
  814   case REX_WRX:
  815   case REX_WRXB:
  816     NOT_LP64(assert(false, "64bit prefixes"));
  817     is_64bit = true;
  818     goto again_after_prefix;
  819 
  820   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  821   case 0x88: // movb a, r
  822   case 0x89: // movl a, r
  823   case 0x8A: // movb r, a
  824   case 0x8B: // movl r, a
  825   case 0x8F: // popl a
  826     debug_only(has_disp32 = true);
  827     break;
  828 
  829   case 0x68: // pushq #32
  830     if (which == end_pc_operand) {
  831       return ip + 4;
  832     }
  833     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  834     return ip;                  // not produced by emit_operand
  835 
  836   case 0x66: // movw ... (size prefix)
  837     again_after_size_prefix2:
  838     switch (0xFF & *ip++) {
  839     case REX:
  840     case REX_B:
  841     case REX_X:
  842     case REX_XB:
  843     case REX_R:
  844     case REX_RB:
  845     case REX_RX:
  846     case REX_RXB:
  847     case REX_W:
  848     case REX_WB:
  849     case REX_WX:
  850     case REX_WXB:
  851     case REX_WR:
  852     case REX_WRB:
  853     case REX_WRX:
  854     case REX_WRXB:
  855       NOT_LP64(assert(false, "64bit prefix found"));
  856       goto again_after_size_prefix2;
  857     case 0x8B: // movw r, a
  858     case 0x89: // movw a, r
  859       debug_only(has_disp32 = true);
  860       break;
  861     case 0xC7: // movw a, #16
  862       debug_only(has_disp32 = true);
  863       tail_size = 2;  // the imm16
  864       break;
  865     case 0x0F: // several SSE/SSE2 variants
  866       ip--;    // reparse the 0x0F
  867       goto again_after_prefix;
  868     default:
  869       ShouldNotReachHere();
  870     }
  871     break;
  872 
  873   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  874     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  875     // these asserts are somewhat nonsensical
  876 #ifndef _LP64
  877     assert(which == imm_operand || which == disp32_operand,
  878            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  879 #else
  880     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  881            which == narrow_oop_operand && !is_64bit,
  882            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  883 #endif // _LP64
  884     return ip;
  885 
  886   case 0x69: // imul r, a, #32
  887   case 0xC7: // movl a, #32(oop?)
  888     tail_size = 4;
  889     debug_only(has_disp32 = true); // has both kinds of operands!
  890     break;
  891 
  892   case 0x0F: // movx..., etc.
  893     switch (0xFF & *ip++) {
  894     case 0x3A: // pcmpestri
  895       tail_size = 1;
  896     case 0x38: // ptest, pmovzxbw
  897       ip++; // skip opcode
  898       debug_only(has_disp32 = true); // has both kinds of operands!
  899       break;
  900 
  901     case 0x70: // pshufd r, r/a, #8
  902       debug_only(has_disp32 = true); // has both kinds of operands!
  903     case 0x73: // psrldq r, #8
  904       tail_size = 1;
  905       break;
  906 
  907     case 0x10: // movups
  908     case 0x11: // movups
  909     case 0x12: // movlps
  910     case 0x28: // movaps
  911     case 0x2E: // ucomiss
  912     case 0x2F: // comiss
  913     case 0x54: // andps
  914     case 0x55: // andnps
  915     case 0x56: // orps
  916     case 0x57: // xorps
  917     case 0x58: // addpd
  918     case 0x59: // mulpd
  919     case 0x6E: // movd
  920     case 0x7E: // movd
  921     case 0x6F: // movdq
  922     case 0x7F: // movdq
  923     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  924     case 0xFE: // paddd
  925       debug_only(has_disp32 = true);
  926       break;
  927 
  928     case 0xAD: // shrd r, a, %cl
  929     case 0xAF: // imul r, a
  930     case 0xBE: // movsbl r, a (movsxb)
  931     case 0xBF: // movswl r, a (movsxw)
  932     case 0xB6: // movzbl r, a (movzxb)
  933     case 0xB7: // movzwl r, a (movzxw)
  934     case REP16(0x40): // cmovl cc, r, a
  935     case 0xB0: // cmpxchgb
  936     case 0xB1: // cmpxchg
  937     case 0xC1: // xaddl
  938     case 0xC7: // cmpxchg8
  939     case REP16(0x90): // setcc a
  940       debug_only(has_disp32 = true);
  941       // fall out of the switch to decode the address
  942       break;
  943 
  944     case 0xC4: // pinsrw r, a, #8
  945       debug_only(has_disp32 = true);
  946     case 0xC5: // pextrw r, r, #8
  947       tail_size = 1;  // the imm8
  948       break;
  949 
  950     case 0xAC: // shrd r, a, #8
  951       debug_only(has_disp32 = true);
  952       tail_size = 1;  // the imm8
  953       break;
  954 
  955     case REP16(0x80): // jcc rdisp32
  956       if (which == end_pc_operand)  return ip + 4;
  957       assert(which == call32_operand, "jcc has no disp32 or imm");
  958       return ip;
  959     default:
  960       ShouldNotReachHere();
  961     }
  962     break;
  963 
  964   case 0x81: // addl a, #32; addl r, #32
  965     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  966     // on 32bit in the case of cmpl, the imm might be an oop
  967     tail_size = 4;
  968     debug_only(has_disp32 = true); // has both kinds of operands!
  969     break;
  970 
  971   case 0x83: // addl a, #8; addl r, #8
  972     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  973     debug_only(has_disp32 = true); // has both kinds of operands!
  974     tail_size = 1;
  975     break;
  976 
  977   case 0x15: // adc rax, #32
  978   case 0x05: // add rax, #32
  979   case 0x25: // and rax, #32
  980   case 0x3D: // cmp rax, #32
  981   case 0x0D: // or  rax, #32
  982   case 0x1D: // sbb rax, #32
  983   case 0x2D: // sub rax, #32
  984   case 0x35: // xor rax, #32
  985     return which == end_pc_operand ? ip + 4 : ip;
  986 
  987   case 0x9B:
  988     switch (0xFF & *ip++) {
  989     case 0xD9: // fnstcw a
  990       debug_only(has_disp32 = true);
  991       break;
  992     default:
  993       ShouldNotReachHere();
  994     }
  995     break;
  996 
  997   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  998   case REP4(0x10): // adc...
  999   case REP4(0x20): // and...
 1000   case REP4(0x30): // xor...
 1001   case REP4(0x08): // or...
 1002   case REP4(0x18): // sbb...
 1003   case REP4(0x28): // sub...
 1004   case 0xF7: // mull a
 1005   case 0x8D: // lea r, a
 1006   case 0x87: // xchg r, a
 1007   case REP4(0x38): // cmp...
 1008   case 0x85: // test r, a
 1009     debug_only(has_disp32 = true); // has both kinds of operands!
 1010     break;
 1011 
 1012   case 0xA8: // testb rax, #8
 1013     return which == end_pc_operand ? ip + 1 : ip;
 1014   case 0xA9: // testl/testq rax, #32
 1015     return which == end_pc_operand ? ip + 4 : ip;
 1016 
 1017   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 1018   case 0xC6: // movb a, #8
 1019   case 0x80: // cmpb a, #8
 1020   case 0x6B: // imul r, a, #8
 1021     debug_only(has_disp32 = true); // has both kinds of operands!
 1022     tail_size = 1; // the imm8
 1023     break;
 1024 
 1025   case 0xC4: // VEX_3bytes
 1026   case 0xC5: // VEX_2bytes
 1027     assert((UseAVX > 0), "shouldn't have VEX prefix");
 1028     assert(ip == inst+1, "no prefixes allowed");
 1029     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 1030     // but they have prefix 0x0F and processed when 0x0F processed above.
 1031     //
 1032     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1033     // instructions (these instructions are not supported in 64-bit mode).
 1034     // To distinguish them bits [7:6] are set in the VEX second byte since
 1035     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1036     // those VEX bits REX and vvvv bits are inverted.
 1037     //
 1038     // Fortunately C2 doesn't generate these instructions so we don't need
 1039     // to check for them in product version.
 1040 
 1041     // Check second byte
 1042     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1043 
 1044     int vex_opcode;
 1045     // First byte
 1046     if ((0xFF & *inst) == VEX_3bytes) {
 1047       vex_opcode = VEX_OPCODE_MASK & *ip;
 1048       ip++; // third byte
 1049       is_64bit = ((VEX_W & *ip) == VEX_W);
 1050     } else {
 1051       vex_opcode = VEX_OPCODE_0F;
 1052     }
 1053     ip++; // opcode
 1054     // To find the end of instruction (which == end_pc_operand).
 1055     switch (vex_opcode) {
 1056       case VEX_OPCODE_0F:
 1057         switch (0xFF & *ip) {
 1058         case 0x70: // pshufd r, r/a, #8
 1059         case 0x71: // ps[rl|ra|ll]w r, #8
 1060         case 0x72: // ps[rl|ra|ll]d r, #8
 1061         case 0x73: // ps[rl|ra|ll]q r, #8
 1062         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1063         case 0xC4: // pinsrw r, r, r/a, #8
 1064         case 0xC5: // pextrw r/a, r, #8
 1065         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1066           tail_size = 1;  // the imm8
 1067           break;
 1068         }
 1069         break;
 1070       case VEX_OPCODE_0F_3A:
 1071         tail_size = 1;
 1072         break;
 1073     }
 1074     ip++; // skip opcode
 1075     debug_only(has_disp32 = true); // has both kinds of operands!
 1076     break;
 1077 
 1078   case 0x62: // EVEX_4bytes
 1079     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1080     assert(ip == inst+1, "no prefixes allowed");
 1081     // no EVEX collisions, all instructions that have 0x62 opcodes
 1082     // have EVEX versions and are subopcodes of 0x66
 1083     ip++; // skip P0 and examine W in P1
 1084     is_64bit = ((VEX_W & *ip) == VEX_W);
 1085     ip++; // move to P2
 1086     ip++; // skip P2, move to opcode
 1087     // To find the end of instruction (which == end_pc_operand).
 1088     switch (0xFF & *ip) {
 1089     case 0x22: // pinsrd r, r/a, #8
 1090     case 0x61: // pcmpestri r, r/a, #8
 1091     case 0x70: // pshufd r, r/a, #8
 1092     case 0x73: // psrldq r, #8
 1093     case 0x1f: // evpcmpd/evpcmpq
 1094     case 0x3f: // evpcmpb/evpcmpw
 1095       tail_size = 1;  // the imm8
 1096       break;
 1097     default:
 1098       break;
 1099     }
 1100     ip++; // skip opcode
 1101     debug_only(has_disp32 = true); // has both kinds of operands!
 1102     break;
 1103 
 1104   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1105   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1106   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1107   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1108   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1109   case 0xDF: // fild_d a; fistp_d a
 1110   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1111   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1112   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1113     debug_only(has_disp32 = true);
 1114     break;
 1115 
 1116   case 0xE8: // call rdisp32
 1117   case 0xE9: // jmp  rdisp32
 1118     if (which == end_pc_operand)  return ip + 4;
 1119     assert(which == call32_operand, "call has no disp32 or imm");
 1120     return ip;
 1121 
 1122   case 0xF0:                    // Lock
 1123     goto again_after_prefix;
 1124 
 1125   case 0xF3:                    // For SSE
 1126   case 0xF2:                    // For SSE2
 1127     switch (0xFF & *ip++) {
 1128     case REX:
 1129     case REX_B:
 1130     case REX_X:
 1131     case REX_XB:
 1132     case REX_R:
 1133     case REX_RB:
 1134     case REX_RX:
 1135     case REX_RXB:
 1136     case REX_W:
 1137     case REX_WB:
 1138     case REX_WX:
 1139     case REX_WXB:
 1140     case REX_WR:
 1141     case REX_WRB:
 1142     case REX_WRX:
 1143     case REX_WRXB:
 1144       NOT_LP64(assert(false, "found 64bit prefix"));
 1145       ip++;
 1146     default:
 1147       ip++;
 1148     }
 1149     debug_only(has_disp32 = true); // has both kinds of operands!
 1150     break;
 1151 
 1152   default:
 1153     ShouldNotReachHere();
 1154 
 1155 #undef REP8
 1156 #undef REP16
 1157   }
 1158 
 1159   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1160 #ifdef _LP64
 1161   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1162 #else
 1163   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1164   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1165 #endif // LP64
 1166   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1167 
 1168   // parse the output of emit_operand
 1169   int op2 = 0xFF & *ip++;
 1170   int base = op2 & 0x07;
 1171   int op3 = -1;
 1172   const int b100 = 4;
 1173   const int b101 = 5;
 1174   if (base == b100 && (op2 >> 6) != 3) {
 1175     op3 = 0xFF & *ip++;
 1176     base = op3 & 0x07;   // refetch the base
 1177   }
 1178   // now ip points at the disp (if any)
 1179 
 1180   switch (op2 >> 6) {
 1181   case 0:
 1182     // [00 reg  100][ss index base]
 1183     // [00 reg  100][00   100  esp]
 1184     // [00 reg base]
 1185     // [00 reg  100][ss index  101][disp32]
 1186     // [00 reg  101]               [disp32]
 1187 
 1188     if (base == b101) {
 1189       if (which == disp32_operand)
 1190         return ip;              // caller wants the disp32
 1191       ip += 4;                  // skip the disp32
 1192     }
 1193     break;
 1194 
 1195   case 1:
 1196     // [01 reg  100][ss index base][disp8]
 1197     // [01 reg  100][00   100  esp][disp8]
 1198     // [01 reg base]               [disp8]
 1199     ip += 1;                    // skip the disp8
 1200     break;
 1201 
 1202   case 2:
 1203     // [10 reg  100][ss index base][disp32]
 1204     // [10 reg  100][00   100  esp][disp32]
 1205     // [10 reg base]               [disp32]
 1206     if (which == disp32_operand)
 1207       return ip;                // caller wants the disp32
 1208     ip += 4;                    // skip the disp32
 1209     break;
 1210 
 1211   case 3:
 1212     // [11 reg base]  (not a memory addressing mode)
 1213     break;
 1214   }
 1215 
 1216   if (which == end_pc_operand) {
 1217     return ip + tail_size;
 1218   }
 1219 
 1220 #ifdef _LP64
 1221   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1222 #else
 1223   assert(which == imm_operand, "instruction has only an imm field");
 1224 #endif // LP64
 1225   return ip;
 1226 }
 1227 
 1228 address Assembler::locate_next_instruction(address inst) {
 1229   // Secretly share code with locate_operand:
 1230   return locate_operand(inst, end_pc_operand);
 1231 }
 1232 
 1233 
 1234 #ifdef ASSERT
 1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1236   address inst = inst_mark();
 1237   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1238   address opnd;
 1239 
 1240   Relocation* r = rspec.reloc();
 1241   if (r->type() == relocInfo::none) {
 1242     return;
 1243   } else if (r->is_call() || format == call32_operand) {
 1244     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1245     opnd = locate_operand(inst, call32_operand);
 1246   } else if (r->is_data()) {
 1247     assert(format == imm_operand || format == disp32_operand
 1248            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1249     opnd = locate_operand(inst, (WhichOperand)format);
 1250   } else {
 1251     assert(format == imm_operand, "cannot specify a format");
 1252     return;
 1253   }
 1254   assert(opnd == pc(), "must put operand where relocs can find it");
 1255 }
 1256 #endif // ASSERT
 1257 
 1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
 1259   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1260 }
 1261 
 1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
 1263   if (adr.isxmmindex()) {
 1264      emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1265   } else {
 1266      emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1267   }
 1268 }
 1269 
 1270 // Now the Assembler instructions (identical for 32/64 bits)
 1271 
 1272 void Assembler::adcl(Address dst, int32_t imm32) {
 1273   InstructionMark im(this);
 1274   prefix(dst);
 1275   emit_arith_operand(0x81, rdx, dst, imm32);
 1276 }
 1277 
 1278 void Assembler::adcl(Address dst, Register src) {
 1279   InstructionMark im(this);
 1280   prefix(dst, src);
 1281   emit_int8(0x11);
 1282   emit_operand(src, dst, 0);
 1283 }
 1284 
 1285 void Assembler::adcl(Register dst, int32_t imm32) {
 1286   prefix(dst);
 1287   emit_arith(0x81, 0xD0, dst, imm32);
 1288 }
 1289 
 1290 void Assembler::adcl(Register dst, Address src) {
 1291   InstructionMark im(this);
 1292   prefix(src, dst);
 1293   emit_int8(0x13);
 1294   emit_operand(dst, src, 0);
 1295 }
 1296 
 1297 void Assembler::adcl(Register dst, Register src) {
 1298   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1299   emit_arith(0x13, 0xC0, dst, src);
 1300 }
 1301 
 1302 void Assembler::addl(Address dst, int32_t imm32) {
 1303   InstructionMark im(this);
 1304   prefix(dst);
 1305   emit_arith_operand(0x81, rax, dst, imm32);
 1306 }
 1307 
 1308 void Assembler::addb(Address dst, int imm8) {
 1309   InstructionMark im(this);
 1310   prefix(dst);
 1311   emit_int8((unsigned char)0x80);
 1312   emit_operand(rax, dst, 1);
 1313   emit_int8(imm8);
 1314 }
 1315 
 1316 void Assembler::addw(Register dst, Register src) {
 1317   emit_int8(0x66);
 1318   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1319   emit_arith(0x03, 0xC0, dst, src);
 1320 }
 1321 
 1322 void Assembler::addw(Address dst, int imm16) {
 1323   InstructionMark im(this);
 1324   emit_int8(0x66);
 1325   prefix(dst);
 1326   emit_int8((unsigned char)0x81);
 1327   emit_operand(rax, dst, 2);
 1328   emit_int16(imm16);
 1329 }
 1330 
 1331 void Assembler::addl(Address dst, Register src) {
 1332   InstructionMark im(this);
 1333   prefix(dst, src);
 1334   emit_int8(0x01);
 1335   emit_operand(src, dst, 0);
 1336 }
 1337 
 1338 void Assembler::addl(Register dst, int32_t imm32) {
 1339   prefix(dst);
 1340   emit_arith(0x81, 0xC0, dst, imm32);
 1341 }
 1342 
 1343 void Assembler::addl(Register dst, Address src) {
 1344   InstructionMark im(this);
 1345   prefix(src, dst);
 1346   emit_int8(0x03);
 1347   emit_operand(dst, src, 0);
 1348 }
 1349 
 1350 void Assembler::addl(Register dst, Register src) {
 1351   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1352   emit_arith(0x03, 0xC0, dst, src);
 1353 }
 1354 
 1355 void Assembler::addr_nop_4() {
 1356   assert(UseAddressNop, "no CPU support");
 1357   // 4 bytes: NOP DWORD PTR [EAX+0]
 1358   emit_int32(0x0F,
 1359              0x1F,
 1360              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1361              0);   // 8-bits offset (1 byte)
 1362 }
 1363 
 1364 void Assembler::addr_nop_5() {
 1365   assert(UseAddressNop, "no CPU support");
 1366   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1367   emit_int32(0x0F,
 1368              0x1F,
 1369              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1370              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1371   emit_int8(0);     // 8-bits offset (1 byte)
 1372 }
 1373 
 1374 void Assembler::addr_nop_7() {
 1375   assert(UseAddressNop, "no CPU support");
 1376   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1377   emit_int24(0x0F,
 1378              0x1F,
 1379              (unsigned char)0x80);
 1380                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1381   emit_int32(0);   // 32-bits offset (4 bytes)
 1382 }
 1383 
 1384 void Assembler::addr_nop_8() {
 1385   assert(UseAddressNop, "no CPU support");
 1386   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1387   emit_int32(0x0F,
 1388              0x1F,
 1389              (unsigned char)0x84,
 1390                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1391              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1392   emit_int32(0);    // 32-bits offset (4 bytes)
 1393 }
 1394 
 1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1397   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1398   attributes.set_rex_vex_w_reverted();
 1399   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1400   emit_int16(0x58, (0xC0 | encode));
 1401 }
 1402 
 1403 void Assembler::addsd(XMMRegister dst, Address src) {
 1404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1405   InstructionMark im(this);
 1406   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1407   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1408   attributes.set_rex_vex_w_reverted();
 1409   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1410   emit_int8(0x58);
 1411   emit_operand(dst, src, 0);
 1412 }
 1413 
 1414 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1415   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1416   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1417   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1418   emit_int16(0x58, (0xC0 | encode));
 1419 }
 1420 
 1421 void Assembler::addss(XMMRegister dst, Address src) {
 1422   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1423   InstructionMark im(this);
 1424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1425   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1426   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1427   emit_int8(0x58);
 1428   emit_operand(dst, src, 0);
 1429 }
 1430 
 1431 void Assembler::aesdec(XMMRegister dst, Address src) {
 1432   assert(VM_Version::supports_aes(), "");
 1433   InstructionMark im(this);
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int8((unsigned char)0xDE);
 1437   emit_operand(dst, src, 0);
 1438 }
 1439 
 1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1441   assert(VM_Version::supports_aes(), "");
 1442   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1443   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1448   assert(VM_Version::supports_avx512_vaes(), "");
 1449   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1450   attributes.set_is_evex_instruction();
 1451   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1453 }
 1454 
 1455 
 1456 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionMark im(this);
 1459   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1460   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1461   emit_int8((unsigned char)0xDF);
 1462   emit_operand(dst, src, 0);
 1463 }
 1464 
 1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1466   assert(VM_Version::supports_aes(), "");
 1467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1468   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1469   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1470 }
 1471 
 1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1473   assert(VM_Version::supports_avx512_vaes(), "");
 1474   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1475   attributes.set_is_evex_instruction();
 1476   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1477   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1478 }
 1479 
 1480 void Assembler::aesenc(XMMRegister dst, Address src) {
 1481   assert(VM_Version::supports_aes(), "");
 1482   InstructionMark im(this);
 1483   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1484   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1485   emit_int8((unsigned char)0xDC);
 1486   emit_operand(dst, src, 0);
 1487 }
 1488 
 1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1490   assert(VM_Version::supports_aes(), "");
 1491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1492   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1493   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1494 }
 1495 
 1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1497   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1499   attributes.set_is_evex_instruction();
 1500   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1501   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1502 }
 1503 
 1504 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1505   assert(VM_Version::supports_aes(), "");
 1506   InstructionMark im(this);
 1507   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1508   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1509   emit_int8((unsigned char)0xDD);
 1510   emit_operand(dst, src, 0);
 1511 }
 1512 
 1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1514   assert(VM_Version::supports_aes(), "");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1521   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1522   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1523   attributes.set_is_evex_instruction();
 1524   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1526 }
 1527 
 1528 void Assembler::andb(Address dst, Register src) {
 1529   InstructionMark im(this);
 1530   prefix(dst, src, true);
 1531   emit_int8(0x20);
 1532   emit_operand(src, dst, 0);
 1533 }
 1534 
 1535 void Assembler::andw(Register dst, Register src) {
 1536   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1537   emit_arith(0x23, 0xC0, dst, src);
 1538 }
 1539 
 1540 void Assembler::andl(Address dst, int32_t imm32) {
 1541   InstructionMark im(this);
 1542   prefix(dst);
 1543   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1544 }
 1545 
 1546 void Assembler::andl(Register dst, int32_t imm32) {
 1547   prefix(dst);
 1548   emit_arith(0x81, 0xE0, dst, imm32);
 1549 }
 1550 
 1551 void Assembler::andl(Address dst, Register src) {
 1552   InstructionMark im(this);
 1553   prefix(dst, src);
 1554   emit_int8(0x21);
 1555   emit_operand(src, dst, 0);
 1556 }
 1557 
 1558 void Assembler::andl(Register dst, Address src) {
 1559   InstructionMark im(this);
 1560   prefix(src, dst);
 1561   emit_int8(0x23);
 1562   emit_operand(dst, src, 0);
 1563 }
 1564 
 1565 void Assembler::andl(Register dst, Register src) {
 1566   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1567   emit_arith(0x23, 0xC0, dst, src);
 1568 }
 1569 
 1570 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1571   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1573   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1574   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1575 }
 1576 
 1577 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1578   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1579   InstructionMark im(this);
 1580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1581   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1582   emit_int8((unsigned char)0xF2);
 1583   emit_operand(dst, src2, 0);
 1584 }
 1585 
 1586 void Assembler::bsfl(Register dst, Register src) {
 1587   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1588   emit_int24(0x0F,
 1589              (unsigned char)0xBC,
 1590              0xC0 | encode);
 1591 }
 1592 
 1593 void Assembler::bsrl(Register dst, Register src) {
 1594   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1595   emit_int24(0x0F,
 1596              (unsigned char)0xBD,
 1597              0xC0 | encode);
 1598 }
 1599 
 1600 void Assembler::bswapl(Register reg) { // bswap
 1601   int encode = prefix_and_encode(reg->encoding());
 1602   emit_int16(0x0F, (0xC8 | encode));
 1603 }
 1604 
 1605 void Assembler::blsil(Register dst, Register src) {
 1606   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1607   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1608   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1609   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1610 }
 1611 
 1612 void Assembler::blsil(Register dst, Address src) {
 1613   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1614   InstructionMark im(this);
 1615   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1616   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1617   emit_int8((unsigned char)0xF3);
 1618   emit_operand(rbx, src, 0);
 1619 }
 1620 
 1621 void Assembler::blsmskl(Register dst, Register src) {
 1622   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1623   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1624   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1625   emit_int16((unsigned char)0xF3,
 1626              0xC0 | encode);
 1627 }
 1628 
 1629 void Assembler::blsmskl(Register dst, Address src) {
 1630   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1631   InstructionMark im(this);
 1632   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1633   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1634   emit_int8((unsigned char)0xF3);
 1635   emit_operand(rdx, src, 0);
 1636 }
 1637 
 1638 void Assembler::blsrl(Register dst, Register src) {
 1639   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1640   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1641   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1642   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1643 }
 1644 
 1645 void Assembler::blsrl(Register dst, Address src) {
 1646   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1647   InstructionMark im(this);
 1648   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1649   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1650   emit_int8((unsigned char)0xF3);
 1651   emit_operand(rcx, src, 0);
 1652 }
 1653 
 1654 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1655   // suspect disp32 is always good
 1656   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1657 
 1658   if (L.is_bound()) {
 1659     const int long_size = 5;
 1660     int offs = (int)( target(L) - pc() );
 1661     assert(offs <= 0, "assembler error");
 1662     InstructionMark im(this);
 1663     // 1110 1000 #32-bit disp
 1664     emit_int8((unsigned char)0xE8);
 1665     emit_data(offs - long_size, rtype, operand);
 1666   } else {
 1667     InstructionMark im(this);
 1668     // 1110 1000 #32-bit disp
 1669     L.add_patch_at(code(), locator());
 1670 
 1671     emit_int8((unsigned char)0xE8);
 1672     emit_data(int(0), rtype, operand);
 1673   }
 1674 }
 1675 
 1676 void Assembler::call(Register dst) {
 1677   int encode = prefix_and_encode(dst->encoding());
 1678   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1679 }
 1680 
 1681 
 1682 void Assembler::call(Address adr) {
 1683   InstructionMark im(this);
 1684   prefix(adr);
 1685   emit_int8((unsigned char)0xFF);
 1686   emit_operand(rdx, adr, 0);
 1687 }
 1688 
 1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1690   InstructionMark im(this);
 1691   emit_int8((unsigned char)0xE8);
 1692   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1693   // Entry is NULL in case of a scratch emit.
 1694   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1695   // Technically, should use call32_operand, but this format is
 1696   // implied by the fact that we're emitting a call instruction.
 1697 
 1698   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1699   emit_data((int) disp, rspec, operand);
 1700 }
 1701 
 1702 void Assembler::cdql() {
 1703   emit_int8((unsigned char)0x99);
 1704 }
 1705 
 1706 void Assembler::cld() {
 1707   emit_int8((unsigned char)0xFC);
 1708 }
 1709 
 1710 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1711   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1712   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1713   emit_int24(0x0F,
 1714              0x40 | cc,
 1715              0xC0 | encode);
 1716 }
 1717 
 1718 
 1719 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1720   InstructionMark im(this);
 1721   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1722   prefix(src, dst);
 1723   emit_int16(0x0F, (0x40 | cc));
 1724   emit_operand(dst, src, 0);
 1725 }
 1726 
 1727 void Assembler::cmpb(Address dst, int imm8) {
 1728   InstructionMark im(this);
 1729   prefix(dst);
 1730   emit_int8((unsigned char)0x80);
 1731   emit_operand(rdi, dst, 1);
 1732   emit_int8(imm8);
 1733 }
 1734 
 1735 void Assembler::cmpl(Address dst, int32_t imm32) {
 1736   InstructionMark im(this);
 1737   prefix(dst);
 1738   emit_arith_operand(0x81, as_Register(7), dst, imm32);
 1739 }
 1740 
 1741 void Assembler::cmpl(Register dst, int32_t imm32) {
 1742   prefix(dst);
 1743   emit_arith(0x81, 0xF8, dst, imm32);
 1744 }
 1745 
 1746 void Assembler::cmpl(Register dst, Register src) {
 1747   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1748   emit_arith(0x3B, 0xC0, dst, src);
 1749 }
 1750 
 1751 void Assembler::cmpl(Register dst, Address src) {
 1752   InstructionMark im(this);
 1753   prefix(src, dst);
 1754   emit_int8(0x3B);
 1755   emit_operand(dst, src, 0);
 1756 }
 1757 
 1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
 1759   InstructionMark im(this);
 1760   prefix(dst);
 1761   emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
 1762 }
 1763 
 1764 void Assembler::cmpw(Address dst, int imm16) {
 1765   InstructionMark im(this);
 1766   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1767   emit_int16(0x66, (unsigned char)0x81);
 1768   emit_operand(rdi, dst, 2);
 1769   emit_int16(imm16);
 1770 }
 1771 
 1772 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1773 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1774 // The ZF is set if the compared values were equal, and cleared otherwise.
 1775 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1776   InstructionMark im(this);
 1777   prefix(adr, reg);
 1778   emit_int16(0x0F, (unsigned char)0xB1);
 1779   emit_operand(reg, adr, 0);
 1780 }
 1781 
 1782 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1783   InstructionMark im(this);
 1784   size_prefix();
 1785   prefix(adr, reg);
 1786   emit_int16(0x0F, (unsigned char)0xB1);
 1787   emit_operand(reg, adr, 0);
 1788 }
 1789 
 1790 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1791 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1792 // The ZF is set if the compared values were equal, and cleared otherwise.
 1793 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1794   InstructionMark im(this);
 1795   prefix(adr, reg, true);
 1796   emit_int16(0x0F, (unsigned char)0xB0);
 1797   emit_operand(reg, adr, 0);
 1798 }
 1799 
 1800 void Assembler::comisd(XMMRegister dst, Address src) {
 1801   // NOTE: dbx seems to decode this as comiss even though the
 1802   // 0x66 is there. Strangely ucomisd comes out correct
 1803   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1804   InstructionMark im(this);
 1805   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1806   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1807   attributes.set_rex_vex_w_reverted();
 1808   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1809   emit_int8(0x2F);
 1810   emit_operand(dst, src, 0);
 1811 }
 1812 
 1813 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1814   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1815   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1816   attributes.set_rex_vex_w_reverted();
 1817   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1818   emit_int16(0x2F, (0xC0 | encode));
 1819 }
 1820 
 1821 void Assembler::comiss(XMMRegister dst, Address src) {
 1822   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1823   InstructionMark im(this);
 1824   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1825   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1826   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1827   emit_int8(0x2F);
 1828   emit_operand(dst, src, 0);
 1829 }
 1830 
 1831 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1832   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1833   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1835   emit_int16(0x2F, (0xC0 | encode));
 1836 }
 1837 
 1838 void Assembler::cpuid() {
 1839   emit_int16(0x0F, (unsigned char)0xA2);
 1840 }
 1841 
 1842 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1843 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1844 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1845 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1846 //
 1847 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1848 //
 1849 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1850 //
 1851 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1852 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1853   assert(VM_Version::supports_sse4_2(), "");
 1854   int8_t w = 0x01;
 1855   Prefix p = Prefix_EMPTY;
 1856 
 1857   emit_int8((unsigned char)0xF2);
 1858   switch (sizeInBytes) {
 1859   case 1:
 1860     w = 0;
 1861     break;
 1862   case 2:
 1863   case 4:
 1864     break;
 1865   LP64_ONLY(case 8:)
 1866     // This instruction is not valid in 32 bits
 1867     // Note:
 1868     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1869     //
 1870     // Page B - 72   Vol. 2C says
 1871     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1872     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1873     //                                                                            F0!!!
 1874     // while 3 - 208 Vol. 2A
 1875     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1876     //
 1877     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1878     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1879     p = REX_W;
 1880     break;
 1881   default:
 1882     assert(0, "Unsupported value for a sizeInBytes argument");
 1883     break;
 1884   }
 1885   LP64_ONLY(prefix(crc, v, p);)
 1886   emit_int32(0x0F,
 1887              0x38,
 1888              0xF0 | w,
 1889              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1890 }
 1891 
 1892 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1893   assert(VM_Version::supports_sse4_2(), "");
 1894   InstructionMark im(this);
 1895   int8_t w = 0x01;
 1896   Prefix p = Prefix_EMPTY;
 1897 
 1898   emit_int8((int8_t)0xF2);
 1899   switch (sizeInBytes) {
 1900   case 1:
 1901     w = 0;
 1902     break;
 1903   case 2:
 1904   case 4:
 1905     break;
 1906   LP64_ONLY(case 8:)
 1907     // This instruction is not valid in 32 bits
 1908     p = REX_W;
 1909     break;
 1910   default:
 1911     assert(0, "Unsupported value for a sizeInBytes argument");
 1912     break;
 1913   }
 1914   LP64_ONLY(prefix(crc, adr, p);)
 1915   emit_int24(0x0F, 0x38, (0xF0 | w));
 1916   emit_operand(crc, adr, 0);
 1917 }
 1918 
 1919 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1920   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1922   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1923   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1924 }
 1925 
 1926 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1927   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1928   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1929   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1930   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1931 }
 1932 
 1933 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 1934   assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
 1935   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1936   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1937   emit_int24(0x1D, (0xC0 | encode), imm8);
 1938 }
 1939 
 1940 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
 1941   assert(VM_Version::supports_avx512vl(), "");
 1942   InstructionMark im(this);
 1943   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 1944   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
 1945   attributes.reset_is_clear_context();
 1946   attributes.set_embedded_opmask_register_specifier(mask);
 1947   attributes.set_is_evex_instruction();
 1948   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1949   emit_int8(0x1D);
 1950   emit_operand(src, dst, 1);
 1951   emit_int8(imm8);
 1952 }
 1953 
 1954 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1955   assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
 1956   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
 1957   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1958   emit_int16(0x13, (0xC0 | encode));
 1959 }
 1960 
 1961 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1962   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1964   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1965   emit_int16(0x5B, (0xC0 | encode));
 1966 }
 1967 
 1968 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1969   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1970   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1971   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1972   emit_int16(0x5B, (0xC0 | encode));
 1973 }
 1974 
 1975 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1977   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1978   attributes.set_rex_vex_w_reverted();
 1979   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1980   emit_int16(0x5A, (0xC0 | encode));
 1981 }
 1982 
 1983 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1984   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1985   InstructionMark im(this);
 1986   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1987   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1988   attributes.set_rex_vex_w_reverted();
 1989   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1990   emit_int8(0x5A);
 1991   emit_operand(dst, src, 0);
 1992 }
 1993 
 1994 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1996   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1997   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1998   emit_int16(0x2A, (0xC0 | encode));
 1999 }
 2000 
 2001 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 2002   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2003   InstructionMark im(this);
 2004   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2005   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2006   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2007   emit_int8(0x2A);
 2008   emit_operand(dst, src, 0);
 2009 }
 2010 
 2011 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 2012   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2013   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2014   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2015   emit_int16(0x2A, (0xC0 | encode));
 2016 }
 2017 
 2018 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 2019   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2020   InstructionMark im(this);
 2021   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2022   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2023   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2024   emit_int8(0x2A);
 2025   emit_operand(dst, src, 0);
 2026 }
 2027 
 2028 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 2029   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2030   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2031   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2032   emit_int16(0x2A, (0xC0 | encode));
 2033 }
 2034 
 2035 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 2036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2038   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2039   emit_int16(0x5A, (0xC0 | encode));
 2040 }
 2041 
 2042 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 2043   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2044   InstructionMark im(this);
 2045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2046   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2047   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2048   emit_int8(0x5A);
 2049   emit_operand(dst, src, 0);
 2050 }
 2051 
 2052 
 2053 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 2054   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2055   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2056   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2057   emit_int16(0x2C, (0xC0 | encode));
 2058 }
 2059 
 2060 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 2061   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2063   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2064   emit_int16(0x2D, (0xC0 | encode));
 2065 }
 2066 
 2067 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2068   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2070   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2071   emit_int16(0x2C, (0xC0 | encode));
 2072 }
 2073 
 2074 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2075   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2076   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2077   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2078   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2079   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2080 }
 2081 
 2082 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2083   assert(VM_Version::supports_ssse3(), "");
 2084   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2085   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2086   emit_int16(0x1C, (0xC0 | encode));
 2087 }
 2088 
 2089 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2090   assert(VM_Version::supports_ssse3(), "");
 2091   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2092   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2093   emit_int16(0x1D, (0xC0 | encode));
 2094 }
 2095 
 2096 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2097   assert(VM_Version::supports_ssse3(), "");
 2098   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2099   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x1E, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2105          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2106          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2107   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2108   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2109   emit_int16(0x1C, (0xC0 | encode));
 2110 }
 2111 
 2112 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2113   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2114          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2115          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2116   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2117   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2118   emit_int16(0x1D, (0xC0 | encode));
 2119 }
 2120 
 2121 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2122   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2123   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2124   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2125   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2126   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2127   emit_int16(0x1E, (0xC0 | encode));
 2128 }
 2129 
 2130 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2131   assert(UseAVX > 2, "");
 2132   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2133   attributes.set_is_evex_instruction();
 2134   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2135   emit_int16(0x1F, (0xC0 | encode));
 2136 }
 2137 
 2138 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2139   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2140   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2141   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2142   emit_int16(0x5A, (0xC0 | encode));
 2143 }
 2144 
 2145 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2146   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2147   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2149   attributes.set_rex_vex_w_reverted();
 2150   emit_int16(0x5A, (0xC0 | encode));
 2151 }
 2152 
 2153 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2154   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2155   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2156   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2157   emit_int16(0x5B, (0xC0 | encode));
 2158 }
 2159 
 2160 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2161   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2162   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2163   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2164   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2165 }
 2166 
 2167 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2168   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2169   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2170   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2171   emit_int16(0x5B, (0xC0 | encode));
 2172 }
 2173 
 2174 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2175   assert(VM_Version::supports_avx512dq(), "");
 2176   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2177   attributes.set_is_evex_instruction();
 2178   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2179   emit_int16(0x7A, (0xC0 | encode));
 2180 }
 2181 
 2182 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2183   assert(VM_Version::supports_avx512dq(), "");
 2184   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2185   attributes.set_is_evex_instruction();
 2186   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2187   emit_int16(0x7B, (0xC0 | encode));
 2188 }
 2189 
 2190 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2191   assert(VM_Version::supports_avx512dq(), "");
 2192   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2193   attributes.set_is_evex_instruction();
 2194   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2195   emit_int16(0x5B, (0xC0 | encode));
 2196 }
 2197 
 2198 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2199   assert(VM_Version::supports_avx512dq(), "");
 2200   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2201   attributes.set_is_evex_instruction();
 2202   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2203   emit_int16(0x7A, (0xC0 | encode));
 2204 }
 2205 
 2206 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2207   assert(VM_Version::supports_avx512dq(), "");
 2208   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2209   attributes.set_is_evex_instruction();
 2210   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2211   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2212 }
 2213 
 2214 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2215   assert(VM_Version::supports_avx512bw(), "");
 2216   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2217   attributes.set_is_evex_instruction();
 2218   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2219   emit_int16(0x30, (0xC0 | encode));
 2220 }
 2221 
 2222 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2223   assert(UseAVX > 2, "");
 2224   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2225   attributes.set_is_evex_instruction();
 2226   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2227   emit_int16(0x33, (0xC0 | encode));
 2228 }
 2229 
 2230 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2231   assert(UseAVX > 2, "");
 2232   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2233   attributes.set_is_evex_instruction();
 2234   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2235   emit_int16(0x31, (0xC0 | encode));
 2236 }
 2237 
 2238 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2239   assert(UseAVX > 2, "");
 2240   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2241   attributes.set_is_evex_instruction();
 2242   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2243   emit_int16(0x35, (0xC0 | encode));
 2244 }
 2245 
 2246 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2247   assert(UseAVX > 2, "");
 2248   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2249   attributes.set_is_evex_instruction();
 2250   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2251   emit_int16(0x32, (0xC0 | encode));
 2252 }
 2253 
 2254 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2255   assert(UseAVX > 2, "");
 2256   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2257   attributes.set_is_evex_instruction();
 2258   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2259   emit_int16(0x34, (0xC0 | encode));
 2260 }
 2261 
 2262 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2263   assert(UseAVX > 2, "");
 2264   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2265   attributes.set_is_evex_instruction();
 2266   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2267   emit_int16(0x25, (0xC0 | encode));
 2268 }
 2269 
 2270 void Assembler::decl(Address dst) {
 2271   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2272   InstructionMark im(this);
 2273   prefix(dst);
 2274   emit_int8((unsigned char)0xFF);
 2275   emit_operand(rcx, dst, 0);
 2276 }
 2277 
 2278 void Assembler::divsd(XMMRegister dst, Address src) {
 2279   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2280   InstructionMark im(this);
 2281   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2282   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2283   attributes.set_rex_vex_w_reverted();
 2284   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2285   emit_int8(0x5E);
 2286   emit_operand(dst, src, 0);
 2287 }
 2288 
 2289 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2290   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2291   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2292   attributes.set_rex_vex_w_reverted();
 2293   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2294   emit_int16(0x5E, (0xC0 | encode));
 2295 }
 2296 
 2297 void Assembler::divss(XMMRegister dst, Address src) {
 2298   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2299   InstructionMark im(this);
 2300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2301   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2302   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2303   emit_int8(0x5E);
 2304   emit_operand(dst, src, 0);
 2305 }
 2306 
 2307 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2308   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2310   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2311   emit_int16(0x5E, (0xC0 | encode));
 2312 }
 2313 
 2314 void Assembler::hlt() {
 2315   emit_int8((unsigned char)0xF4);
 2316 }
 2317 
 2318 void Assembler::idivl(Register src) {
 2319   int encode = prefix_and_encode(src->encoding());
 2320   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2321 }
 2322 
 2323 void Assembler::divl(Register src) { // Unsigned
 2324   int encode = prefix_and_encode(src->encoding());
 2325   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2326 }
 2327 
 2328 void Assembler::imull(Register src) {
 2329   int encode = prefix_and_encode(src->encoding());
 2330   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2331 }
 2332 
 2333 void Assembler::imull(Register dst, Register src) {
 2334   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2335   emit_int24(0x0F,
 2336              (unsigned char)0xAF,
 2337              (0xC0 | encode));
 2338 }
 2339 
 2340 void Assembler::imull(Register dst, Address src, int32_t value) {
 2341   InstructionMark im(this);
 2342   prefix(src, dst);
 2343   if (is8bit(value)) {
 2344     emit_int8((unsigned char)0x6B);
 2345     emit_operand(dst, src, 1);
 2346     emit_int8(value);
 2347   } else {
 2348     emit_int8((unsigned char)0x69);
 2349     emit_operand(dst, src, 4);
 2350     emit_int32(value);
 2351   }
 2352 }
 2353 
 2354 void Assembler::imull(Register dst, Register src, int value) {
 2355   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2356   if (is8bit(value)) {
 2357     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2358   } else {
 2359     emit_int16(0x69, (0xC0 | encode));
 2360     emit_int32(value);
 2361   }
 2362 }
 2363 
 2364 void Assembler::imull(Register dst, Address src) {
 2365   InstructionMark im(this);
 2366   prefix(src, dst);
 2367   emit_int16(0x0F, (unsigned char)0xAF);
 2368   emit_operand(dst, src, 0);
 2369 }
 2370 
 2371 
 2372 void Assembler::incl(Address dst) {
 2373   // Don't use it directly. Use MacroAssembler::increment() instead.
 2374   InstructionMark im(this);
 2375   prefix(dst);
 2376   emit_int8((unsigned char)0xFF);
 2377   emit_operand(rax, dst, 0);
 2378 }
 2379 
 2380 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2381   InstructionMark im(this);
 2382   assert((0 <= cc) && (cc < 16), "illegal cc");
 2383   if (L.is_bound()) {
 2384     address dst = target(L);
 2385     assert(dst != NULL, "jcc most probably wrong");
 2386 
 2387     const int short_size = 2;
 2388     const int long_size = 6;
 2389     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2390     if (maybe_short && is8bit(offs - short_size)) {
 2391       // 0111 tttn #8-bit disp
 2392       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2393     } else {
 2394       // 0000 1111 1000 tttn #32-bit disp
 2395       assert(is_simm32(offs - long_size),
 2396              "must be 32bit offset (call4)");
 2397       emit_int16(0x0F, (0x80 | cc));
 2398       emit_int32(offs - long_size);
 2399     }
 2400   } else {
 2401     // Note: could eliminate cond. jumps to this jump if condition
 2402     //       is the same however, seems to be rather unlikely case.
 2403     // Note: use jccb() if label to be bound is very close to get
 2404     //       an 8-bit displacement
 2405     L.add_patch_at(code(), locator());
 2406     emit_int16(0x0F, (0x80 | cc));
 2407     emit_int32(0);
 2408   }
 2409 }
 2410 
 2411 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2412   if (L.is_bound()) {
 2413     const int short_size = 2;
 2414     address entry = target(L);
 2415 #ifdef ASSERT
 2416     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2417     intptr_t delta = short_branch_delta();
 2418     if (delta != 0) {
 2419       dist += (dist < 0 ? (-delta) :delta);
 2420     }
 2421     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2422 #endif
 2423     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2424     // 0111 tttn #8-bit disp
 2425     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2426   } else {
 2427     InstructionMark im(this);
 2428     L.add_patch_at(code(), locator(), file, line);
 2429     emit_int16(0x70 | cc, 0);
 2430   }
 2431 }
 2432 
 2433 void Assembler::jmp(Address adr) {
 2434   InstructionMark im(this);
 2435   prefix(adr);
 2436   emit_int8((unsigned char)0xFF);
 2437   emit_operand(rsp, adr, 0);
 2438 }
 2439 
 2440 void Assembler::jmp(Label& L, bool maybe_short) {
 2441   if (L.is_bound()) {
 2442     address entry = target(L);
 2443     assert(entry != NULL, "jmp most probably wrong");
 2444     InstructionMark im(this);
 2445     const int short_size = 2;
 2446     const int long_size = 5;
 2447     intptr_t offs = entry - pc();
 2448     if (maybe_short && is8bit(offs - short_size)) {
 2449       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2450     } else {
 2451       emit_int8((unsigned char)0xE9);
 2452       emit_int32(offs - long_size);
 2453     }
 2454   } else {
 2455     // By default, forward jumps are always 32-bit displacements, since
 2456     // we can't yet know where the label will be bound.  If you're sure that
 2457     // the forward jump will not run beyond 256 bytes, use jmpb to
 2458     // force an 8-bit displacement.
 2459     InstructionMark im(this);
 2460     L.add_patch_at(code(), locator());
 2461     emit_int8((unsigned char)0xE9);
 2462     emit_int32(0);
 2463   }
 2464 }
 2465 
 2466 void Assembler::jmp(Register entry) {
 2467   int encode = prefix_and_encode(entry->encoding());
 2468   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2469 }
 2470 
 2471 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2472   InstructionMark im(this);
 2473   emit_int8((unsigned char)0xE9);
 2474   assert(dest != NULL, "must have a target");
 2475   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2476   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2477   emit_data(disp, rspec.reloc(), call32_operand);
 2478 }
 2479 
 2480 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2481   if (L.is_bound()) {
 2482     const int short_size = 2;
 2483     address entry = target(L);
 2484     assert(entry != NULL, "jmp most probably wrong");
 2485 #ifdef ASSERT
 2486     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2487     intptr_t delta = short_branch_delta();
 2488     if (delta != 0) {
 2489       dist += (dist < 0 ? (-delta) :delta);
 2490     }
 2491     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2492 #endif
 2493     intptr_t offs = entry - pc();
 2494     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2495   } else {
 2496     InstructionMark im(this);
 2497     L.add_patch_at(code(), locator(), file, line);
 2498     emit_int16((unsigned char)0xEB, 0);
 2499   }
 2500 }
 2501 
 2502 void Assembler::ldmxcsr( Address src) {
 2503   if (UseAVX > 0 ) {
 2504     InstructionMark im(this);
 2505     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2506     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2507     emit_int8((unsigned char)0xAE);
 2508     emit_operand(as_Register(2), src, 0);
 2509   } else {
 2510     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2511     InstructionMark im(this);
 2512     prefix(src);
 2513     emit_int16(0x0F, (unsigned char)0xAE);
 2514     emit_operand(as_Register(2), src, 0);
 2515   }
 2516 }
 2517 
 2518 void Assembler::leal(Register dst, Address src) {
 2519   InstructionMark im(this);
 2520   prefix(src, dst);
 2521   emit_int8((unsigned char)0x8D);
 2522   emit_operand(dst, src, 0);
 2523 }
 2524 
 2525 void Assembler::lfence() {
 2526   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2527 }
 2528 
 2529 void Assembler::lock() {
 2530   emit_int8((unsigned char)0xF0);
 2531 }
 2532 
 2533 void Assembler::size_prefix() {
 2534   emit_int8(0x66);
 2535 }
 2536 
 2537 void Assembler::lzcntl(Register dst, Register src) {
 2538   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2539   emit_int8((unsigned char)0xF3);
 2540   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2541   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2542 }
 2543 
 2544 void Assembler::lzcntl(Register dst, Address src) {
 2545   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2546   InstructionMark im(this);
 2547   emit_int8((unsigned char)0xF3);
 2548   prefix(src, dst);
 2549   emit_int16(0x0F, (unsigned char)0xBD);
 2550   emit_operand(dst, src, 0);
 2551 }
 2552 
 2553 // Emit mfence instruction
 2554 void Assembler::mfence() {
 2555   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2556   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2557 }
 2558 
 2559 // Emit sfence instruction
 2560 void Assembler::sfence() {
 2561   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2562   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2563 }
 2564 
 2565 void Assembler::mov(Register dst, Register src) {
 2566   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2567 }
 2568 
 2569 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2570   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2571   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2572   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2573   attributes.set_rex_vex_w_reverted();
 2574   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2575   emit_int16(0x28, (0xC0 | encode));
 2576 }
 2577 
 2578 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2579   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2580   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2581   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2582   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2583   emit_int16(0x28, (0xC0 | encode));
 2584 }
 2585 
 2586 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2587   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2589   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2590   emit_int16(0x16, (0xC0 | encode));
 2591 }
 2592 
 2593 void Assembler::movb(Register dst, Address src) {
 2594   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2595   InstructionMark im(this);
 2596   prefix(src, dst, true);
 2597   emit_int8((unsigned char)0x8A);
 2598   emit_operand(dst, src, 0);
 2599 }
 2600 
 2601 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2602   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2603   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2604   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2605   attributes.set_rex_vex_w_reverted();
 2606   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2607   emit_int16(0x12, 0xC0 | encode);
 2608 }
 2609 
 2610 void Assembler::movddup(XMMRegister dst, Address src) {
 2611   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2612   InstructionMark im(this);
 2613   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2614   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2615   attributes.set_rex_vex_w_reverted();
 2616   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2617   emit_int8(0x12);
 2618   emit_operand(dst, src, 0);
 2619 }
 2620 
 2621 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2622   assert(VM_Version::supports_avx(), "");
 2623   InstructionMark im(this);
 2624   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2625   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2626   attributes.set_rex_vex_w_reverted();
 2627   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2628   emit_int8(0x12);
 2629   emit_operand(dst, src, 0);
 2630 }
 2631 
 2632 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2633   assert(VM_Version::supports_avx512dq(), "");
 2634   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2635   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2636   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2637 }
 2638 
 2639 void Assembler::kmovbl(KRegister dst, Register src) {
 2640   assert(VM_Version::supports_avx512dq(), "");
 2641   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2642   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2643   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2644 }
 2645 
 2646 void Assembler::kmovbl(Register dst, KRegister src) {
 2647   assert(VM_Version::supports_avx512dq(), "");
 2648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2649   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2650   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2651 }
 2652 
 2653 void Assembler::kmovwl(KRegister dst, Register src) {
 2654   assert(VM_Version::supports_evex(), "");
 2655   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2656   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2657   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2658 }
 2659 
 2660 void Assembler::kmovwl(Register dst, KRegister src) {
 2661   assert(VM_Version::supports_evex(), "");
 2662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2663   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2664   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2665 }
 2666 
 2667 void Assembler::kmovwl(KRegister dst, Address src) {
 2668   assert(VM_Version::supports_evex(), "");
 2669   InstructionMark im(this);
 2670   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2671   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2672   emit_int8((unsigned char)0x90);
 2673   emit_operand(dst, src, 0);
 2674 }
 2675 
 2676 void Assembler::kmovwl(Address dst, KRegister src) {
 2677   assert(VM_Version::supports_evex(), "");
 2678   InstructionMark im(this);
 2679   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2680   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2681   emit_int8((unsigned char)0x91);
 2682   emit_operand(src, dst, 0);
 2683 }
 2684 
 2685 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2686   assert(VM_Version::supports_evex(), "");
 2687   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2688   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2689   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2690 }
 2691 
 2692 void Assembler::kmovdl(KRegister dst, Register src) {
 2693   assert(VM_Version::supports_avx512bw(), "");
 2694   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2695   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2696   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2697 }
 2698 
 2699 void Assembler::kmovdl(Register dst, KRegister src) {
 2700   assert(VM_Version::supports_avx512bw(), "");
 2701   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2702   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2703   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2704 }
 2705 
 2706 void Assembler::kmovql(KRegister dst, KRegister src) {
 2707   assert(VM_Version::supports_avx512bw(), "");
 2708   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2709   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2710   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2711 }
 2712 
 2713 void Assembler::kmovql(KRegister dst, Address src) {
 2714   assert(VM_Version::supports_avx512bw(), "");
 2715   InstructionMark im(this);
 2716   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2717   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2718   emit_int8((unsigned char)0x90);
 2719   emit_operand(dst, src, 0);
 2720 }
 2721 
 2722 void Assembler::kmovql(Address dst, KRegister src) {
 2723   assert(VM_Version::supports_avx512bw(), "");
 2724   InstructionMark im(this);
 2725   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2726   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2727   emit_int8((unsigned char)0x91);
 2728   emit_operand(src, dst, 0);
 2729 }
 2730 
 2731 void Assembler::kmovql(KRegister dst, Register src) {
 2732   assert(VM_Version::supports_avx512bw(), "");
 2733   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2734   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2735   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2736 }
 2737 
 2738 void Assembler::kmovql(Register dst, KRegister src) {
 2739   assert(VM_Version::supports_avx512bw(), "");
 2740   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2741   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2742   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2743 }
 2744 
 2745 void Assembler::knotwl(KRegister dst, KRegister src) {
 2746   assert(VM_Version::supports_evex(), "");
 2747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2748   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2749   emit_int16(0x44, (0xC0 | encode));
 2750 }
 2751 
 2752 void Assembler::knotbl(KRegister dst, KRegister src) {
 2753   assert(VM_Version::supports_evex(), "");
 2754   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2755   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2756   emit_int16(0x44, (0xC0 | encode));
 2757 }
 2758 
 2759 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2760   assert(VM_Version::supports_avx512dq(), "");
 2761   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2762   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2763   emit_int16(0x45, (0xC0 | encode));
 2764 }
 2765 
 2766 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2767   assert(VM_Version::supports_evex(), "");
 2768   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2769   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2770   emit_int16(0x45, (0xC0 | encode));
 2771 }
 2772 
 2773 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2774   assert(VM_Version::supports_avx512bw(), "");
 2775   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2776   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2777   emit_int16(0x45, (0xC0 | encode));
 2778 }
 2779 
 2780 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2781   assert(VM_Version::supports_avx512bw(), "");
 2782   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2783   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2784   emit_int16(0x45, (0xC0 | encode));
 2785 }
 2786 
 2787 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2788   assert(VM_Version::supports_avx512dq(), "");
 2789   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2790   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2791   emit_int16(0x47, (0xC0 | encode));
 2792 }
 2793 
 2794 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2795   assert(VM_Version::supports_evex(), "");
 2796   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2797   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2798   emit_int16(0x47, (0xC0 | encode));
 2799 }
 2800 
 2801 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2802   assert(VM_Version::supports_avx512bw(), "");
 2803   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2804   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2805   emit_int16(0x47, (0xC0 | encode));
 2806 }
 2807 
 2808 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2809   assert(VM_Version::supports_avx512bw(), "");
 2810   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2811   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2812   emit_int16(0x47, (0xC0 | encode));
 2813 }
 2814 
 2815 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2816   assert(VM_Version::supports_avx512dq(), "");
 2817   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2818   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2819   emit_int16(0x41, (0xC0 | encode));
 2820 }
 2821 
 2822 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2823   assert(VM_Version::supports_evex(), "");
 2824   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2825   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2826   emit_int16(0x41, (0xC0 | encode));
 2827 }
 2828 
 2829 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2830   assert(VM_Version::supports_avx512bw(), "");
 2831   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2832   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2833   emit_int16(0x41, (0xC0 | encode));
 2834 }
 2835 
 2836 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2837   assert(VM_Version::supports_avx512bw(), "");
 2838   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2839   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2840   emit_int16(0x41, (0xC0 | encode));
 2841 }
 2842 
 2843 void Assembler::knotdl(KRegister dst, KRegister src) {
 2844   assert(VM_Version::supports_avx512bw(), "");
 2845   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2846   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2847   emit_int16(0x44, (0xC0 | encode));
 2848 }
 2849 
 2850 void Assembler::knotql(KRegister dst, KRegister src) {
 2851   assert(VM_Version::supports_avx512bw(), "");
 2852   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2853   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2854   emit_int16(0x44, (0xC0 | encode));
 2855 }
 2856 
 2857 // This instruction produces ZF or CF flags
 2858 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2859   assert(VM_Version::supports_avx512dq(), "");
 2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2861   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2862   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2863 }
 2864 
 2865 // This instruction produces ZF or CF flags
 2866 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2867   assert(VM_Version::supports_evex(), "");
 2868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2869   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2870   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2871 }
 2872 
 2873 // This instruction produces ZF or CF flags
 2874 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2875   assert(VM_Version::supports_avx512bw(), "");
 2876   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2877   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2878   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2879 }
 2880 
 2881 // This instruction produces ZF or CF flags
 2882 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2883   assert(VM_Version::supports_avx512bw(), "");
 2884   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2885   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2886   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2887 }
 2888 
 2889 // This instruction produces ZF or CF flags
 2890 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2891   assert(VM_Version::supports_avx512bw(), "");
 2892   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2893   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2894   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2895 }
 2896 
 2897 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2898   assert(VM_Version::supports_avx512bw(), "");
 2899   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2900   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2901   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2902 }
 2903 
 2904 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2905   assert(VM_Version::supports_avx512dq(), "");
 2906   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2907   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2908   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2909 }
 2910 
 2911 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2912   assert(VM_Version::supports_avx512dq(), "");
 2913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2914   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2915   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2916 }
 2917 
 2918 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2919   assert(VM_Version::supports_avx512bw(), "");
 2920   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2921   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2922   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2923 }
 2924 
 2925 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2926   assert(VM_Version::supports_avx512bw(), "");
 2927   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2928   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2929   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2930 }
 2931 
 2932 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2933   assert(VM_Version::supports_avx512dq(), "");
 2934   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2935   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2936   emit_int16(0x46, (0xC0 | encode));
 2937 }
 2938 
 2939 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2940   assert(VM_Version::supports_avx512dq(), "");
 2941   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2942   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2943   emit_int16(0x32, (0xC0 | encode));
 2944   emit_int8(imm8);
 2945 }
 2946 
 2947 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2948   assert(VM_Version::supports_avx512bw(), "");
 2949   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2950   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2951   emit_int16(0x33, (0xC0 | encode));
 2952   emit_int8(imm8);
 2953 }
 2954 
 2955 
 2956 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2957   assert(VM_Version::supports_avx512dq(), "");
 2958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2959   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2960   emit_int16(0x30, (0xC0 | encode));
 2961 }
 2962 
 2963 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2964   assert(VM_Version::supports_evex(), "");
 2965   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2966   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2967   emit_int16(0x30, (0xC0 | encode));
 2968   emit_int8(imm8);
 2969 }
 2970 
 2971 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2972   assert(VM_Version::supports_avx512bw(), "");
 2973   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2974   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2975   emit_int16(0x31, (0xC0 | encode));
 2976   emit_int8(imm8);
 2977 }
 2978 
 2979 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2980   assert(VM_Version::supports_avx512bw(), "");
 2981   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2982   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2983   emit_int16(0x31, (0xC0 | encode));
 2984   emit_int8(imm8);
 2985 }
 2986 
 2987 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 2988   assert(VM_Version::supports_avx512bw(), "");
 2989   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2990   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2991   emit_int16(0x4B, (0xC0 | encode));
 2992 }
 2993 
 2994 void Assembler::movb(Address dst, int imm8) {
 2995   InstructionMark im(this);
 2996    prefix(dst);
 2997   emit_int8((unsigned char)0xC6);
 2998   emit_operand(rax, dst, 1);
 2999   emit_int8(imm8);
 3000 }
 3001 
 3002 
 3003 void Assembler::movb(Address dst, Register src) {
 3004   assert(src->has_byte_register(), "must have byte register");
 3005   InstructionMark im(this);
 3006   prefix(dst, src, true);
 3007   emit_int8((unsigned char)0x88);
 3008   emit_operand(src, dst, 0);
 3009 }
 3010 
 3011 void Assembler::movdl(XMMRegister dst, Register src) {
 3012   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3013   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3014   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3015   emit_int16(0x6E, (0xC0 | encode));
 3016 }
 3017 
 3018 void Assembler::movdl(Register dst, XMMRegister src) {
 3019   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3021   // swap src/dst to get correct prefix
 3022   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3023   emit_int16(0x7E, (0xC0 | encode));
 3024 }
 3025 
 3026 void Assembler::movdl(XMMRegister dst, Address src) {
 3027   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3028   InstructionMark im(this);
 3029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3030   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3031   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3032   emit_int8(0x6E);
 3033   emit_operand(dst, src, 0);
 3034 }
 3035 
 3036 void Assembler::movdl(Address dst, XMMRegister src) {
 3037   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3038   InstructionMark im(this);
 3039   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3040   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3041   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3042   emit_int8(0x7E);
 3043   emit_operand(src, dst, 0);
 3044 }
 3045 
 3046 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 3047   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3048   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3049   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3050   emit_int16(0x6F, (0xC0 | encode));
 3051 }
 3052 
 3053 void Assembler::movdqa(XMMRegister dst, Address src) {
 3054   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3055   InstructionMark im(this);
 3056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3057   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3058   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3059   emit_int8(0x6F);
 3060   emit_operand(dst, src, 0);
 3061 }
 3062 
 3063 void Assembler::movdqu(XMMRegister dst, Address src) {
 3064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3065   InstructionMark im(this);
 3066   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3067   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3068   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3069   emit_int8(0x6F);
 3070   emit_operand(dst, src, 0);
 3071 }
 3072 
 3073 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 3074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3075   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3076   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3077   emit_int16(0x6F, (0xC0 | encode));
 3078 }
 3079 
 3080 void Assembler::movdqu(Address dst, XMMRegister src) {
 3081   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3082   InstructionMark im(this);
 3083   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3084   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3085   attributes.reset_is_clear_context();
 3086   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3087   emit_int8(0x7F);
 3088   emit_operand(src, dst, 0);
 3089 }
 3090 
 3091 // Move Unaligned 256bit Vector
 3092 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 3093   assert(UseAVX > 0, "");
 3094   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3095   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3096   emit_int16(0x6F, (0xC0 | encode));
 3097 }
 3098 
 3099 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3100   assert(UseAVX > 0, "");
 3101   InstructionMark im(this);
 3102   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3103   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3104   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3105   emit_int8(0x6F);
 3106   emit_operand(dst, src, 0);
 3107 }
 3108 
 3109 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3110   assert(UseAVX > 0, "");
 3111   InstructionMark im(this);
 3112   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3113   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3114   attributes.reset_is_clear_context();
 3115   // swap src<->dst for encoding
 3116   assert(src != xnoreg, "sanity");
 3117   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3118   emit_int8(0x7F);
 3119   emit_operand(src, dst, 0);
 3120 }
 3121 
 3122 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3123   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3124   InstructionMark im(this);
 3125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3126   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3127   emit_int8((unsigned char)0x8C);
 3128   emit_operand(dst, src, 0);
 3129 }
 3130 
 3131 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3132   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3133   InstructionMark im(this);
 3134   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3135   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3136   emit_int8((unsigned char)0x8C);
 3137   emit_operand(dst, src, 0);
 3138 }
 3139 
 3140 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3141   assert(UseAVX > 0, "requires some form of AVX");
 3142   InstructionMark im(this);
 3143   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3144   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3145   emit_int8(0x2C);
 3146   emit_operand(dst, src, 0);
 3147 }
 3148 
 3149 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3150   assert(UseAVX > 0, "requires some form of AVX");
 3151   InstructionMark im(this);
 3152   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3153   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3154   emit_int8(0x2D);
 3155   emit_operand(dst, src, 0);
 3156 }
 3157 
 3158 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3159   assert(UseAVX > 0, "");
 3160   InstructionMark im(this);
 3161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3162   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3163   emit_int8(0x2E);
 3164   emit_operand(src, dst, 0);
 3165 }
 3166 
 3167 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3168   assert(UseAVX > 0, "");
 3169   InstructionMark im(this);
 3170   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3171   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3172   emit_int8(0x2F);
 3173   emit_operand(src, dst, 0);
 3174 }
 3175 
 3176 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3177 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3178   assert(VM_Version::supports_avx512vlbw(), "");
 3179   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3180   attributes.set_embedded_opmask_register_specifier(mask);
 3181   attributes.set_is_evex_instruction();
 3182   if (merge) {
 3183     attributes.reset_is_clear_context();
 3184   }
 3185   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3186   emit_int16(0x6F, (0xC0 | encode));
 3187 }
 3188 
 3189 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
 3190   // Unmasked instruction
 3191   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3192 }
 3193 
 3194 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3195   assert(VM_Version::supports_avx512vlbw(), "");
 3196   InstructionMark im(this);
 3197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3198   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3199   attributes.set_embedded_opmask_register_specifier(mask);
 3200   attributes.set_is_evex_instruction();
 3201   if (merge) {
 3202     attributes.reset_is_clear_context();
 3203   }
 3204   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3205   emit_int8(0x6F);
 3206   emit_operand(dst, src, 0);
 3207 }
 3208 
 3209 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
 3210   // Unmasked instruction
 3211   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3212 }
 3213 
 3214 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3215   assert(VM_Version::supports_avx512vlbw(), "");
 3216   assert(src != xnoreg, "sanity");
 3217   InstructionMark im(this);
 3218   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3219   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3220   attributes.set_embedded_opmask_register_specifier(mask);
 3221   attributes.set_is_evex_instruction();
 3222   if (merge) {
 3223     attributes.reset_is_clear_context();
 3224   }
 3225   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3226   emit_int8(0x7F);
 3227   emit_operand(src, dst, 0);
 3228 }
 3229 
 3230 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
 3231   // Unmasked instruction
 3232   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3233 }
 3234 
 3235 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3236   assert(VM_Version::supports_avx512vlbw(), "");
 3237   InstructionMark im(this);
 3238   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3239   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3240   attributes.set_embedded_opmask_register_specifier(mask);
 3241   attributes.set_is_evex_instruction();
 3242   if (merge) {
 3243     attributes.reset_is_clear_context();
 3244   }
 3245   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3246   emit_int8(0x6F);
 3247   emit_operand(dst, src, 0);
 3248 }
 3249 
 3250 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
 3251   // Unmasked instruction
 3252   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3253 }
 3254 
 3255 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3256   assert(VM_Version::supports_avx512vlbw(), "");
 3257   assert(src != xnoreg, "sanity");
 3258   InstructionMark im(this);
 3259   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3260   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3261   attributes.set_embedded_opmask_register_specifier(mask);
 3262   attributes.set_is_evex_instruction();
 3263   if (merge) {
 3264     attributes.reset_is_clear_context();
 3265   }
 3266   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3267   emit_int8(0x7F);
 3268   emit_operand(src, dst, 0);
 3269 }
 3270 
 3271 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3272   // Unmasked instruction
 3273   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3274 }
 3275 
 3276 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3277   assert(VM_Version::supports_evex(), "");
 3278   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3279   attributes.set_embedded_opmask_register_specifier(mask);
 3280   attributes.set_is_evex_instruction();
 3281   if (merge) {
 3282     attributes.reset_is_clear_context();
 3283   }
 3284   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3285   emit_int16(0x6F, (0xC0 | encode));
 3286 }
 3287 
 3288 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3289   // Unmasked instruction
 3290   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3291 }
 3292 
 3293 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3294   assert(VM_Version::supports_evex(), "");
 3295   InstructionMark im(this);
 3296   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3297   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3298   attributes.set_embedded_opmask_register_specifier(mask);
 3299   attributes.set_is_evex_instruction();
 3300   if (merge) {
 3301     attributes.reset_is_clear_context();
 3302   }
 3303   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3304   emit_int8(0x6F);
 3305   emit_operand(dst, src, 0);
 3306 }
 3307 
 3308 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3309   // Unmasked isntruction
 3310   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3311 }
 3312 
 3313 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3314   assert(VM_Version::supports_evex(), "");
 3315   assert(src != xnoreg, "sanity");
 3316   InstructionMark im(this);
 3317   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3318   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3319   attributes.set_embedded_opmask_register_specifier(mask);
 3320   attributes.set_is_evex_instruction();
 3321   if (merge) {
 3322     attributes.reset_is_clear_context();
 3323   }
 3324   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3325   emit_int8(0x7F);
 3326   emit_operand(src, dst, 0);
 3327 }
 3328 
 3329 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3330   // Unmasked instruction
 3331   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3332 }
 3333 
 3334 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3335   assert(VM_Version::supports_evex(), "");
 3336   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3337   attributes.set_embedded_opmask_register_specifier(mask);
 3338   attributes.set_is_evex_instruction();
 3339   if (merge) {
 3340     attributes.reset_is_clear_context();
 3341   }
 3342   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3343   emit_int16(0x6F, (0xC0 | encode));
 3344 }
 3345 
 3346 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3347   // Unmasked instruction
 3348   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3349 }
 3350 
 3351 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3352   assert(VM_Version::supports_evex(), "");
 3353   InstructionMark im(this);
 3354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3355   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3356   attributes.set_embedded_opmask_register_specifier(mask);
 3357   attributes.set_is_evex_instruction();
 3358   if (merge) {
 3359     attributes.reset_is_clear_context();
 3360   }
 3361   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3362   emit_int8(0x6F);
 3363   emit_operand(dst, src, 0);
 3364 }
 3365 
 3366 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3367   // Unmasked instruction
 3368   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3369 }
 3370 
 3371 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3372   assert(VM_Version::supports_evex(), "");
 3373   assert(src != xnoreg, "sanity");
 3374   InstructionMark im(this);
 3375   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3376   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3377   attributes.set_embedded_opmask_register_specifier(mask);
 3378   if (merge) {
 3379     attributes.reset_is_clear_context();
 3380   }
 3381   attributes.set_is_evex_instruction();
 3382   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3383   emit_int8(0x7F);
 3384   emit_operand(src, dst, 0);
 3385 }
 3386 
 3387 // Uses zero extension on 64bit
 3388 
 3389 void Assembler::movl(Register dst, int32_t imm32) {
 3390   int encode = prefix_and_encode(dst->encoding());
 3391   emit_int8(0xB8 | encode);
 3392   emit_int32(imm32);
 3393 }
 3394 
 3395 void Assembler::movl(Register dst, Register src) {
 3396   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3397   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3398 }
 3399 
 3400 void Assembler::movl(Register dst, Address src) {
 3401   InstructionMark im(this);
 3402   prefix(src, dst);
 3403   emit_int8((unsigned char)0x8B);
 3404   emit_operand(dst, src, 0);
 3405 }
 3406 
 3407 void Assembler::movl(Address dst, int32_t imm32) {
 3408   InstructionMark im(this);
 3409   prefix(dst);
 3410   emit_int8((unsigned char)0xC7);
 3411   emit_operand(rax, dst, 4);
 3412   emit_int32(imm32);
 3413 }
 3414 
 3415 void Assembler::movl(Address dst, Register src) {
 3416   InstructionMark im(this);
 3417   prefix(dst, src);
 3418   emit_int8((unsigned char)0x89);
 3419   emit_operand(src, dst, 0);
 3420 }
 3421 
 3422 // New cpus require to use movsd and movss to avoid partial register stall
 3423 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3424 // The selection is done in MacroAssembler::movdbl() and movflt().
 3425 void Assembler::movlpd(XMMRegister dst, Address src) {
 3426   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3427   InstructionMark im(this);
 3428   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3429   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3430   attributes.set_rex_vex_w_reverted();
 3431   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3432   emit_int8(0x12);
 3433   emit_operand(dst, src, 0);
 3434 }
 3435 
 3436 void Assembler::movq(XMMRegister dst, Address src) {
 3437   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3438   InstructionMark im(this);
 3439   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3440   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3441   attributes.set_rex_vex_w_reverted();
 3442   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3443   emit_int8(0x7E);
 3444   emit_operand(dst, src, 0);
 3445 }
 3446 
 3447 void Assembler::movq(Address dst, XMMRegister src) {
 3448   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3449   InstructionMark im(this);
 3450   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3451   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3452   attributes.set_rex_vex_w_reverted();
 3453   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3454   emit_int8((unsigned char)0xD6);
 3455   emit_operand(src, dst, 0);
 3456 }
 3457 
 3458 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3460   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3461   attributes.set_rex_vex_w_reverted();
 3462   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3463   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3464 }
 3465 
 3466 void Assembler::movq(Register dst, XMMRegister src) {
 3467   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3468   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3469   // swap src/dst to get correct prefix
 3470   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3471   emit_int16(0x7E, (0xC0 | encode));
 3472 }
 3473 
 3474 void Assembler::movq(XMMRegister dst, Register src) {
 3475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3477   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3478   emit_int16(0x6E, (0xC0 | encode));
 3479 }
 3480 
 3481 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3482   InstructionMark im(this);
 3483   prefix(src, dst);
 3484   emit_int16(0x0F, (unsigned char)0xBE);
 3485   emit_operand(dst, src, 0);
 3486 }
 3487 
 3488 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3489   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3490   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3491   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3492 }
 3493 
 3494 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3495   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3496   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3497   attributes.set_rex_vex_w_reverted();
 3498   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3499   emit_int16(0x10, (0xC0 | encode));
 3500 }
 3501 
 3502 void Assembler::movsd(XMMRegister dst, Address src) {
 3503   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3504   InstructionMark im(this);
 3505   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3506   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3507   attributes.set_rex_vex_w_reverted();
 3508   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3509   emit_int8(0x10);
 3510   emit_operand(dst, src, 0);
 3511 }
 3512 
 3513 void Assembler::movsd(Address dst, XMMRegister src) {
 3514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3515   InstructionMark im(this);
 3516   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3517   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3518   attributes.reset_is_clear_context();
 3519   attributes.set_rex_vex_w_reverted();
 3520   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3521   emit_int8(0x11);
 3522   emit_operand(src, dst, 0);
 3523 }
 3524 
 3525 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3526   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3527   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3528   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3529   emit_int16(0x10, (0xC0 | encode));
 3530 }
 3531 
 3532 void Assembler::movss(XMMRegister dst, Address src) {
 3533   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3534   InstructionMark im(this);
 3535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3536   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3537   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3538   emit_int8(0x10);
 3539   emit_operand(dst, src, 0);
 3540 }
 3541 
 3542 void Assembler::movss(Address dst, XMMRegister src) {
 3543   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3544   InstructionMark im(this);
 3545   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3547   attributes.reset_is_clear_context();
 3548   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3549   emit_int8(0x11);
 3550   emit_operand(src, dst, 0);
 3551 }
 3552 
 3553 void Assembler::movswl(Register dst, Address src) { // movsxw
 3554   InstructionMark im(this);
 3555   prefix(src, dst);
 3556   emit_int16(0x0F, (unsigned char)0xBF);
 3557   emit_operand(dst, src, 0);
 3558 }
 3559 
 3560 void Assembler::movswl(Register dst, Register src) { // movsxw
 3561   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3562   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3563 }
 3564 
 3565 void Assembler::movups(XMMRegister dst, Address src) {
 3566   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3567   InstructionMark im(this);
 3568   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3569   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3570   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3571   emit_int8(0x10);
 3572   emit_operand(dst, src, 0);
 3573 }
 3574 
 3575 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
 3576   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3577   InstructionMark im(this);
 3578   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3579   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3580   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3581   emit_int8(0x10);
 3582   emit_operand(dst, src, 0);
 3583 }
 3584 
 3585 void Assembler::movups(Address dst, XMMRegister src) {
 3586   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3587   InstructionMark im(this);
 3588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3589   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3590   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3591   emit_int8(0x11);
 3592   emit_operand(src, dst, 0);
 3593 }
 3594 
 3595 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
 3596   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3597   InstructionMark im(this);
 3598   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3599   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3600   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3601   emit_int8(0x11);
 3602   emit_operand(src, dst, 0);
 3603 }
 3604 
 3605 void Assembler::movw(Address dst, int imm16) {
 3606   InstructionMark im(this);
 3607 
 3608   emit_int8(0x66); // switch to 16-bit mode
 3609   prefix(dst);
 3610   emit_int8((unsigned char)0xC7);
 3611   emit_operand(rax, dst, 2);
 3612   emit_int16(imm16);
 3613 }
 3614 
 3615 void Assembler::movw(Register dst, Address src) {
 3616   InstructionMark im(this);
 3617   emit_int8(0x66);
 3618   prefix(src, dst);
 3619   emit_int8((unsigned char)0x8B);
 3620   emit_operand(dst, src, 0);
 3621 }
 3622 
 3623 void Assembler::movw(Address dst, Register src) {
 3624   InstructionMark im(this);
 3625   emit_int8(0x66);
 3626   prefix(dst, src);
 3627   emit_int8((unsigned char)0x89);
 3628   emit_operand(src, dst, 0);
 3629 }
 3630 
 3631 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3632   InstructionMark im(this);
 3633   prefix(src, dst);
 3634   emit_int16(0x0F, (unsigned char)0xB6);
 3635   emit_operand(dst, src, 0);
 3636 }
 3637 
 3638 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3639   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3640   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3641   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3642 }
 3643 
 3644 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3645   InstructionMark im(this);
 3646   prefix(src, dst);
 3647   emit_int16(0x0F, (unsigned char)0xB7);
 3648   emit_operand(dst, src, 0);
 3649 }
 3650 
 3651 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3652   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3653   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3654 }
 3655 
 3656 void Assembler::mull(Address src) {
 3657   InstructionMark im(this);
 3658   prefix(src);
 3659   emit_int8((unsigned char)0xF7);
 3660   emit_operand(rsp, src, 0);
 3661 }
 3662 
 3663 void Assembler::mull(Register src) {
 3664   int encode = prefix_and_encode(src->encoding());
 3665   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3666 }
 3667 
 3668 void Assembler::mulsd(XMMRegister dst, Address src) {
 3669   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3670   InstructionMark im(this);
 3671   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3672   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3673   attributes.set_rex_vex_w_reverted();
 3674   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3675   emit_int8(0x59);
 3676   emit_operand(dst, src, 0);
 3677 }
 3678 
 3679 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3680   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3681   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3682   attributes.set_rex_vex_w_reverted();
 3683   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3684   emit_int16(0x59, (0xC0 | encode));
 3685 }
 3686 
 3687 void Assembler::mulss(XMMRegister dst, Address src) {
 3688   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3689   InstructionMark im(this);
 3690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3691   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3692   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3693   emit_int8(0x59);
 3694   emit_operand(dst, src, 0);
 3695 }
 3696 
 3697 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3698   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3699   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3700   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3701   emit_int16(0x59, (0xC0 | encode));
 3702 }
 3703 
 3704 void Assembler::negl(Register dst) {
 3705   int encode = prefix_and_encode(dst->encoding());
 3706   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3707 }
 3708 
 3709 void Assembler::negl(Address dst) {
 3710   InstructionMark im(this);
 3711   prefix(dst);
 3712   emit_int8((unsigned char)0xF7);
 3713   emit_operand(as_Register(3), dst, 0);
 3714 }
 3715 
 3716 void Assembler::nop(int i) {
 3717 #ifdef ASSERT
 3718   assert(i > 0, " ");
 3719   // The fancy nops aren't currently recognized by debuggers making it a
 3720   // pain to disassemble code while debugging. If asserts are on clearly
 3721   // speed is not an issue so simply use the single byte traditional nop
 3722   // to do alignment.
 3723 
 3724   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3725   return;
 3726 
 3727 #endif // ASSERT
 3728 
 3729   if (UseAddressNop && VM_Version::is_intel()) {
 3730     //
 3731     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3732     //  1: 0x90
 3733     //  2: 0x66 0x90
 3734     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3735     //  4: 0x0F 0x1F 0x40 0x00
 3736     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3737     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3738     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3739     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3740     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3741     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3742     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3743 
 3744     // The rest coding is Intel specific - don't use consecutive address nops
 3745 
 3746     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3747     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3748     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3749     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3750 
 3751     while(i >= 15) {
 3752       // For Intel don't generate consecutive address nops (mix with regular nops)
 3753       i -= 15;
 3754       emit_int24(0x66, 0x66, 0x66);
 3755       addr_nop_8();
 3756       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3757     }
 3758     switch (i) {
 3759       case 14:
 3760         emit_int8(0x66); // size prefix
 3761       case 13:
 3762         emit_int8(0x66); // size prefix
 3763       case 12:
 3764         addr_nop_8();
 3765         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3766         break;
 3767       case 11:
 3768         emit_int8(0x66); // size prefix
 3769       case 10:
 3770         emit_int8(0x66); // size prefix
 3771       case 9:
 3772         emit_int8(0x66); // size prefix
 3773       case 8:
 3774         addr_nop_8();
 3775         break;
 3776       case 7:
 3777         addr_nop_7();
 3778         break;
 3779       case 6:
 3780         emit_int8(0x66); // size prefix
 3781       case 5:
 3782         addr_nop_5();
 3783         break;
 3784       case 4:
 3785         addr_nop_4();
 3786         break;
 3787       case 3:
 3788         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3789         emit_int8(0x66); // size prefix
 3790       case 2:
 3791         emit_int8(0x66); // size prefix
 3792       case 1:
 3793         emit_int8((unsigned char)0x90);
 3794                          // nop
 3795         break;
 3796       default:
 3797         assert(i == 0, " ");
 3798     }
 3799     return;
 3800   }
 3801   if (UseAddressNop && VM_Version::is_amd_family()) {
 3802     //
 3803     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3804     //  1: 0x90
 3805     //  2: 0x66 0x90
 3806     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3807     //  4: 0x0F 0x1F 0x40 0x00
 3808     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3809     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3810     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3811     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3812     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3813     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3814     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3815 
 3816     // The rest coding is AMD specific - use consecutive address nops
 3817 
 3818     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3819     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3820     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3821     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3822     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3823     //     Size prefixes (0x66) are added for larger sizes
 3824 
 3825     while(i >= 22) {
 3826       i -= 11;
 3827       emit_int24(0x66, 0x66, 0x66);
 3828       addr_nop_8();
 3829     }
 3830     // Generate first nop for size between 21-12
 3831     switch (i) {
 3832       case 21:
 3833         i -= 1;
 3834         emit_int8(0x66); // size prefix
 3835       case 20:
 3836       case 19:
 3837         i -= 1;
 3838         emit_int8(0x66); // size prefix
 3839       case 18:
 3840       case 17:
 3841         i -= 1;
 3842         emit_int8(0x66); // size prefix
 3843       case 16:
 3844       case 15:
 3845         i -= 8;
 3846         addr_nop_8();
 3847         break;
 3848       case 14:
 3849       case 13:
 3850         i -= 7;
 3851         addr_nop_7();
 3852         break;
 3853       case 12:
 3854         i -= 6;
 3855         emit_int8(0x66); // size prefix
 3856         addr_nop_5();
 3857         break;
 3858       default:
 3859         assert(i < 12, " ");
 3860     }
 3861 
 3862     // Generate second nop for size between 11-1
 3863     switch (i) {
 3864       case 11:
 3865         emit_int8(0x66); // size prefix
 3866       case 10:
 3867         emit_int8(0x66); // size prefix
 3868       case 9:
 3869         emit_int8(0x66); // size prefix
 3870       case 8:
 3871         addr_nop_8();
 3872         break;
 3873       case 7:
 3874         addr_nop_7();
 3875         break;
 3876       case 6:
 3877         emit_int8(0x66); // size prefix
 3878       case 5:
 3879         addr_nop_5();
 3880         break;
 3881       case 4:
 3882         addr_nop_4();
 3883         break;
 3884       case 3:
 3885         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3886         emit_int8(0x66); // size prefix
 3887       case 2:
 3888         emit_int8(0x66); // size prefix
 3889       case 1:
 3890         emit_int8((unsigned char)0x90);
 3891                          // nop
 3892         break;
 3893       default:
 3894         assert(i == 0, " ");
 3895     }
 3896     return;
 3897   }
 3898 
 3899   if (UseAddressNop && VM_Version::is_zx()) {
 3900     //
 3901     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3902     //  1: 0x90
 3903     //  2: 0x66 0x90
 3904     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3905     //  4: 0x0F 0x1F 0x40 0x00
 3906     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3907     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3908     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3909     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3910     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3911     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3912     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3913 
 3914     // The rest coding is ZX specific - don't use consecutive address nops
 3915 
 3916     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3917     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3918     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3919     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3920 
 3921     while (i >= 15) {
 3922       // For ZX don't generate consecutive address nops (mix with regular nops)
 3923       i -= 15;
 3924       emit_int24(0x66, 0x66, 0x66);
 3925       addr_nop_8();
 3926       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3927     }
 3928     switch (i) {
 3929       case 14:
 3930         emit_int8(0x66); // size prefix
 3931       case 13:
 3932         emit_int8(0x66); // size prefix
 3933       case 12:
 3934         addr_nop_8();
 3935         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3936         break;
 3937       case 11:
 3938         emit_int8(0x66); // size prefix
 3939       case 10:
 3940         emit_int8(0x66); // size prefix
 3941       case 9:
 3942         emit_int8(0x66); // size prefix
 3943       case 8:
 3944         addr_nop_8();
 3945         break;
 3946       case 7:
 3947         addr_nop_7();
 3948         break;
 3949       case 6:
 3950         emit_int8(0x66); // size prefix
 3951       case 5:
 3952         addr_nop_5();
 3953         break;
 3954       case 4:
 3955         addr_nop_4();
 3956         break;
 3957       case 3:
 3958         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3959         emit_int8(0x66); // size prefix
 3960       case 2:
 3961         emit_int8(0x66); // size prefix
 3962       case 1:
 3963         emit_int8((unsigned char)0x90);
 3964                          // nop
 3965         break;
 3966       default:
 3967         assert(i == 0, " ");
 3968     }
 3969     return;
 3970   }
 3971 
 3972   // Using nops with size prefixes "0x66 0x90".
 3973   // From AMD Optimization Guide:
 3974   //  1: 0x90
 3975   //  2: 0x66 0x90
 3976   //  3: 0x66 0x66 0x90
 3977   //  4: 0x66 0x66 0x66 0x90
 3978   //  5: 0x66 0x66 0x90 0x66 0x90
 3979   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3980   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3981   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3982   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3983   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3984   //
 3985   while (i > 12) {
 3986     i -= 4;
 3987     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3988   }
 3989   // 1 - 12 nops
 3990   if (i > 8) {
 3991     if (i > 9) {
 3992       i -= 1;
 3993       emit_int8(0x66);
 3994     }
 3995     i -= 3;
 3996     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3997   }
 3998   // 1 - 8 nops
 3999   if (i > 4) {
 4000     if (i > 6) {
 4001       i -= 1;
 4002       emit_int8(0x66);
 4003     }
 4004     i -= 3;
 4005     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4006   }
 4007   switch (i) {
 4008     case 4:
 4009       emit_int8(0x66);
 4010     case 3:
 4011       emit_int8(0x66);
 4012     case 2:
 4013       emit_int8(0x66);
 4014     case 1:
 4015       emit_int8((unsigned char)0x90);
 4016       break;
 4017     default:
 4018       assert(i == 0, " ");
 4019   }
 4020 }
 4021 
 4022 void Assembler::notl(Register dst) {
 4023   int encode = prefix_and_encode(dst->encoding());
 4024   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 4025 }
 4026 
 4027 void Assembler::orw(Register dst, Register src) {
 4028   (void)prefix_and_encode(dst->encoding(), src->encoding());
 4029   emit_arith(0x0B, 0xC0, dst, src);
 4030 }
 4031 
 4032 void Assembler::orl(Address dst, int32_t imm32) {
 4033   InstructionMark im(this);
 4034   prefix(dst);
 4035   emit_arith_operand(0x81, rcx, dst, imm32);
 4036 }
 4037 
 4038 void Assembler::orl(Register dst, int32_t imm32) {
 4039   prefix(dst);
 4040   emit_arith(0x81, 0xC8, dst, imm32);
 4041 }
 4042 
 4043 void Assembler::orl(Register dst, Address src) {
 4044   InstructionMark im(this);
 4045   prefix(src, dst);
 4046   emit_int8(0x0B);
 4047   emit_operand(dst, src, 0);
 4048 }
 4049 
 4050 void Assembler::orl(Register dst, Register src) {
 4051   (void) prefix_and_encode(dst->encoding(), src->encoding());
 4052   emit_arith(0x0B, 0xC0, dst, src);
 4053 }
 4054 
 4055 void Assembler::orl(Address dst, Register src) {
 4056   InstructionMark im(this);
 4057   prefix(dst, src);
 4058   emit_int8(0x09);
 4059   emit_operand(src, dst, 0);
 4060 }
 4061 
 4062 void Assembler::orb(Address dst, int imm8) {
 4063   InstructionMark im(this);
 4064   prefix(dst);
 4065   emit_int8((unsigned char)0x80);
 4066   emit_operand(rcx, dst, 1);
 4067   emit_int8(imm8);
 4068 }
 4069 
 4070 void Assembler::orb(Address dst, Register src) {
 4071   InstructionMark im(this);
 4072   prefix(dst, src, true);
 4073   emit_int8(0x08);
 4074   emit_operand(src, dst, 0);
 4075 }
 4076 
 4077 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 4078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4080   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4081   emit_int16(0x63, (0xC0 | encode));
 4082 }
 4083 
 4084 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4085   assert(UseAVX > 0, "some form of AVX must be enabled");
 4086   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4087   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4088   emit_int16(0x63, (0xC0 | encode));
 4089 }
 4090 
 4091 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 4092   assert(VM_Version::supports_sse2(), "");
 4093   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4094   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4095   emit_int16(0x6B, (0xC0 | encode));
 4096 }
 4097 
 4098 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4099   assert(UseAVX > 0, "some form of AVX must be enabled");
 4100   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4101   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4102   emit_int16(0x6B, (0xC0 | encode));
 4103 }
 4104 
 4105 void Assembler::packuswb(XMMRegister dst, Address src) {
 4106   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4107   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4108   InstructionMark im(this);
 4109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4110   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4111   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4112   emit_int8(0x67);
 4113   emit_operand(dst, src, 0);
 4114 }
 4115 
 4116 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 4117   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4118   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4119   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4120   emit_int16(0x67, (0xC0 | encode));
 4121 }
 4122 
 4123 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4124   assert(UseAVX > 0, "some form of AVX must be enabled");
 4125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4126   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4127   emit_int16(0x67, (0xC0 | encode));
 4128 }
 4129 
 4130 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4131   assert(VM_Version::supports_sse4_1(), "");
 4132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4133   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4134   emit_int16(0x2B, (0xC0 | encode));
 4135 }
 4136 
 4137 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4138   assert(UseAVX > 0, "some form of AVX must be enabled");
 4139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4140   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4141   emit_int16(0x2B, (0xC0 | encode));
 4142 }
 4143 
 4144 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4145   assert(VM_Version::supports_avx2(), "");
 4146   assert(vector_len != AVX_128bit, "");
 4147   // VEX.256.66.0F3A.W1 00 /r ib
 4148   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4149   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4150   emit_int24(0x00, (0xC0 | encode), imm8);
 4151 }
 4152 
 4153 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4154   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4155          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4156   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4157   attributes.set_is_evex_instruction();
 4158   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4159   emit_int16(0x36, (0xC0 | encode));
 4160 }
 4161 
 4162 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4163   assert(VM_Version::supports_avx512_vbmi(), "");
 4164   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4165   attributes.set_is_evex_instruction();
 4166   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4167   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4168 }
 4169 
 4170 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4171   assert(VM_Version::supports_avx512_vbmi(), "");
 4172   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4173   attributes.set_is_evex_instruction();
 4174   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4175   emit_int8((unsigned char)0x8D);
 4176   emit_operand(dst, src, 0);
 4177 }
 4178 
 4179 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4180   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4181          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4182          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4183   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4184   attributes.set_is_evex_instruction();
 4185   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4186   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4187 }
 4188 
 4189 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4190   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4191   // VEX.NDS.256.66.0F38.W0 36 /r
 4192   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4193   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4194   emit_int16(0x36, (0xC0 | encode));
 4195 }
 4196 
 4197 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4198   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4199   // VEX.NDS.256.66.0F38.W0 36 /r
 4200   InstructionMark im(this);
 4201   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4202   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4203   emit_int8(0x36);
 4204   emit_operand(dst, src, 0);
 4205 }
 4206 
 4207 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4208   assert(VM_Version::supports_avx2(), "");
 4209   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4211   emit_int24(0x46, (0xC0 | encode), imm8);
 4212 }
 4213 
 4214 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4215   assert(VM_Version::supports_avx(), "");
 4216   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4217   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4218   emit_int24(0x06, (0xC0 | encode), imm8);
 4219 }
 4220 
 4221 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4222   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4223   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4225   emit_int24(0x04, (0xC0 | encode), imm8);
 4226 }
 4227 
 4228 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4229   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4230   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4231   attributes.set_rex_vex_w_reverted();
 4232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4233   emit_int24(0x05, (0xC0 | encode), imm8);
 4234 }
 4235 
 4236 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4237   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4238   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4239   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4240   emit_int24(0x01, (0xC0 | encode), imm8);
 4241 }
 4242 
 4243 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4244   assert(VM_Version::supports_evex(), "");
 4245   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4246   attributes.set_is_evex_instruction();
 4247   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4248   emit_int16(0x76, (0xC0 | encode));
 4249 }
 4250 
 4251 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4252   assert(VM_Version::supports_avx512_vbmi(), "");
 4253   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4254   attributes.set_is_evex_instruction();
 4255   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4256   emit_int16(0x7D, (0xC0 | encode));
 4257 }
 4258 
 4259 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4260   assert(VM_Version::supports_avx512_vbmi(), "");
 4261   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4262   attributes.set_is_evex_instruction();
 4263   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4264   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4265 }
 4266 
 4267 void Assembler::pause() {
 4268   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4269 }
 4270 
 4271 void Assembler::ud2() {
 4272   emit_int16(0x0F, 0x0B);
 4273 }
 4274 
 4275 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4276   assert(VM_Version::supports_sse4_2(), "");
 4277   InstructionMark im(this);
 4278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4279   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4280   emit_int8(0x61);
 4281   emit_operand(dst, src, 1);
 4282   emit_int8(imm8);
 4283 }
 4284 
 4285 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4286   assert(VM_Version::supports_sse4_2(), "");
 4287   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4288   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4289   emit_int24(0x61, (0xC0 | encode), imm8);
 4290 }
 4291 
 4292 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4293 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4294   assert(VM_Version::supports_sse2(), "");
 4295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4296   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4297   emit_int16(0x74, (0xC0 | encode));
 4298 }
 4299 
 4300 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4301   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4302   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4303   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4304   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4305   emit_int16(cond_encoding, (0xC0 | encode));
 4306 }
 4307 
 4308 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4309 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4310   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4311   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4312   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4313   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4314   emit_int16(0x74, (0xC0 | encode));
 4315 }
 4316 
 4317 // In this context, kdst is written the mask used to process the equal components
 4318 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4319   assert(VM_Version::supports_avx512bw(), "");
 4320   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4321   attributes.set_is_evex_instruction();
 4322   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4323   emit_int16(0x74, (0xC0 | encode));
 4324 }
 4325 
 4326 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4327   assert(VM_Version::supports_avx512vlbw(), "");
 4328   InstructionMark im(this);
 4329   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4330   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4331   attributes.set_is_evex_instruction();
 4332   int dst_enc = kdst->encoding();
 4333   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4334   emit_int8(0x64);
 4335   emit_operand(as_Register(dst_enc), src, 0);
 4336 }
 4337 
 4338 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4339   assert(VM_Version::supports_avx512vlbw(), "");
 4340   InstructionMark im(this);
 4341   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4342   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4343   attributes.reset_is_clear_context();
 4344   attributes.set_embedded_opmask_register_specifier(mask);
 4345   attributes.set_is_evex_instruction();
 4346   int dst_enc = kdst->encoding();
 4347   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4348   emit_int8(0x64);
 4349   emit_operand(as_Register(dst_enc), src, 0);
 4350 }
 4351 
 4352 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4353   assert(VM_Version::supports_avx512vlbw(), "");
 4354   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4355   attributes.set_is_evex_instruction();
 4356   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4357   emit_int24(0x3E, (0xC0 | encode), vcc);
 4358 }
 4359 
 4360 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4361   assert(VM_Version::supports_avx512vlbw(), "");
 4362   InstructionMark im(this);
 4363   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4364   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4365   attributes.set_is_evex_instruction();
 4366   int dst_enc = kdst->encoding();
 4367   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4368   emit_int8(0x3E);
 4369   emit_operand(as_Register(dst_enc), src, 1);
 4370   emit_int8(vcc);
 4371 }
 4372 
 4373 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4374   assert(VM_Version::supports_avx512bw(), "");
 4375   InstructionMark im(this);
 4376   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4377   attributes.set_is_evex_instruction();
 4378   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4379   int dst_enc = kdst->encoding();
 4380   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4381   emit_int8(0x74);
 4382   emit_operand(as_Register(dst_enc), src, 0);
 4383 }
 4384 
 4385 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4386   assert(VM_Version::supports_avx512vlbw(), "");
 4387   InstructionMark im(this);
 4388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4389   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4390   attributes.reset_is_clear_context();
 4391   attributes.set_embedded_opmask_register_specifier(mask);
 4392   attributes.set_is_evex_instruction();
 4393   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4394   emit_int8(0x74);
 4395   emit_operand(as_Register(kdst->encoding()), src, 0);
 4396 }
 4397 
 4398 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4399 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4400   assert(VM_Version::supports_sse2(), "");
 4401   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4402   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4403   emit_int16(0x75, (0xC0 | encode));
 4404 }
 4405 
 4406 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4407 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4408   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4409   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4410   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4411   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4412   emit_int16(0x75, (0xC0 | encode));
 4413 }
 4414 
 4415 // In this context, kdst is written the mask used to process the equal components
 4416 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4417   assert(VM_Version::supports_avx512bw(), "");
 4418   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4419   attributes.set_is_evex_instruction();
 4420   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4421   emit_int16(0x75, (0xC0 | encode));
 4422 }
 4423 
 4424 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4425   assert(VM_Version::supports_avx512bw(), "");
 4426   InstructionMark im(this);
 4427   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4428   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4429   attributes.set_is_evex_instruction();
 4430   int dst_enc = kdst->encoding();
 4431   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4432   emit_int8(0x75);
 4433   emit_operand(as_Register(dst_enc), src, 0);
 4434 }
 4435 
 4436 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4437 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4438   assert(VM_Version::supports_sse2(), "");
 4439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4440   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4441   emit_int16(0x76, (0xC0 | encode));
 4442 }
 4443 
 4444 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4445 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4446   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4447   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4448   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4450   emit_int16(0x76, (0xC0 | encode));
 4451 }
 4452 
 4453 // In this context, kdst is written the mask used to process the equal components
 4454 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4455   assert(VM_Version::supports_evex(), "");
 4456   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4457   attributes.set_is_evex_instruction();
 4458   attributes.reset_is_clear_context();
 4459   attributes.set_embedded_opmask_register_specifier(mask);
 4460   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4461   emit_int16(0x76, (0xC0 | encode));
 4462 }
 4463 
 4464 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4465   assert(VM_Version::supports_evex(), "");
 4466   InstructionMark im(this);
 4467   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4468   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4469   attributes.set_is_evex_instruction();
 4470   attributes.reset_is_clear_context();
 4471   attributes.set_embedded_opmask_register_specifier(mask);
 4472   int dst_enc = kdst->encoding();
 4473   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4474   emit_int8(0x76);
 4475   emit_operand(as_Register(dst_enc), src, 0);
 4476 }
 4477 
 4478 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4479 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4480   assert(VM_Version::supports_sse4_1(), "");
 4481   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4482   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4483   emit_int16(0x29, (0xC0 | encode));
 4484 }
 4485 
 4486 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4487   assert(VM_Version::supports_evex(), "");
 4488   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4489   attributes.set_is_evex_instruction();
 4490   attributes.reset_is_clear_context();
 4491   attributes.set_embedded_opmask_register_specifier(mask);
 4492   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4493   emit_int16(0x29, (0xC0 | encode));
 4494 }
 4495 
 4496 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4497   assert(VM_Version::supports_avx(), "");
 4498   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4499   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4500   emit_int16(cond_encoding, (0xC0 | encode));
 4501 }
 4502 
 4503 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4504 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4505   assert(VM_Version::supports_avx(), "");
 4506   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4507   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4508   emit_int16(0x29, (0xC0 | encode));
 4509 }
 4510 
 4511 // In this context, kdst is written the mask used to process the equal components
 4512 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4513   assert(VM_Version::supports_evex(), "");
 4514   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4515   attributes.reset_is_clear_context();
 4516   attributes.set_is_evex_instruction();
 4517   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4518   emit_int16(0x29, (0xC0 | encode));
 4519 }
 4520 
 4521 // In this context, kdst is written the mask used to process the equal components
 4522 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4523   assert(VM_Version::supports_evex(), "");
 4524   InstructionMark im(this);
 4525   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4526   attributes.reset_is_clear_context();
 4527   attributes.set_is_evex_instruction();
 4528   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4529   int dst_enc = kdst->encoding();
 4530   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4531   emit_int8(0x29);
 4532   emit_operand(as_Register(dst_enc), src, 0);
 4533 }
 4534 
 4535 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4536   assert(VM_Version::supports_sse4_1(), "");
 4537   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4538   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4539   emit_int16(0x37, (0xC0 | encode));
 4540 }
 4541 
 4542 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4543   assert(VM_Version::supports_sse2(), "");
 4544   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4545   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4546   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4547 }
 4548 
 4549 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4550   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4551          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4552   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4553   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4554   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4555 }
 4556 
 4557 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4558   assert(VM_Version::supports_avx(), "");
 4559   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4560   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4561   emit_int16(0x50, (0xC0 | encode));
 4562 }
 4563 
 4564 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4565   assert(VM_Version::supports_avx(), "");
 4566   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4567   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4568   emit_int16(0x50, (0xC0 | encode));
 4569 }
 4570 
 4571 
 4572 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4573   assert(VM_Version::supports_sse4_1(), "");
 4574   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4575   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4576   emit_int24(0x16, (0xC0 | encode), imm8);
 4577 }
 4578 
 4579 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4580   assert(VM_Version::supports_sse4_1(), "");
 4581   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4582   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4583   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4584   emit_int8(0x16);
 4585   emit_operand(src, dst, 1);
 4586   emit_int8(imm8);
 4587 }
 4588 
 4589 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4590   assert(VM_Version::supports_sse4_1(), "");
 4591   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4592   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4593   emit_int24(0x16, (0xC0 | encode), imm8);
 4594 }
 4595 
 4596 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4597   assert(VM_Version::supports_sse4_1(), "");
 4598   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4599   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4600   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4601   emit_int8(0x16);
 4602   emit_operand(src, dst, 1);
 4603   emit_int8(imm8);
 4604 }
 4605 
 4606 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4607   assert(VM_Version::supports_sse2(), "");
 4608   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4609   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4610   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4611 }
 4612 
 4613 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4614   assert(VM_Version::supports_sse4_1(), "");
 4615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4616   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4617   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4618   emit_int8(0x15);
 4619   emit_operand(src, dst, 1);
 4620   emit_int8(imm8);
 4621 }
 4622 
 4623 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4624   assert(VM_Version::supports_sse4_1(), "");
 4625   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4626   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4627   emit_int24(0x14, (0xC0 | encode), imm8);
 4628 }
 4629 
 4630 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4631   assert(VM_Version::supports_sse4_1(), "");
 4632   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4633   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4634   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4635   emit_int8(0x14);
 4636   emit_operand(src, dst, 1);
 4637   emit_int8(imm8);
 4638 }
 4639 
 4640 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4641   assert(VM_Version::supports_sse4_1(), "");
 4642   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4643   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4644   emit_int24(0x22, (0xC0 | encode), imm8);
 4645 }
 4646 
 4647 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4648   assert(VM_Version::supports_sse4_1(), "");
 4649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4650   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4651   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4652   emit_int8(0x22);
 4653   emit_operand(dst, src, 1);
 4654   emit_int8(imm8);
 4655 }
 4656 
 4657 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4658   assert(VM_Version::supports_avx(), "");
 4659   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4660   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4661   emit_int24(0x22, (0xC0 | encode), imm8);
 4662 }
 4663 
 4664 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4665   assert(VM_Version::supports_sse4_1(), "");
 4666   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4667   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4668   emit_int24(0x22, (0xC0 | encode), imm8);
 4669 }
 4670 
 4671 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4672   assert(VM_Version::supports_sse4_1(), "");
 4673   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4674   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4675   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4676   emit_int8(0x22);
 4677   emit_operand(dst, src, 1);
 4678   emit_int8(imm8);
 4679 }
 4680 
 4681 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4682   assert(VM_Version::supports_avx(), "");
 4683   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4684   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4685   emit_int24(0x22, (0xC0 | encode), imm8);
 4686 }
 4687 
 4688 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4689   assert(VM_Version::supports_sse2(), "");
 4690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4691   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4692   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4693 }
 4694 
 4695 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4696   assert(VM_Version::supports_sse2(), "");
 4697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4698   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4699   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4700   emit_int8((unsigned char)0xC4);
 4701   emit_operand(dst, src, 1);
 4702   emit_int8(imm8);
 4703 }
 4704 
 4705 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4706   assert(VM_Version::supports_avx(), "");
 4707   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4708   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4709   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4710 }
 4711 
 4712 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4713   assert(VM_Version::supports_sse4_1(), "");
 4714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4715   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4716   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4717   emit_int8(0x20);
 4718   emit_operand(dst, src, 1);
 4719   emit_int8(imm8);
 4720 }
 4721 
 4722 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4723   assert(VM_Version::supports_sse4_1(), "");
 4724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4725   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4726   emit_int24(0x20, (0xC0 | encode), imm8);
 4727 }
 4728 
 4729 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4730   assert(VM_Version::supports_avx(), "");
 4731   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4732   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4733   emit_int24(0x20, (0xC0 | encode), imm8);
 4734 }
 4735 
 4736 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4737   assert(VM_Version::supports_sse4_1(), "");
 4738   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4739   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4740   emit_int24(0x21, (0xC0 | encode), imm8);
 4741 }
 4742 
 4743 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4744   assert(VM_Version::supports_avx(), "");
 4745   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4746   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4747   emit_int24(0x21, (0xC0 | encode), imm8);
 4748 }
 4749 
 4750 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4751   assert(VM_Version::supports_sse4_1(), "");
 4752   InstructionMark im(this);
 4753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4754   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4755   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4756   emit_int8(0x30);
 4757   emit_operand(dst, src, 0);
 4758 }
 4759 
 4760 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4761   assert(VM_Version::supports_sse4_1(), "");
 4762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4763   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4764   emit_int16(0x30, (0xC0 | encode));
 4765 }
 4766 
 4767 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4768   assert(VM_Version::supports_sse4_1(), "");
 4769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4770   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4771   emit_int16(0x20, (0xC0 | encode));
 4772 }
 4773 
 4774 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4775   assert(VM_Version::supports_sse4_1(), "");
 4776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4777   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4778   emit_int16(0x35, (0xC0 | encode));
 4779 }
 4780 
 4781 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4782   assert(VM_Version::supports_sse4_1(), "");
 4783   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4784   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4785   emit_int16(0x21, (0xC0 | encode));
 4786 }
 4787 
 4788 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4789   assert(VM_Version::supports_sse4_1(), "");
 4790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4791   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4792   emit_int16(0x31, (0xC0 | encode));
 4793 }
 4794 
 4795 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4796   assert(VM_Version::supports_sse4_1(), "");
 4797   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4798   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4799   emit_int16(0x22, (0xC0 | encode));
 4800 }
 4801 
 4802 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4803   assert(VM_Version::supports_sse4_1(), "");
 4804   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4805   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4806   emit_int16(0x23, (0xC0 | encode));
 4807 }
 4808 
 4809 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4810   assert(VM_Version::supports_avx(), "");
 4811   InstructionMark im(this);
 4812   assert(dst != xnoreg, "sanity");
 4813   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4814   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4815   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4816   emit_int8(0x30);
 4817   emit_operand(dst, src, 0);
 4818 }
 4819 
 4820 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4821   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4822   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4823   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4824   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4825   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4826   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4827 }
 4828 
 4829 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4830   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4831   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4832   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4833   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4835   emit_int16(0x20, (0xC0 | encode));
 4836 }
 4837 
 4838 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4839   assert(VM_Version::supports_avx512vlbw(), "");
 4840   assert(dst != xnoreg, "sanity");
 4841   InstructionMark im(this);
 4842   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4843   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4844   attributes.set_embedded_opmask_register_specifier(mask);
 4845   attributes.set_is_evex_instruction();
 4846   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4847   emit_int8(0x30);
 4848   emit_operand(dst, src, 0);
 4849 }
 4850 
 4851 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4852   assert(VM_Version::supports_evex(), "");
 4853   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4854   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4855   attributes.set_is_evex_instruction();
 4856   attributes.set_embedded_opmask_register_specifier(mask);
 4857   if (merge) {
 4858     attributes.reset_is_clear_context();
 4859   }
 4860   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4861   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4862 }
 4863 
 4864 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4865   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4866   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4868   emit_int16(0x35, (0xC0 | encode));
 4869 }
 4870 
 4871 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4872   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4875   emit_int16(0x31, (0xC0 | encode));
 4876 }
 4877 
 4878 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4879   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4880   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4882   emit_int16(0x32, (0xC0 | encode));
 4883 }
 4884 
 4885 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4886   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4887          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4888              VM_Version::supports_evex(), "");
 4889   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4890   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4891   emit_int16(0x21, (0xC0 | encode));
 4892 }
 4893 
 4894 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4895   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4896          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4897              VM_Version::supports_evex(), "");
 4898   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4899   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4900   emit_int16(0x22, (0xC0 | encode));
 4901 }
 4902 
 4903 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4904   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4905          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4906              VM_Version::supports_evex(), "");
 4907   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4908   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4909   emit_int16(0x23, (0xC0 | encode));
 4910 }
 4911 
 4912 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4913   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4914          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4915              VM_Version::supports_evex(), "");
 4916   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4917   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4918   emit_int16(0x24, (0xC0 | encode));
 4919 }
 4920 
 4921 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4922   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4923          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4924              VM_Version::supports_evex(), "");
 4925   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4926   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4927   emit_int16(0x25, (0xC0 | encode));
 4928 }
 4929 
 4930 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4931   assert(VM_Version::supports_avx512vlbw(), "");
 4932   assert(src != xnoreg, "sanity");
 4933   InstructionMark im(this);
 4934   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4935   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4936   attributes.set_is_evex_instruction();
 4937   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4938   emit_int8(0x30);
 4939   emit_operand(src, dst, 0);
 4940 }
 4941 
 4942 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4943   assert(VM_Version::supports_avx512vlbw(), "");
 4944   assert(src != xnoreg, "sanity");
 4945   InstructionMark im(this);
 4946   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4947   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4948   attributes.reset_is_clear_context();
 4949   attributes.set_embedded_opmask_register_specifier(mask);
 4950   attributes.set_is_evex_instruction();
 4951   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4952   emit_int8(0x30);
 4953   emit_operand(src, dst, 0);
 4954 }
 4955 
 4956 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4957   assert(VM_Version::supports_evex(), "");
 4958   assert(src != xnoreg, "sanity");
 4959   InstructionMark im(this);
 4960   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4961   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4962   attributes.set_is_evex_instruction();
 4963   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4964   emit_int8(0x31);
 4965   emit_operand(src, dst, 0);
 4966 }
 4967 
 4968 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4969   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4970   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4971   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4972   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4973   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4974   emit_int16(0x33, (0xC0 | encode));
 4975 }
 4976 
 4977 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4978   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4979   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4980   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4981   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4982   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4983   emit_int16(0x34, (0xC0 | encode));
 4984 }
 4985 
 4986 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4989   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4990   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4991 }
 4992 
 4993 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4994   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4995     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4996     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4997   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4998   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4999   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5000 }
 5001 
 5002 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5003 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5004        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5005        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5006   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5007   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5008   emit_int16(0x04, (0xC0 | encode));
 5009 }
 5010 
 5011 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5012   assert(VM_Version::supports_evex(), "");
 5013   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 5014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5015   attributes.set_is_evex_instruction();
 5016   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5017   emit_int16(0x52, (0xC0 | encode));
 5018 }
 5019 
 5020 // generic
 5021 void Assembler::pop(Register dst) {
 5022   int encode = prefix_and_encode(dst->encoding());
 5023   emit_int8(0x58 | encode);
 5024 }
 5025 
 5026 void Assembler::popcntl(Register dst, Address src) {
 5027   assert(VM_Version::supports_popcnt(), "must support");
 5028   InstructionMark im(this);
 5029   emit_int8((unsigned char)0xF3);
 5030   prefix(src, dst);
 5031   emit_int16(0x0F, (unsigned char)0xB8);
 5032   emit_operand(dst, src, 0);
 5033 }
 5034 
 5035 void Assembler::popcntl(Register dst, Register src) {
 5036   assert(VM_Version::supports_popcnt(), "must support");
 5037   emit_int8((unsigned char)0xF3);
 5038   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5039   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 5040 }
 5041 
 5042 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5043   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5044   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5045   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5046   attributes.set_embedded_opmask_register_specifier(mask);
 5047   attributes.set_is_evex_instruction();
 5048   if (merge) {
 5049     attributes.reset_is_clear_context();
 5050   }
 5051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5052   emit_int16(0x54, (0xC0 | encode));
 5053 }
 5054 
 5055 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5056   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5057   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5058   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5059   attributes.set_is_evex_instruction();
 5060   attributes.set_embedded_opmask_register_specifier(mask);
 5061   if (merge) {
 5062     attributes.reset_is_clear_context();
 5063   }
 5064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5065   emit_int16(0x54, (0xC0 | encode));
 5066 }
 5067 
 5068 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5069   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5070   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5072   attributes.set_is_evex_instruction();
 5073   attributes.set_embedded_opmask_register_specifier(mask);
 5074   if (merge) {
 5075     attributes.reset_is_clear_context();
 5076   }
 5077   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5078   emit_int16(0x55, (0xC0 | encode));
 5079 }
 5080 
 5081 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5082   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5083   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5084   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5085   attributes.set_is_evex_instruction();
 5086   attributes.set_embedded_opmask_register_specifier(mask);
 5087   if (merge) {
 5088     attributes.reset_is_clear_context();
 5089   }
 5090   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5091   emit_int16(0x55, (0xC0 | encode));
 5092 }
 5093 
 5094 void Assembler::popf() {
 5095   emit_int8((unsigned char)0x9D);
 5096 }
 5097 
 5098 #ifndef _LP64 // no 32bit push/pop on amd64
 5099 void Assembler::popl(Address dst) {
 5100   // NOTE: this will adjust stack by 8byte on 64bits
 5101   InstructionMark im(this);
 5102   prefix(dst);
 5103   emit_int8((unsigned char)0x8F);
 5104   emit_operand(rax, dst, 0);
 5105 }
 5106 #endif
 5107 
 5108 void Assembler::prefetchnta(Address src) {
 5109   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5110   InstructionMark im(this);
 5111   prefix(src);
 5112   emit_int16(0x0F, 0x18);
 5113   emit_operand(rax, src, 0); // 0, src
 5114 }
 5115 
 5116 void Assembler::prefetchr(Address src) {
 5117   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5118   InstructionMark im(this);
 5119   prefix(src);
 5120   emit_int16(0x0F, 0x0D);
 5121   emit_operand(rax, src, 0); // 0, src
 5122 }
 5123 
 5124 void Assembler::prefetcht0(Address src) {
 5125   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5126   InstructionMark im(this);
 5127   prefix(src);
 5128   emit_int16(0x0F, 0x18);
 5129   emit_operand(rcx, src, 0); // 1, src
 5130 }
 5131 
 5132 void Assembler::prefetcht1(Address src) {
 5133   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5134   InstructionMark im(this);
 5135   prefix(src);
 5136   emit_int16(0x0F, 0x18);
 5137   emit_operand(rdx, src, 0); // 2, src
 5138 }
 5139 
 5140 void Assembler::prefetcht2(Address src) {
 5141   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5142   InstructionMark im(this);
 5143   prefix(src);
 5144   emit_int16(0x0F, 0x18);
 5145   emit_operand(rbx, src, 0); // 3, src
 5146 }
 5147 
 5148 void Assembler::prefetchw(Address src) {
 5149   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5150   InstructionMark im(this);
 5151   prefix(src);
 5152   emit_int16(0x0F, 0x0D);
 5153   emit_operand(rcx, src, 0); // 1, src
 5154 }
 5155 
 5156 void Assembler::prefix(Prefix p) {
 5157   emit_int8(p);
 5158 }
 5159 
 5160 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5161   assert(VM_Version::supports_ssse3(), "");
 5162   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5163   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5164   emit_int16(0x00, (0xC0 | encode));
 5165 }
 5166 
 5167 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 5168   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 5169   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5170   attributes.set_is_evex_instruction();
 5171   attributes.set_embedded_opmask_register_specifier(mask);
 5172   if (merge) {
 5173     attributes.reset_is_clear_context();
 5174   }
 5175   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5176   emit_int16(0x00, (0xC0 | encode));
 5177 }
 5178 
 5179 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5180   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5181          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5182          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5183   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5184   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5185   emit_int16(0x00, (0xC0 | encode));
 5186 }
 5187 
 5188 void Assembler::pshufb(XMMRegister dst, Address src) {
 5189   assert(VM_Version::supports_ssse3(), "");
 5190   InstructionMark im(this);
 5191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5192   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5193   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5194   emit_int8(0x00);
 5195   emit_operand(dst, src, 0);
 5196 }
 5197 
 5198 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5199   assert(isByte(mode), "invalid value");
 5200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5201   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5202   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5203   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5204   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5205 }
 5206 
 5207 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5208   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5209          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5210          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5211   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5212   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5213   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5214   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5215 }
 5216 
 5217 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5218   assert(isByte(mode), "invalid value");
 5219   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5220   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5221   InstructionMark im(this);
 5222   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5223   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5224   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5225   emit_int8(0x70);
 5226   emit_operand(dst, src, 1);
 5227   emit_int8(mode & 0xFF);
 5228 }
 5229 
 5230 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5231   assert(isByte(mode), "invalid value");
 5232   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5233   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5234   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5235   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5236 }
 5237 
 5238 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5239   assert(isByte(mode), "invalid value");
 5240   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5242   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5243   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5244 }
 5245 
 5246 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5247   assert(isByte(mode), "invalid value");
 5248   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5249   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5250   InstructionMark im(this);
 5251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5252   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5253   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5254   emit_int8(0x70);
 5255   emit_operand(dst, src, 1);
 5256   emit_int8(mode & 0xFF);
 5257 }
 5258 
 5259 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5260   assert(VM_Version::supports_evex(), "requires EVEX support");
 5261   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5262   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5263   attributes.set_is_evex_instruction();
 5264   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5265   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5266 }
 5267 
 5268 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5269   assert(isByte(imm8), "invalid value");
 5270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5272   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5273   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5274 }
 5275 
 5276 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5277   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5278   attributes.set_rex_vex_w_reverted();
 5279   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5280   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5281 }
 5282 
 5283 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
 5284   assert(isByte(imm8), "invalid value");
 5285   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5287   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5288   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5289 }
 5290 
 5291 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5292   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5294   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5295 }
 5296 
 5297 void Assembler::psrldq(XMMRegister dst, int shift) {
 5298   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5301   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5302   emit_int24(0x73, (0xC0 | encode), shift);
 5303 }
 5304 
 5305 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5306   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5307          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5308          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5309   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5310   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5311   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5312 }
 5313 
 5314 void Assembler::pslldq(XMMRegister dst, int shift) {
 5315   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5318   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5319   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5320   emit_int24(0x73, (0xC0 | encode), shift);
 5321 }
 5322 
 5323 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5324   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5325          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5326          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5327   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5328   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5329   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5330 }
 5331 
 5332 void Assembler::ptest(XMMRegister dst, Address src) {
 5333   assert(VM_Version::supports_sse4_1(), "");
 5334   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5335   InstructionMark im(this);
 5336   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5337   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5338   emit_int8(0x17);
 5339   emit_operand(dst, src, 0);
 5340 }
 5341 
 5342 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5343   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5344   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5345   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5346   emit_int8(0x17);
 5347   emit_int8((0xC0 | encode));
 5348 }
 5349 
 5350 void Assembler::vptest(XMMRegister dst, Address src) {
 5351   assert(VM_Version::supports_avx(), "");
 5352   InstructionMark im(this);
 5353   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5354   assert(dst != xnoreg, "sanity");
 5355   // swap src<->dst for encoding
 5356   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5357   emit_int8(0x17);
 5358   emit_operand(dst, src, 0);
 5359 }
 5360 
 5361 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5362   assert(VM_Version::supports_avx(), "");
 5363   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5364   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5365   emit_int16(0x17, (0xC0 | encode));
 5366 }
 5367 
 5368 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5369   assert(VM_Version::supports_avx(), "");
 5370   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5371   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5372   emit_int16(0x17, (0xC0 | encode));
 5373 }
 5374 
 5375 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5376   assert(VM_Version::supports_avx512vlbw(), "");
 5377   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5379   attributes.set_is_evex_instruction();
 5380   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5381   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5382 }
 5383 
 5384 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5386   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5387   InstructionMark im(this);
 5388   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5389   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5390   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5391   emit_int8(0x60);
 5392   emit_operand(dst, src, 0);
 5393 }
 5394 
 5395 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5397   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5398   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5399   emit_int16(0x60, (0xC0 | encode));
 5400 }
 5401 
 5402 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5403   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5404   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5405   InstructionMark im(this);
 5406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5407   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5408   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5409   emit_int8(0x62);
 5410   emit_operand(dst, src, 0);
 5411 }
 5412 
 5413 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5414   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5415   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5416   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5417   emit_int16(0x62, (0xC0 | encode));
 5418 }
 5419 
 5420 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5422   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5423   attributes.set_rex_vex_w_reverted();
 5424   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5425   emit_int16(0x6C, (0xC0 | encode));
 5426 }
 5427 
 5428 void Assembler::push(int32_t imm32) {
 5429   // in 64bits we push 64bits onto the stack but only
 5430   // take a 32bit immediate
 5431   emit_int8(0x68);
 5432   emit_int32(imm32);
 5433 }
 5434 
 5435 void Assembler::push(Register src) {
 5436   int encode = prefix_and_encode(src->encoding());
 5437   emit_int8(0x50 | encode);
 5438 }
 5439 
 5440 void Assembler::pushf() {
 5441   emit_int8((unsigned char)0x9C);
 5442 }
 5443 
 5444 #ifndef _LP64 // no 32bit push/pop on amd64
 5445 void Assembler::pushl(Address src) {
 5446   // Note this will push 64bit on 64bit
 5447   InstructionMark im(this);
 5448   prefix(src);
 5449   emit_int8((unsigned char)0xFF);
 5450   emit_operand(rsi, src, 0);
 5451 }
 5452 #endif
 5453 
 5454 void Assembler::rcll(Register dst, int imm8) {
 5455   assert(isShiftCount(imm8), "illegal shift count");
 5456   int encode = prefix_and_encode(dst->encoding());
 5457   if (imm8 == 1) {
 5458     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5459   } else {
 5460     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5461   }
 5462 }
 5463 
 5464 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5465   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5467   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5468   emit_int16(0x53, (0xC0 | encode));
 5469 }
 5470 
 5471 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5472   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5473   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5474   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5475   emit_int16(0x53, (0xC0 | encode));
 5476 }
 5477 
 5478 void Assembler::rdtsc() {
 5479   emit_int16(0x0F, 0x31);
 5480 }
 5481 
 5482 // copies data from [esi] to [edi] using rcx pointer sized words
 5483 // generic
 5484 void Assembler::rep_mov() {
 5485   // REP
 5486   // MOVSQ
 5487   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5488   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5489 }
 5490 
 5491 // sets rcx bytes with rax, value at [edi]
 5492 void Assembler::rep_stosb() {
 5493   // REP
 5494   // STOSB
 5495   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5496   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5497 }
 5498 
 5499 // sets rcx pointer sized words with rax, value at [edi]
 5500 // generic
 5501 void Assembler::rep_stos() {
 5502   // REP
 5503   // LP64:STOSQ, LP32:STOSD
 5504   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5505   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5506 }
 5507 
 5508 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5509 // generic
 5510 void Assembler::repne_scan() { // repne_scan
 5511   // SCASQ
 5512   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5513   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5514 }
 5515 
 5516 #ifdef _LP64
 5517 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5518 // generic
 5519 void Assembler::repne_scanl() { // repne_scan
 5520   // SCASL
 5521   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5522 }
 5523 #endif
 5524 
 5525 void Assembler::ret(int imm16) {
 5526   if (imm16 == 0) {
 5527     emit_int8((unsigned char)0xC3);
 5528   } else {
 5529     emit_int8((unsigned char)0xC2);
 5530     emit_int16(imm16);
 5531   }
 5532 }
 5533 
 5534 void Assembler::roll(Register dst, int imm8) {
 5535   assert(isShiftCount(imm8), "illegal shift count");
 5536   int encode = prefix_and_encode(dst->encoding());
 5537   if (imm8 == 1) {
 5538     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5539   } else {
 5540     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5541   }
 5542 }
 5543 
 5544 void Assembler::roll(Register dst) {
 5545   int encode = prefix_and_encode(dst->encoding());
 5546   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5547 }
 5548 
 5549 void Assembler::rorl(Register dst, int imm8) {
 5550   assert(isShiftCount(imm8), "illegal shift count");
 5551   int encode = prefix_and_encode(dst->encoding());
 5552   if (imm8 == 1) {
 5553     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5554   } else {
 5555     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5556   }
 5557 }
 5558 
 5559 void Assembler::rorl(Register dst) {
 5560   int encode = prefix_and_encode(dst->encoding());
 5561   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5562 }
 5563 
 5564 #ifdef _LP64
 5565 void Assembler::rorq(Register dst) {
 5566   int encode = prefixq_and_encode(dst->encoding());
 5567   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5568 }
 5569 
 5570 void Assembler::rorq(Register dst, int imm8) {
 5571   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5572   int encode = prefixq_and_encode(dst->encoding());
 5573   if (imm8 == 1) {
 5574     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5575   } else {
 5576     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5577   }
 5578 }
 5579 
 5580 void Assembler::rolq(Register dst) {
 5581   int encode = prefixq_and_encode(dst->encoding());
 5582   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5583 }
 5584 
 5585 void Assembler::rolq(Register dst, int imm8) {
 5586   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5587   int encode = prefixq_and_encode(dst->encoding());
 5588   if (imm8 == 1) {
 5589     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5590   } else {
 5591     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5592   }
 5593 }
 5594 #endif
 5595 
 5596 void Assembler::sahf() {
 5597 #ifdef _LP64
 5598   // Not supported in 64bit mode
 5599   ShouldNotReachHere();
 5600 #endif
 5601   emit_int8((unsigned char)0x9E);
 5602 }
 5603 
 5604 void Assembler::sall(Address dst, int imm8) {
 5605   InstructionMark im(this);
 5606   assert(isShiftCount(imm8), "illegal shift count");
 5607   prefix(dst);
 5608   if (imm8 == 1) {
 5609     emit_int8((unsigned char)0xD1);
 5610     emit_operand(as_Register(4), dst, 0);
 5611   }
 5612   else {
 5613     emit_int8((unsigned char)0xC1);
 5614     emit_operand(as_Register(4), dst, 1);
 5615     emit_int8(imm8);
 5616   }
 5617 }
 5618 
 5619 void Assembler::sall(Address dst) {
 5620   InstructionMark im(this);
 5621   prefix(dst);
 5622   emit_int8((unsigned char)0xD3);
 5623   emit_operand(as_Register(4), dst, 0);
 5624 }
 5625 
 5626 void Assembler::sall(Register dst, int imm8) {
 5627   assert(isShiftCount(imm8), "illegal shift count");
 5628   int encode = prefix_and_encode(dst->encoding());
 5629   if (imm8 == 1) {
 5630     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5631   } else {
 5632     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5633   }
 5634 }
 5635 
 5636 void Assembler::sall(Register dst) {
 5637   int encode = prefix_and_encode(dst->encoding());
 5638   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5639 }
 5640 
 5641 void Assembler::sarl(Address dst, int imm8) {
 5642   assert(isShiftCount(imm8), "illegal shift count");
 5643   InstructionMark im(this);
 5644   prefix(dst);
 5645   if (imm8 == 1) {
 5646     emit_int8((unsigned char)0xD1);
 5647     emit_operand(as_Register(7), dst, 0);
 5648   }
 5649   else {
 5650     emit_int8((unsigned char)0xC1);
 5651     emit_operand(as_Register(7), dst, 1);
 5652     emit_int8(imm8);
 5653   }
 5654 }
 5655 
 5656 void Assembler::sarl(Address dst) {
 5657   InstructionMark im(this);
 5658   prefix(dst);
 5659   emit_int8((unsigned char)0xD3);
 5660   emit_operand(as_Register(7), dst, 0);
 5661 }
 5662 
 5663 void Assembler::sarl(Register dst, int imm8) {
 5664   int encode = prefix_and_encode(dst->encoding());
 5665   assert(isShiftCount(imm8), "illegal shift count");
 5666   if (imm8 == 1) {
 5667     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5668   } else {
 5669     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5670   }
 5671 }
 5672 
 5673 void Assembler::sarl(Register dst) {
 5674   int encode = prefix_and_encode(dst->encoding());
 5675   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5676 }
 5677 
 5678 void Assembler::sbbl(Address dst, int32_t imm32) {
 5679   InstructionMark im(this);
 5680   prefix(dst);
 5681   emit_arith_operand(0x81, rbx, dst, imm32);
 5682 }
 5683 
 5684 void Assembler::sbbl(Register dst, int32_t imm32) {
 5685   prefix(dst);
 5686   emit_arith(0x81, 0xD8, dst, imm32);
 5687 }
 5688 
 5689 
 5690 void Assembler::sbbl(Register dst, Address src) {
 5691   InstructionMark im(this);
 5692   prefix(src, dst);
 5693   emit_int8(0x1B);
 5694   emit_operand(dst, src, 0);
 5695 }
 5696 
 5697 void Assembler::sbbl(Register dst, Register src) {
 5698   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5699   emit_arith(0x1B, 0xC0, dst, src);
 5700 }
 5701 
 5702 void Assembler::setb(Condition cc, Register dst) {
 5703   assert(0 <= cc && cc < 16, "illegal cc");
 5704   int encode = prefix_and_encode(dst->encoding(), true);
 5705   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5706 }
 5707 
 5708 void Assembler::sete(Register dst) {
 5709   int encode = prefix_and_encode(dst->encoding(), true);
 5710   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5711 }
 5712 
 5713 void Assembler::setl(Register dst) {
 5714   int encode = prefix_and_encode(dst->encoding(), true);
 5715   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5716 }
 5717 
 5718 void Assembler::setne(Register dst) {
 5719   int encode = prefix_and_encode(dst->encoding(), true);
 5720   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5721 }
 5722 
 5723 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5724   assert(VM_Version::supports_ssse3(), "");
 5725   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5726   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5727   emit_int24(0x0F, (0xC0 | encode), imm8);
 5728 }
 5729 
 5730 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5731   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5732          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5733          0, "");
 5734   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5735   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5736   emit_int24(0x0F, (0xC0 | encode), imm8);
 5737 }
 5738 
 5739 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5740   assert(VM_Version::supports_evex(), "");
 5741   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5742   attributes.set_is_evex_instruction();
 5743   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5744   emit_int24(0x3, (0xC0 | encode), imm8);
 5745 }
 5746 
 5747 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5748   assert(VM_Version::supports_sse4_1(), "");
 5749   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5750   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5751   emit_int24(0x0E, (0xC0 | encode), imm8);
 5752 }
 5753 
 5754 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5755   assert(VM_Version::supports_sha(), "");
 5756   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5757   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5758 }
 5759 
 5760 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5761   assert(VM_Version::supports_sha(), "");
 5762   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5763   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5764 }
 5765 
 5766 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5767   assert(VM_Version::supports_sha(), "");
 5768   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5769   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5770 }
 5771 
 5772 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5773   assert(VM_Version::supports_sha(), "");
 5774   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5775   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5776 }
 5777 
 5778 // xmm0 is implicit additional source to this instruction.
 5779 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5780   assert(VM_Version::supports_sha(), "");
 5781   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5782   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5783 }
 5784 
 5785 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5786   assert(VM_Version::supports_sha(), "");
 5787   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5788   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5789 }
 5790 
 5791 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5792   assert(VM_Version::supports_sha(), "");
 5793   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5794   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5795 }
 5796 
 5797 
 5798 void Assembler::shll(Register dst, int imm8) {
 5799   assert(isShiftCount(imm8), "illegal shift count");
 5800   int encode = prefix_and_encode(dst->encoding());
 5801   if (imm8 == 1 ) {
 5802     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5803   } else {
 5804     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5805   }
 5806 }
 5807 
 5808 void Assembler::shll(Register dst) {
 5809   int encode = prefix_and_encode(dst->encoding());
 5810   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5811 }
 5812 
 5813 void Assembler::shrl(Register dst, int imm8) {
 5814   assert(isShiftCount(imm8), "illegal shift count");
 5815   int encode = prefix_and_encode(dst->encoding());
 5816   if (imm8 == 1) {
 5817     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5818   }
 5819   else {
 5820     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5821   }
 5822 }
 5823 
 5824 void Assembler::shrl(Register dst) {
 5825   int encode = prefix_and_encode(dst->encoding());
 5826   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5827 }
 5828 
 5829 void Assembler::shrl(Address dst) {
 5830   InstructionMark im(this);
 5831   prefix(dst);
 5832   emit_int8((unsigned char)0xD3);
 5833   emit_operand(as_Register(5), dst, 0);
 5834 }
 5835 
 5836 void Assembler::shrl(Address dst, int imm8) {
 5837   InstructionMark im(this);
 5838   assert(isShiftCount(imm8), "illegal shift count");
 5839   prefix(dst);
 5840   if (imm8 == 1) {
 5841     emit_int8((unsigned char)0xD1);
 5842     emit_operand(as_Register(5), dst, 0);
 5843   }
 5844   else {
 5845     emit_int8((unsigned char)0xC1);
 5846     emit_operand(as_Register(5), dst, 1);
 5847     emit_int8(imm8);
 5848   }
 5849 }
 5850 
 5851 
 5852 void Assembler::shldl(Register dst, Register src) {
 5853   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5854   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5855 }
 5856 
 5857 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5858   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5859   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5860 }
 5861 
 5862 void Assembler::shrdl(Register dst, Register src) {
 5863   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5864   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5865 }
 5866 
 5867 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5868   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5869   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5870 }
 5871 
 5872 // copies a single word from [esi] to [edi]
 5873 void Assembler::smovl() {
 5874   emit_int8((unsigned char)0xA5);
 5875 }
 5876 
 5877 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5878   assert(VM_Version::supports_sse4_1(), "");
 5879   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5880   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5881   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5882 }
 5883 
 5884 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5885   assert(VM_Version::supports_sse4_1(), "");
 5886   InstructionMark im(this);
 5887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5888   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5889   emit_int8(0x0B);
 5890   emit_operand(dst, src, 1);
 5891   emit_int8((unsigned char)rmode);
 5892 }
 5893 
 5894 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5895   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5896   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5897   attributes.set_rex_vex_w_reverted();
 5898   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5899   emit_int16(0x51, (0xC0 | encode));
 5900 }
 5901 
 5902 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5903   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5904   InstructionMark im(this);
 5905   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5906   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5907   attributes.set_rex_vex_w_reverted();
 5908   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5909   emit_int8(0x51);
 5910   emit_operand(dst, src, 0);
 5911 }
 5912 
 5913 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5914   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5915   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5916   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5917   emit_int16(0x51, (0xC0 | encode));
 5918 }
 5919 
 5920 void Assembler::std() {
 5921   emit_int8((unsigned char)0xFD);
 5922 }
 5923 
 5924 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5925   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5926   InstructionMark im(this);
 5927   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5928   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5929   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5930   emit_int8(0x51);
 5931   emit_operand(dst, src, 0);
 5932 }
 5933 
 5934 void Assembler::stmxcsr( Address dst) {
 5935   if (UseAVX > 0 ) {
 5936     assert(VM_Version::supports_avx(), "");
 5937     InstructionMark im(this);
 5938     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5939     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5940     emit_int8((unsigned char)0xAE);
 5941     emit_operand(as_Register(3), dst, 0);
 5942   } else {
 5943     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5944     InstructionMark im(this);
 5945     prefix(dst);
 5946     emit_int16(0x0F, (unsigned char)0xAE);
 5947     emit_operand(as_Register(3), dst, 0);
 5948   }
 5949 }
 5950 
 5951 void Assembler::subl(Address dst, int32_t imm32) {
 5952   InstructionMark im(this);
 5953   prefix(dst);
 5954   emit_arith_operand(0x81, rbp, dst, imm32);
 5955 }
 5956 
 5957 void Assembler::subl(Address dst, Register src) {
 5958   InstructionMark im(this);
 5959   prefix(dst, src);
 5960   emit_int8(0x29);
 5961   emit_operand(src, dst, 0);
 5962 }
 5963 
 5964 void Assembler::subl(Register dst, int32_t imm32) {
 5965   prefix(dst);
 5966   emit_arith(0x81, 0xE8, dst, imm32);
 5967 }
 5968 
 5969 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5970 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5971   prefix(dst);
 5972   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5973 }
 5974 
 5975 void Assembler::subl(Register dst, Address src) {
 5976   InstructionMark im(this);
 5977   prefix(src, dst);
 5978   emit_int8(0x2B);
 5979   emit_operand(dst, src, 0);
 5980 }
 5981 
 5982 void Assembler::subl(Register dst, Register src) {
 5983   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5984   emit_arith(0x2B, 0xC0, dst, src);
 5985 }
 5986 
 5987 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5988   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5989   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5990   attributes.set_rex_vex_w_reverted();
 5991   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5992   emit_int16(0x5C, (0xC0 | encode));
 5993 }
 5994 
 5995 void Assembler::subsd(XMMRegister dst, Address src) {
 5996   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5997   InstructionMark im(this);
 5998   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5999   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6000   attributes.set_rex_vex_w_reverted();
 6001   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6002   emit_int8(0x5C);
 6003   emit_operand(dst, src, 0);
 6004 }
 6005 
 6006 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 6007   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6008   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 6009   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6010   emit_int16(0x5C, (0xC0 | encode));
 6011 }
 6012 
 6013 void Assembler::subss(XMMRegister dst, Address src) {
 6014   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6015   InstructionMark im(this);
 6016   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6017   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6018   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6019   emit_int8(0x5C);
 6020   emit_operand(dst, src, 0);
 6021 }
 6022 
 6023 void Assembler::testb(Register dst, int imm8) {
 6024   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 6025   if (dst == rax) {
 6026     emit_int8((unsigned char)0xA8);
 6027     emit_int8(imm8);
 6028   } else {
 6029     (void) prefix_and_encode(dst->encoding(), true);
 6030     emit_arith_b(0xF6, 0xC0, dst, imm8);
 6031   }
 6032 }
 6033 
 6034 void Assembler::testb(Address dst, int imm8) {
 6035   InstructionMark im(this);
 6036   prefix(dst);
 6037   emit_int8((unsigned char)0xF6);
 6038   emit_operand(rax, dst, 1);
 6039   emit_int8(imm8);
 6040 }
 6041 
 6042 void Assembler::testl(Address dst, int32_t imm32) {
 6043   InstructionMark im(this);
 6044   prefix(dst);
 6045   emit_int8((unsigned char)0xF7);
 6046   emit_operand(as_Register(0), dst, 4);
 6047   emit_int32(imm32);
 6048 }
 6049 
 6050 void Assembler::testl(Register dst, int32_t imm32) {
 6051   // not using emit_arith because test
 6052   // doesn't support sign-extension of
 6053   // 8bit operands
 6054   if (dst == rax) {
 6055     emit_int8((unsigned char)0xA9);
 6056     emit_int32(imm32);
 6057   } else {
 6058     int encode = dst->encoding();
 6059     encode = prefix_and_encode(encode);
 6060     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 6061     emit_int32(imm32);
 6062   }
 6063 }
 6064 
 6065 void Assembler::testl(Register dst, Register src) {
 6066   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6067   emit_arith(0x85, 0xC0, dst, src);
 6068 }
 6069 
 6070 void Assembler::testl(Register dst, Address src) {
 6071   InstructionMark im(this);
 6072   prefix(src, dst);
 6073   emit_int8((unsigned char)0x85);
 6074   emit_operand(dst, src, 0);
 6075 }
 6076 
 6077 void Assembler::tzcntl(Register dst, Register src) {
 6078   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6079   emit_int8((unsigned char)0xF3);
 6080   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6081   emit_int24(0x0F,
 6082              (unsigned char)0xBC,
 6083              0xC0 | encode);
 6084 }
 6085 
 6086 void Assembler::tzcntl(Register dst, Address src) {
 6087   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6088   InstructionMark im(this);
 6089   emit_int8((unsigned char)0xF3);
 6090   prefix(src, dst);
 6091   emit_int16(0x0F, (unsigned char)0xBC);
 6092   emit_operand(dst, src, 0);
 6093 }
 6094 
 6095 void Assembler::tzcntq(Register dst, Register src) {
 6096   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6097   emit_int8((unsigned char)0xF3);
 6098   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 6099   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 6100 }
 6101 
 6102 void Assembler::tzcntq(Register dst, Address src) {
 6103   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6104   InstructionMark im(this);
 6105   emit_int8((unsigned char)0xF3);
 6106   prefixq(src, dst);
 6107   emit_int16(0x0F, (unsigned char)0xBC);
 6108   emit_operand(dst, src, 0);
 6109 }
 6110 
 6111 void Assembler::ucomisd(XMMRegister dst, Address src) {
 6112   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6113   InstructionMark im(this);
 6114   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6115   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6116   attributes.set_rex_vex_w_reverted();
 6117   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6118   emit_int8(0x2E);
 6119   emit_operand(dst, src, 0);
 6120 }
 6121 
 6122 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6123   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6124   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6125   attributes.set_rex_vex_w_reverted();
 6126   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6127   emit_int16(0x2E, (0xC0 | encode));
 6128 }
 6129 
 6130 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6132   InstructionMark im(this);
 6133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6134   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6135   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6136   emit_int8(0x2E);
 6137   emit_operand(dst, src, 0);
 6138 }
 6139 
 6140 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6141   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6142   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6143   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6144   emit_int16(0x2E, (0xC0 | encode));
 6145 }
 6146 
 6147 void Assembler::xabort(int8_t imm8) {
 6148   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6149 }
 6150 
 6151 void Assembler::xaddb(Address dst, Register src) {
 6152   InstructionMark im(this);
 6153   prefix(dst, src, true);
 6154   emit_int16(0x0F, (unsigned char)0xC0);
 6155   emit_operand(src, dst, 0);
 6156 }
 6157 
 6158 void Assembler::xaddw(Address dst, Register src) {
 6159   InstructionMark im(this);
 6160   emit_int8(0x66);
 6161   prefix(dst, src);
 6162   emit_int16(0x0F, (unsigned char)0xC1);
 6163   emit_operand(src, dst, 0);
 6164 }
 6165 
 6166 void Assembler::xaddl(Address dst, Register src) {
 6167   InstructionMark im(this);
 6168   prefix(dst, src);
 6169   emit_int16(0x0F, (unsigned char)0xC1);
 6170   emit_operand(src, dst, 0);
 6171 }
 6172 
 6173 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6174   InstructionMark im(this);
 6175   relocate(rtype);
 6176   if (abort.is_bound()) {
 6177     address entry = target(abort);
 6178     assert(entry != NULL, "abort entry NULL");
 6179     intptr_t offset = entry - pc();
 6180     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6181     emit_int32(offset - 6); // 2 opcode + 4 address
 6182   } else {
 6183     abort.add_patch_at(code(), locator());
 6184     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6185     emit_int32(0);
 6186   }
 6187 }
 6188 
 6189 void Assembler::xchgb(Register dst, Address src) { // xchg
 6190   InstructionMark im(this);
 6191   prefix(src, dst, true);
 6192   emit_int8((unsigned char)0x86);
 6193   emit_operand(dst, src, 0);
 6194 }
 6195 
 6196 void Assembler::xchgw(Register dst, Address src) { // xchg
 6197   InstructionMark im(this);
 6198   emit_int8(0x66);
 6199   prefix(src, dst);
 6200   emit_int8((unsigned char)0x87);
 6201   emit_operand(dst, src, 0);
 6202 }
 6203 
 6204 void Assembler::xchgl(Register dst, Address src) { // xchg
 6205   InstructionMark im(this);
 6206   prefix(src, dst);
 6207   emit_int8((unsigned char)0x87);
 6208   emit_operand(dst, src, 0);
 6209 }
 6210 
 6211 void Assembler::xchgl(Register dst, Register src) {
 6212   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6213   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6214 }
 6215 
 6216 void Assembler::xend() {
 6217   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6218 }
 6219 
 6220 void Assembler::xgetbv() {
 6221   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6222 }
 6223 
 6224 void Assembler::xorl(Address dst, int32_t imm32) {
 6225   InstructionMark im(this);
 6226   prefix(dst);
 6227   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6228 }
 6229 
 6230 void Assembler::xorl(Register dst, int32_t imm32) {
 6231   prefix(dst);
 6232   emit_arith(0x81, 0xF0, dst, imm32);
 6233 }
 6234 
 6235 void Assembler::xorl(Register dst, Address src) {
 6236   InstructionMark im(this);
 6237   prefix(src, dst);
 6238   emit_int8(0x33);
 6239   emit_operand(dst, src, 0);
 6240 }
 6241 
 6242 void Assembler::xorl(Register dst, Register src) {
 6243   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6244   emit_arith(0x33, 0xC0, dst, src);
 6245 }
 6246 
 6247 void Assembler::xorl(Address dst, Register src) {
 6248   InstructionMark im(this);
 6249   prefix(dst, src);
 6250   emit_int8(0x31);
 6251   emit_operand(src, dst, 0);
 6252 }
 6253 
 6254 void Assembler::xorb(Register dst, Address src) {
 6255   InstructionMark im(this);
 6256   prefix(src, dst);
 6257   emit_int8(0x32);
 6258   emit_operand(dst, src, 0);
 6259 }
 6260 
 6261 void Assembler::xorb(Address dst, Register src) {
 6262   InstructionMark im(this);
 6263   prefix(dst, src, true);
 6264   emit_int8(0x30);
 6265   emit_operand(src, dst, 0);
 6266 }
 6267 
 6268 void Assembler::xorw(Register dst, Register src) {
 6269   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6270   emit_arith(0x33, 0xC0, dst, src);
 6271 }
 6272 
 6273 // AVX 3-operands scalar float-point arithmetic instructions
 6274 
 6275 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6276   assert(VM_Version::supports_avx(), "");
 6277   InstructionMark im(this);
 6278   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6279   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6280   attributes.set_rex_vex_w_reverted();
 6281   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6282   emit_int8(0x58);
 6283   emit_operand(dst, src, 0);
 6284 }
 6285 
 6286 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6287   assert(VM_Version::supports_avx(), "");
 6288   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6289   attributes.set_rex_vex_w_reverted();
 6290   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6291   emit_int16(0x58, (0xC0 | encode));
 6292 }
 6293 
 6294 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6295   assert(VM_Version::supports_avx(), "");
 6296   InstructionMark im(this);
 6297   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6298   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6299   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6300   emit_int8(0x58);
 6301   emit_operand(dst, src, 0);
 6302 }
 6303 
 6304 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6305   assert(VM_Version::supports_avx(), "");
 6306   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6307   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6308   emit_int16(0x58, (0xC0 | encode));
 6309 }
 6310 
 6311 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6312   assert(VM_Version::supports_avx(), "");
 6313   InstructionMark im(this);
 6314   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6315   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6316   attributes.set_rex_vex_w_reverted();
 6317   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6318   emit_int8(0x5E);
 6319   emit_operand(dst, src, 0);
 6320 }
 6321 
 6322 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6323   assert(VM_Version::supports_avx(), "");
 6324   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6325   attributes.set_rex_vex_w_reverted();
 6326   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6327   emit_int16(0x5E, (0xC0 | encode));
 6328 }
 6329 
 6330 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6331   assert(VM_Version::supports_avx(), "");
 6332   InstructionMark im(this);
 6333   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6334   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6335   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6336   emit_int8(0x5E);
 6337   emit_operand(dst, src, 0);
 6338 }
 6339 
 6340 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6341   assert(VM_Version::supports_avx(), "");
 6342   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6343   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6344   emit_int16(0x5E, (0xC0 | encode));
 6345 }
 6346 
 6347 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6348   assert(VM_Version::supports_fma(), "");
 6349   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6350   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6351   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6352 }
 6353 
 6354 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6355   assert(VM_Version::supports_fma(), "");
 6356   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6357   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6358   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6359 }
 6360 
 6361 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6362   assert(VM_Version::supports_avx(), "");
 6363   InstructionMark im(this);
 6364   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6365   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6366   attributes.set_rex_vex_w_reverted();
 6367   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6368   emit_int8(0x59);
 6369   emit_operand(dst, src, 0);
 6370 }
 6371 
 6372 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6373   assert(VM_Version::supports_avx(), "");
 6374   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6375   attributes.set_rex_vex_w_reverted();
 6376   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6377   emit_int16(0x59, (0xC0 | encode));
 6378 }
 6379 
 6380 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6381   assert(VM_Version::supports_avx(), "");
 6382   InstructionMark im(this);
 6383   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6384   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6385   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6386   emit_int8(0x59);
 6387   emit_operand(dst, src, 0);
 6388 }
 6389 
 6390 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6391   assert(VM_Version::supports_avx(), "");
 6392   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6393   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6394   emit_int16(0x59, (0xC0 | encode));
 6395 }
 6396 
 6397 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6398   assert(VM_Version::supports_avx(), "");
 6399   InstructionMark im(this);
 6400   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6401   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6402   attributes.set_rex_vex_w_reverted();
 6403   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6404   emit_int8(0x5C);
 6405   emit_operand(dst, src, 0);
 6406 }
 6407 
 6408 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6409   assert(VM_Version::supports_avx(), "");
 6410   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6411   attributes.set_rex_vex_w_reverted();
 6412   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6413   emit_int16(0x5C, (0xC0 | encode));
 6414 }
 6415 
 6416 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6417   assert(VM_Version::supports_avx(), "");
 6418   InstructionMark im(this);
 6419   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6420   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6421   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6422   emit_int8(0x5C);
 6423   emit_operand(dst, src, 0);
 6424 }
 6425 
 6426 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6427   assert(VM_Version::supports_avx(), "");
 6428   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6429   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6430   emit_int16(0x5C, (0xC0 | encode));
 6431 }
 6432 
 6433 //====================VECTOR ARITHMETIC=====================================
 6434 
 6435 // Float-point vector arithmetic
 6436 
 6437 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6439   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6440   attributes.set_rex_vex_w_reverted();
 6441   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6442   emit_int16(0x58, (0xC0 | encode));
 6443 }
 6444 
 6445 void Assembler::addpd(XMMRegister dst, Address src) {
 6446   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6447   InstructionMark im(this);
 6448   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6449   attributes.set_rex_vex_w_reverted();
 6450   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6451   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6452   emit_int8(0x58);
 6453   emit_operand(dst, src, 0);
 6454 }
 6455 
 6456 
 6457 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6458   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6459   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6460   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6461   emit_int16(0x58, (0xC0 | encode));
 6462 }
 6463 
 6464 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6465   assert(VM_Version::supports_avx(), "");
 6466   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6467   attributes.set_rex_vex_w_reverted();
 6468   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6469   emit_int16(0x58, (0xC0 | encode));
 6470 }
 6471 
 6472 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6473   assert(VM_Version::supports_avx(), "");
 6474   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6475   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6476   emit_int16(0x58, (0xC0 | encode));
 6477 }
 6478 
 6479 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6480   assert(VM_Version::supports_avx(), "");
 6481   InstructionMark im(this);
 6482   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6483   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6484   attributes.set_rex_vex_w_reverted();
 6485   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6486   emit_int8(0x58);
 6487   emit_operand(dst, src, 0);
 6488 }
 6489 
 6490 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6491   assert(VM_Version::supports_avx(), "");
 6492   InstructionMark im(this);
 6493   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6494   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6495   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6496   emit_int8(0x58);
 6497   emit_operand(dst, src, 0);
 6498 }
 6499 
 6500 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6501   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6502   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6503   attributes.set_rex_vex_w_reverted();
 6504   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6505   emit_int16(0x5C, (0xC0 | encode));
 6506 }
 6507 
 6508 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6509   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6510   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6511   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6512   emit_int16(0x5C, (0xC0 | encode));
 6513 }
 6514 
 6515 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6516   assert(VM_Version::supports_avx(), "");
 6517   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6518   attributes.set_rex_vex_w_reverted();
 6519   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6520   emit_int16(0x5C, (0xC0 | encode));
 6521 }
 6522 
 6523 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6524   assert(VM_Version::supports_avx(), "");
 6525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6526   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6527   emit_int16(0x5C, (0xC0 | encode));
 6528 }
 6529 
 6530 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6531   assert(VM_Version::supports_avx(), "");
 6532   InstructionMark im(this);
 6533   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6534   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6535   attributes.set_rex_vex_w_reverted();
 6536   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6537   emit_int8(0x5C);
 6538   emit_operand(dst, src, 0);
 6539 }
 6540 
 6541 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6542   assert(VM_Version::supports_avx(), "");
 6543   InstructionMark im(this);
 6544   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6545   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6546   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6547   emit_int8(0x5C);
 6548   emit_operand(dst, src, 0);
 6549 }
 6550 
 6551 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6552   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6553   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6554   attributes.set_rex_vex_w_reverted();
 6555   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6556   emit_int16(0x59, (0xC0 | encode));
 6557 }
 6558 
 6559 void Assembler::mulpd(XMMRegister dst, Address src) {
 6560   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6561   InstructionMark im(this);
 6562   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6563   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6564   attributes.set_rex_vex_w_reverted();
 6565   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6566   emit_int8(0x59);
 6567   emit_operand(dst, src, 0);
 6568 }
 6569 
 6570 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6571   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6573   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6574   emit_int16(0x59, (0xC0 | encode));
 6575 }
 6576 
 6577 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6578   assert(VM_Version::supports_avx(), "");
 6579   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6580   attributes.set_rex_vex_w_reverted();
 6581   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6582   emit_int16(0x59, (0xC0 | encode));
 6583 }
 6584 
 6585 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6586   assert(VM_Version::supports_avx(), "");
 6587   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6588   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6589   emit_int16(0x59, (0xC0 | encode));
 6590 }
 6591 
 6592 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6593   assert(VM_Version::supports_avx(), "");
 6594   InstructionMark im(this);
 6595   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6596   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6597   attributes.set_rex_vex_w_reverted();
 6598   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6599   emit_int8(0x59);
 6600   emit_operand(dst, src, 0);
 6601 }
 6602 
 6603 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6604   assert(VM_Version::supports_avx(), "");
 6605   InstructionMark im(this);
 6606   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6607   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6608   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6609   emit_int8(0x59);
 6610   emit_operand(dst, src, 0);
 6611 }
 6612 
 6613 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6614   assert(VM_Version::supports_fma(), "");
 6615   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6616   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6617   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6618 }
 6619 
 6620 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6621   assert(VM_Version::supports_fma(), "");
 6622   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6623   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6624   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6625 }
 6626 
 6627 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6628   assert(VM_Version::supports_fma(), "");
 6629   InstructionMark im(this);
 6630   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6631   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6632   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6633   emit_int8((unsigned char)0xB8);
 6634   emit_operand(dst, src2, 0);
 6635 }
 6636 
 6637 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6638   assert(VM_Version::supports_fma(), "");
 6639   InstructionMark im(this);
 6640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6641   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6642   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6643   emit_int8((unsigned char)0xB8);
 6644   emit_operand(dst, src2, 0);
 6645 }
 6646 
 6647 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6648   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6649   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6650   attributes.set_rex_vex_w_reverted();
 6651   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6652   emit_int16(0x5E, (0xC0 | encode));
 6653 }
 6654 
 6655 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6656   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6657   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6658   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6659   emit_int16(0x5E, (0xC0 | encode));
 6660 }
 6661 
 6662 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6663   assert(VM_Version::supports_avx(), "");
 6664   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6665   attributes.set_rex_vex_w_reverted();
 6666   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6667   emit_int16(0x5E, (0xC0 | encode));
 6668 }
 6669 
 6670 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6671   assert(VM_Version::supports_avx(), "");
 6672   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6673   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6674   emit_int16(0x5E, (0xC0 | encode));
 6675 }
 6676 
 6677 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6678   assert(VM_Version::supports_avx(), "");
 6679   InstructionMark im(this);
 6680   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6681   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6682   attributes.set_rex_vex_w_reverted();
 6683   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6684   emit_int8(0x5E);
 6685   emit_operand(dst, src, 0);
 6686 }
 6687 
 6688 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6689   assert(VM_Version::supports_avx(), "");
 6690   InstructionMark im(this);
 6691   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6692   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6693   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6694   emit_int8(0x5E);
 6695   emit_operand(dst, src, 0);
 6696 }
 6697 
 6698 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6699   assert(VM_Version::supports_avx(), "");
 6700   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6701   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6702   emit_int24(0x09, (0xC0 | encode), (rmode));
 6703 }
 6704 
 6705 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6706   assert(VM_Version::supports_avx(), "");
 6707   InstructionMark im(this);
 6708   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6709   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6710   emit_int8(0x09);
 6711   emit_operand(dst, src, 1);
 6712   emit_int8((rmode));
 6713 }
 6714 
 6715 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6716   assert(VM_Version::supports_evex(), "requires EVEX support");
 6717   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6718   attributes.set_is_evex_instruction();
 6719   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6720   emit_int24(0x09, (0xC0 | encode), (rmode));
 6721 }
 6722 
 6723 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6724   assert(VM_Version::supports_evex(), "requires EVEX support");
 6725   assert(dst != xnoreg, "sanity");
 6726   InstructionMark im(this);
 6727   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6728   attributes.set_is_evex_instruction();
 6729   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6730   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6731   emit_int8(0x09);
 6732   emit_operand(dst, src, 1);
 6733   emit_int8((rmode));
 6734 }
 6735 
 6736 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6737   assert(VM_Version::supports_avx(), "");
 6738   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6739   attributes.set_rex_vex_w_reverted();
 6740   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6741   emit_int16(0x51, (0xC0 | encode));
 6742 }
 6743 
 6744 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6745   assert(VM_Version::supports_avx(), "");
 6746   InstructionMark im(this);
 6747   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6748   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6749   attributes.set_rex_vex_w_reverted();
 6750   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6751   emit_int8(0x51);
 6752   emit_operand(dst, src, 0);
 6753 }
 6754 
 6755 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6756   assert(VM_Version::supports_avx(), "");
 6757   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6758   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6759   emit_int16(0x51, (0xC0 | encode));
 6760 }
 6761 
 6762 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6763   assert(VM_Version::supports_avx(), "");
 6764   InstructionMark im(this);
 6765   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6766   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6767   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6768   emit_int8(0x51);
 6769   emit_operand(dst, src, 0);
 6770 }
 6771 
 6772 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6773   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6774   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6775   attributes.set_rex_vex_w_reverted();
 6776   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6777   emit_int16(0x54, (0xC0 | encode));
 6778 }
 6779 
 6780 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6781   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6782   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6783   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6784   emit_int16(0x54, (0xC0 | encode));
 6785 }
 6786 
 6787 void Assembler::andps(XMMRegister dst, Address src) {
 6788   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6789   InstructionMark im(this);
 6790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6791   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6792   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6793   emit_int8(0x54);
 6794   emit_operand(dst, src, 0);
 6795 }
 6796 
 6797 void Assembler::andpd(XMMRegister dst, Address src) {
 6798   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6799   InstructionMark im(this);
 6800   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6801   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6802   attributes.set_rex_vex_w_reverted();
 6803   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6804   emit_int8(0x54);
 6805   emit_operand(dst, src, 0);
 6806 }
 6807 
 6808 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6809   assert(VM_Version::supports_avx(), "");
 6810   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6811   attributes.set_rex_vex_w_reverted();
 6812   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6813   emit_int16(0x54, (0xC0 | encode));
 6814 }
 6815 
 6816 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6817   assert(VM_Version::supports_avx(), "");
 6818   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6819   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6820   emit_int16(0x54, (0xC0 | encode));
 6821 }
 6822 
 6823 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6824   assert(VM_Version::supports_avx(), "");
 6825   InstructionMark im(this);
 6826   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6827   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6828   attributes.set_rex_vex_w_reverted();
 6829   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6830   emit_int8(0x54);
 6831   emit_operand(dst, src, 0);
 6832 }
 6833 
 6834 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6835   assert(VM_Version::supports_avx(), "");
 6836   InstructionMark im(this);
 6837   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6838   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6839   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6840   emit_int8(0x54);
 6841   emit_operand(dst, src, 0);
 6842 }
 6843 
 6844 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6845   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6846   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6847   attributes.set_rex_vex_w_reverted();
 6848   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6849   emit_int8(0x15);
 6850   emit_int8((0xC0 | encode));
 6851 }
 6852 
 6853 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6855   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6856   attributes.set_rex_vex_w_reverted();
 6857   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6858   emit_int16(0x14, (0xC0 | encode));
 6859 }
 6860 
 6861 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6862   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6863   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6864   attributes.set_rex_vex_w_reverted();
 6865   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6866   emit_int16(0x57, (0xC0 | encode));
 6867 }
 6868 
 6869 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6870   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6872   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6873   emit_int16(0x57, (0xC0 | encode));
 6874 }
 6875 
 6876 void Assembler::xorpd(XMMRegister dst, Address src) {
 6877   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6878   InstructionMark im(this);
 6879   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6880   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6881   attributes.set_rex_vex_w_reverted();
 6882   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6883   emit_int8(0x57);
 6884   emit_operand(dst, src, 0);
 6885 }
 6886 
 6887 void Assembler::xorps(XMMRegister dst, Address src) {
 6888   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6889   InstructionMark im(this);
 6890   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6891   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6892   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6893   emit_int8(0x57);
 6894   emit_operand(dst, src, 0);
 6895 }
 6896 
 6897 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6898   assert(VM_Version::supports_avx(), "");
 6899   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6900   attributes.set_rex_vex_w_reverted();
 6901   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6902   emit_int16(0x57, (0xC0 | encode));
 6903 }
 6904 
 6905 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6906   assert(VM_Version::supports_avx(), "");
 6907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6908   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6909   emit_int16(0x57, (0xC0 | encode));
 6910 }
 6911 
 6912 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6913   assert(VM_Version::supports_avx(), "");
 6914   InstructionMark im(this);
 6915   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6916   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6917   attributes.set_rex_vex_w_reverted();
 6918   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6919   emit_int8(0x57);
 6920   emit_operand(dst, src, 0);
 6921 }
 6922 
 6923 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6924   assert(VM_Version::supports_avx(), "");
 6925   InstructionMark im(this);
 6926   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6927   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6928   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6929   emit_int8(0x57);
 6930   emit_operand(dst, src, 0);
 6931 }
 6932 
 6933 // Integer vector arithmetic
 6934 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6935   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6936          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6937   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6938   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6939   emit_int16(0x01, (0xC0 | encode));
 6940 }
 6941 
 6942 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6943   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6944          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6945   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6946   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6947   emit_int16(0x02, (0xC0 | encode));
 6948 }
 6949 
 6950 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6951   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6952   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6953   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6954   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6955 }
 6956 
 6957 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6958   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6960   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6961   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6962 }
 6963 
 6964 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6965   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6966   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6967   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6968   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6969 }
 6970 
 6971 void Assembler::paddd(XMMRegister dst, Address src) {
 6972   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6973   InstructionMark im(this);
 6974   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6975   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6976   emit_int8((unsigned char)0xFE);
 6977   emit_operand(dst, src, 0);
 6978 }
 6979 
 6980 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6981   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6982   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6983   attributes.set_rex_vex_w_reverted();
 6984   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6985   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6986 }
 6987 
 6988 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6989   assert(VM_Version::supports_sse3(), "");
 6990   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6991   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6992   emit_int16(0x01, (0xC0 | encode));
 6993 }
 6994 
 6995 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6996   assert(VM_Version::supports_sse3(), "");
 6997   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6998   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6999   emit_int16(0x02, (0xC0 | encode));
 7000 }
 7001 
 7002 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7003   assert(UseAVX > 0, "requires some form of AVX");
 7004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7005   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7006   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7007 }
 7008 
 7009 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7010   assert(UseAVX > 0, "requires some form of AVX");
 7011   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7012   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7013   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7014 }
 7015 
 7016 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7017   assert(UseAVX > 0, "requires some form of AVX");
 7018   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7019   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7020   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7021 }
 7022 
 7023 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7024   assert(UseAVX > 0, "requires some form of AVX");
 7025   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7026   attributes.set_rex_vex_w_reverted();
 7027   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7028   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7029 }
 7030 
 7031 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7032   assert(UseAVX > 0, "requires some form of AVX");
 7033   InstructionMark im(this);
 7034   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7035   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7036   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7037   emit_int8((unsigned char)0xFC);
 7038   emit_operand(dst, src, 0);
 7039 }
 7040 
 7041 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7042   assert(UseAVX > 0, "requires some form of AVX");
 7043   InstructionMark im(this);
 7044   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7045   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7046   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7047   emit_int8((unsigned char)0xFD);
 7048   emit_operand(dst, src, 0);
 7049 }
 7050 
 7051 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7052   assert(UseAVX > 0, "requires some form of AVX");
 7053   InstructionMark im(this);
 7054   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7055   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7056   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7057   emit_int8((unsigned char)0xFE);
 7058   emit_operand(dst, src, 0);
 7059 }
 7060 
 7061 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7062   assert(UseAVX > 0, "requires some form of AVX");
 7063   InstructionMark im(this);
 7064   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7065   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7066   attributes.set_rex_vex_w_reverted();
 7067   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7068   emit_int8((unsigned char)0xD4);
 7069   emit_operand(dst, src, 0);
 7070 }
 7071 
 7072 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 7073   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7074   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7075   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7076   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7077 }
 7078 
 7079 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 7080   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7081   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7082   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7083   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7084 }
 7085 
 7086 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 7087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7088   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7089   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7090 }
 7091 
 7092 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 7093   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7094   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7095   attributes.set_rex_vex_w_reverted();
 7096   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7097   emit_int8((unsigned char)0xFB);
 7098   emit_int8((0xC0 | encode));
 7099 }
 7100 
 7101 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7102   assert(UseAVX > 0, "requires some form of AVX");
 7103   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7104   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7105   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 7106 }
 7107 
 7108 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7109   assert(UseAVX > 0, "requires some form of AVX");
 7110   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7111   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7112   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7113 }
 7114 
 7115 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7116   assert(UseAVX > 0, "requires some form of AVX");
 7117   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7118   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7119   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7120 }
 7121 
 7122 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7123   assert(UseAVX > 0, "requires some form of AVX");
 7124   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7125   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7126   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7127 }
 7128 
 7129 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7130   assert(UseAVX > 0, "requires some form of AVX");
 7131   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7132   attributes.set_rex_vex_w_reverted();
 7133   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7134   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7135 }
 7136 
 7137 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7138   assert(UseAVX > 0, "requires some form of AVX");
 7139   InstructionMark im(this);
 7140   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7141   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7142   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7143   emit_int8((unsigned char)0xF8);
 7144   emit_operand(dst, src, 0);
 7145 }
 7146 
 7147 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7148   assert(UseAVX > 0, "requires some form of AVX");
 7149   InstructionMark im(this);
 7150   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7151   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7152   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7153   emit_int8((unsigned char)0xF9);
 7154   emit_operand(dst, src, 0);
 7155 }
 7156 
 7157 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7158   assert(UseAVX > 0, "requires some form of AVX");
 7159   InstructionMark im(this);
 7160   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7161   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7162   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7163   emit_int8((unsigned char)0xFA);
 7164   emit_operand(dst, src, 0);
 7165 }
 7166 
 7167 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7168   assert(UseAVX > 0, "requires some form of AVX");
 7169   InstructionMark im(this);
 7170   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7171   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7172   attributes.set_rex_vex_w_reverted();
 7173   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7174   emit_int8((unsigned char)0xFB);
 7175   emit_operand(dst, src, 0);
 7176 }
 7177 
 7178 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7179   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7180   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7181   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7182   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7183 }
 7184 
 7185 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7186   assert(VM_Version::supports_sse4_1(), "");
 7187   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7188   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7189   emit_int16(0x40, (0xC0 | encode));
 7190 }
 7191 
 7192 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7193   assert(VM_Version::supports_sse2(), "");
 7194   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7195   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7196   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7197 }
 7198 
 7199 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7200   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7201          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7202          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7204   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7205   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7206 }
 7207 
 7208 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7209   assert(UseAVX > 0, "requires some form of AVX");
 7210   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7211   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7212   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7213 }
 7214 
 7215 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7216   assert(UseAVX > 0, "requires some form of AVX");
 7217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7218   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7219   emit_int16(0x40, (0xC0 | encode));
 7220 }
 7221 
 7222 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7223   assert(UseAVX > 2, "requires some form of EVEX");
 7224   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7225   attributes.set_is_evex_instruction();
 7226   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7227   emit_int16(0x40, (0xC0 | encode));
 7228 }
 7229 
 7230 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7231   assert(UseAVX > 0, "requires some form of AVX");
 7232   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7233   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7234   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7235 }
 7236 
 7237 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7238   assert(UseAVX > 0, "requires some form of AVX");
 7239   InstructionMark im(this);
 7240   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7241   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7242   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7243   emit_int8((unsigned char)0xD5);
 7244   emit_operand(dst, src, 0);
 7245 }
 7246 
 7247 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7248   assert(UseAVX > 0, "requires some form of AVX");
 7249   InstructionMark im(this);
 7250   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7251   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7252   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7253   emit_int8(0x40);
 7254   emit_operand(dst, src, 0);
 7255 }
 7256 
 7257 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7258   assert(UseAVX > 2, "requires some form of EVEX");
 7259   InstructionMark im(this);
 7260   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7261   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7262   attributes.set_is_evex_instruction();
 7263   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7264   emit_int8(0x40);
 7265   emit_operand(dst, src, 0);
 7266 }
 7267 
 7268 // Min, max
 7269 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7270   assert(VM_Version::supports_sse4_1(), "");
 7271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7272   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7273   emit_int16(0x38, (0xC0 | encode));
 7274 }
 7275 
 7276 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7277   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7278         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7279   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7280   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7281   emit_int16(0x38, (0xC0 | encode));
 7282 }
 7283 
 7284 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7285   assert(VM_Version::supports_sse2(), "");
 7286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7287   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7288   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7289 }
 7290 
 7291 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7292   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7293         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7295   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7296   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7297 }
 7298 
 7299 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7300   assert(VM_Version::supports_sse4_1(), "");
 7301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7302   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7303   emit_int16(0x39, (0xC0 | encode));
 7304 }
 7305 
 7306 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7307   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7308         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7309   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7311   emit_int16(0x39, (0xC0 | encode));
 7312 }
 7313 
 7314 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7315   assert(UseAVX > 2, "requires AVX512F");
 7316   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7317   attributes.set_is_evex_instruction();
 7318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7319   emit_int16(0x39, (0xC0 | encode));
 7320 }
 7321 
 7322 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7323   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7324   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7325   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7326   emit_int16(0x5D, (0xC0 | encode));
 7327 }
 7328 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7329   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7330   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7331   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7332   emit_int16(0x5D, (0xC0 | encode));
 7333 }
 7334 
 7335 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7336   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7337   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7338   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7339   emit_int16(0x5D, (0xC0 | encode));
 7340 }
 7341 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7342   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7343   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7344   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7345   emit_int16(0x5D, (0xC0 | encode));
 7346 }
 7347 
 7348 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7349   assert(VM_Version::supports_sse4_1(), "");
 7350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7351   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7352   emit_int16(0x3C, (0xC0 | encode));
 7353 }
 7354 
 7355 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7356   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7357         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7358   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7359   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7360   emit_int16(0x3C, (0xC0 | encode));
 7361 }
 7362 
 7363 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7364   assert(VM_Version::supports_sse2(), "");
 7365   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7366   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7367   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7368 }
 7369 
 7370 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7371   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7372         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7373   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7374   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7375   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7376 }
 7377 
 7378 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7379   assert(VM_Version::supports_sse4_1(), "");
 7380   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7381   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7382   emit_int16(0x3D, (0xC0 | encode));
 7383 }
 7384 
 7385 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7386   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7387         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7389   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7390   emit_int16(0x3D, (0xC0 | encode));
 7391 }
 7392 
 7393 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7394   assert(UseAVX > 2, "requires AVX512F");
 7395   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7396   attributes.set_is_evex_instruction();
 7397   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7398   emit_int16(0x3D, (0xC0 | encode));
 7399 }
 7400 
 7401 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7402   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7404   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7405   emit_int16(0x5F, (0xC0 | encode));
 7406 }
 7407 
 7408 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7409   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7410   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7411   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7412   emit_int16(0x5F, (0xC0 | encode));
 7413 }
 7414 
 7415 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7416   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7417   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7418   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7419   emit_int16(0x5F, (0xC0 | encode));
 7420 }
 7421 
 7422 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7423   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7424   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7425   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7426   emit_int16(0x5F, (0xC0 | encode));
 7427 }
 7428 
 7429 // Shift packed integers left by specified number of bits.
 7430 void Assembler::psllw(XMMRegister dst, int shift) {
 7431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7432   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7433   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7434   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7435   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7436 }
 7437 
 7438 void Assembler::pslld(XMMRegister dst, int shift) {
 7439   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7440   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7441   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7442   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7443   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7444 }
 7445 
 7446 void Assembler::psllq(XMMRegister dst, int shift) {
 7447   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7448   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7449   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7450   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7451   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7452 }
 7453 
 7454 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7455   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7457   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7458   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7459 }
 7460 
 7461 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7462   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7464   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7465   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7466 }
 7467 
 7468 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7469   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7470   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7471   attributes.set_rex_vex_w_reverted();
 7472   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7473   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7474 }
 7475 
 7476 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7477   assert(UseAVX > 0, "requires some form of AVX");
 7478   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7479   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7480   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7481   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7482 }
 7483 
 7484 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7485   assert(UseAVX > 0, "requires some form of AVX");
 7486   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7487   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7488   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7489   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7490   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7491 }
 7492 
 7493 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7494   assert(UseAVX > 0, "requires some form of AVX");
 7495   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7496   attributes.set_rex_vex_w_reverted();
 7497   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7498   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7499   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7500 }
 7501 
 7502 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7503   assert(UseAVX > 0, "requires some form of AVX");
 7504   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7505   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7506   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7507 }
 7508 
 7509 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7510   assert(UseAVX > 0, "requires some form of AVX");
 7511   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7512   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7513   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7514 }
 7515 
 7516 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7517   assert(UseAVX > 0, "requires some form of AVX");
 7518   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7519   attributes.set_rex_vex_w_reverted();
 7520   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7521   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7522 }
 7523 
 7524 // Shift packed integers logically right by specified number of bits.
 7525 void Assembler::psrlw(XMMRegister dst, int shift) {
 7526   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7527   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7528   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7529   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7530   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7531 }
 7532 
 7533 void Assembler::psrld(XMMRegister dst, int shift) {
 7534   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7536   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7537   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7538   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7539 }
 7540 
 7541 void Assembler::psrlq(XMMRegister dst, int shift) {
 7542   // Do not confuse it with psrldq SSE2 instruction which
 7543   // shifts 128 bit value in xmm register by number of bytes.
 7544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7545   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7546   attributes.set_rex_vex_w_reverted();
 7547   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7548   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7549   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7550 }
 7551 
 7552 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7553   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7554   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7555   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7556   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7557 }
 7558 
 7559 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7560   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7561   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7562   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7563   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7564 }
 7565 
 7566 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7567   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7568   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7569   attributes.set_rex_vex_w_reverted();
 7570   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7571   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7572 }
 7573 
 7574 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7575   assert(UseAVX > 0, "requires some form of AVX");
 7576   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7577   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7578   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7579   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7580 }
 7581 
 7582 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7583   assert(UseAVX > 0, "requires some form of AVX");
 7584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7585   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7586   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7587   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7588 }
 7589 
 7590 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7591   assert(UseAVX > 0, "requires some form of AVX");
 7592   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7593   attributes.set_rex_vex_w_reverted();
 7594   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7595   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7596   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7597 }
 7598 
 7599 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7600   assert(UseAVX > 0, "requires some form of AVX");
 7601   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7602   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7603   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7604 }
 7605 
 7606 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7607   assert(UseAVX > 0, "requires some form of AVX");
 7608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7609   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7610   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7611 }
 7612 
 7613 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7614   assert(UseAVX > 0, "requires some form of AVX");
 7615   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7616   attributes.set_rex_vex_w_reverted();
 7617   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7618   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7619 }
 7620 
 7621 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7622   assert(VM_Version::supports_avx512bw(), "");
 7623   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7624   attributes.set_is_evex_instruction();
 7625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7626   emit_int16(0x10, (0xC0 | encode));
 7627 }
 7628 
 7629 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7630   assert(VM_Version::supports_avx512bw(), "");
 7631   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7632   attributes.set_is_evex_instruction();
 7633   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7634   emit_int16(0x12, (0xC0 | encode));
 7635 }
 7636 
 7637 // Shift packed integers arithmetically right by specified number of bits.
 7638 void Assembler::psraw(XMMRegister dst, int shift) {
 7639   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7640   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7641   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7642   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7643   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7644 }
 7645 
 7646 void Assembler::psrad(XMMRegister dst, int shift) {
 7647   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7649   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7650   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7651   emit_int8(0x72);
 7652   emit_int8((0xC0 | encode));
 7653   emit_int8(shift & 0xFF);
 7654 }
 7655 
 7656 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7658   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7659   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7660   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7661 }
 7662 
 7663 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7664   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7666   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7667   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7668 }
 7669 
 7670 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7671   assert(UseAVX > 0, "requires some form of AVX");
 7672   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7673   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7674   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7675   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7676 }
 7677 
 7678 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7679   assert(UseAVX > 0, "requires some form of AVX");
 7680   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7681   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7682   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7683   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7684 }
 7685 
 7686 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7687   assert(UseAVX > 0, "requires some form of AVX");
 7688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7689   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7690   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7691 }
 7692 
 7693 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7694   assert(UseAVX > 0, "requires some form of AVX");
 7695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7696   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7697   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7698 }
 7699 
 7700 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7701   assert(UseAVX > 2, "requires AVX512");
 7702   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7703   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7704   attributes.set_is_evex_instruction();
 7705   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7706   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7707 }
 7708 
 7709 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7710   assert(UseAVX > 2, "requires AVX512");
 7711   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7712   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7713   attributes.set_is_evex_instruction();
 7714   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7715   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7716 }
 7717 
 7718 // logical operations packed integers
 7719 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7721   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7722   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7723   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7724 }
 7725 
 7726 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7727   assert(UseAVX > 0, "requires some form of AVX");
 7728   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7729   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7730   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7731 }
 7732 
 7733 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7734   assert(UseAVX > 0, "requires some form of AVX");
 7735   InstructionMark im(this);
 7736   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7737   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7738   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7739   emit_int8((unsigned char)0xDB);
 7740   emit_operand(dst, src, 0);
 7741 }
 7742 
 7743 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7744   assert(VM_Version::supports_evex(), "");
 7745   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7746   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7747   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7748 }
 7749 
 7750 //Variable Shift packed integers logically left.
 7751 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7752   assert(UseAVX > 1, "requires AVX2");
 7753   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7754   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7755   emit_int16(0x47, (0xC0 | encode));
 7756 }
 7757 
 7758 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7759   assert(UseAVX > 1, "requires AVX2");
 7760   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7761   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7762   emit_int16(0x47, (0xC0 | encode));
 7763 }
 7764 
 7765 //Variable Shift packed integers logically right.
 7766 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7767   assert(UseAVX > 1, "requires AVX2");
 7768   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7769   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7770   emit_int16(0x45, (0xC0 | encode));
 7771 }
 7772 
 7773 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7774   assert(UseAVX > 1, "requires AVX2");
 7775   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7776   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7777   emit_int16(0x45, (0xC0 | encode));
 7778 }
 7779 
 7780 //Variable right Shift arithmetic packed integers .
 7781 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7782   assert(UseAVX > 1, "requires AVX2");
 7783   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7784   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7785   emit_int16(0x46, (0xC0 | encode));
 7786 }
 7787 
 7788 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7789   assert(VM_Version::supports_avx512bw(), "");
 7790   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7791   attributes.set_is_evex_instruction();
 7792   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7793   emit_int16(0x11, (0xC0 | encode));
 7794 }
 7795 
 7796 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7797   assert(UseAVX > 2, "requires AVX512");
 7798   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7799   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7800   attributes.set_is_evex_instruction();
 7801   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7802   emit_int16(0x46, (0xC0 | encode));
 7803 }
 7804 
 7805 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7806   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7807   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7808   attributes.set_is_evex_instruction();
 7809   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7810   emit_int16(0x71, (0xC0 | encode));
 7811 }
 7812 
 7813 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7814   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7815   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7816   attributes.set_is_evex_instruction();
 7817   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7818   emit_int16(0x73, (0xC0 | encode));
 7819 }
 7820 
 7821 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7822   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7823   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7824   attributes.set_rex_vex_w_reverted();
 7825   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7826   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7827 }
 7828 
 7829 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7830   assert(UseAVX > 0, "requires some form of AVX");
 7831   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7832   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7833   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7834 }
 7835 
 7836 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7838   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7839   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7840   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7841 }
 7842 
 7843 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7844   assert(UseAVX > 0, "requires some form of AVX");
 7845   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7846   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7847   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7848 }
 7849 
 7850 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7851   assert(UseAVX > 0, "requires some form of AVX");
 7852   InstructionMark im(this);
 7853   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7854   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7855   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7856   emit_int8((unsigned char)0xEB);
 7857   emit_operand(dst, src, 0);
 7858 }
 7859 
 7860 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7861   assert(VM_Version::supports_evex(), "");
 7862   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7863   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7864   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7865 }
 7866 
 7867 
 7868 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7869   assert(VM_Version::supports_evex(), "");
 7870   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7871   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7872   attributes.set_is_evex_instruction();
 7873   attributes.set_embedded_opmask_register_specifier(mask);
 7874   if (merge) {
 7875     attributes.reset_is_clear_context();
 7876   }
 7877   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7878   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7879 }
 7880 
 7881 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7882   assert(VM_Version::supports_evex(), "");
 7883   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7884   InstructionMark im(this);
 7885   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7886   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7887   attributes.set_is_evex_instruction();
 7888   attributes.set_embedded_opmask_register_specifier(mask);
 7889   if (merge) {
 7890     attributes.reset_is_clear_context();
 7891   }
 7892   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7893   emit_int8((unsigned char)0xEB);
 7894   emit_operand(dst, src, 0);
 7895 }
 7896 
 7897 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7898   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7899   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7900   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7901   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7902 }
 7903 
 7904 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7905   assert(UseAVX > 0, "requires some form of AVX");
 7906   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7907          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7908          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7909   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7910   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7911   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7912 }
 7913 
 7914 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7915   assert(UseAVX > 0, "requires some form of AVX");
 7916   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7917          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7918          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7919   InstructionMark im(this);
 7920   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7921   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7922   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7923   emit_int8((unsigned char)0xEF);
 7924   emit_operand(dst, src, 0);
 7925 }
 7926 
 7927 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7928   assert(UseAVX > 2, "requires some form of EVEX");
 7929   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7930   attributes.set_rex_vex_w_reverted();
 7931   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7932   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7933 }
 7934 
 7935 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7936   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7937   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7938   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7939   attributes.set_is_evex_instruction();
 7940   attributes.set_embedded_opmask_register_specifier(mask);
 7941   if (merge) {
 7942     attributes.reset_is_clear_context();
 7943   }
 7944   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7945   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7946 }
 7947 
 7948 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7949   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7950   InstructionMark im(this);
 7951   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7952   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7953   attributes.set_is_evex_instruction();
 7954   attributes.set_embedded_opmask_register_specifier(mask);
 7955   if (merge) {
 7956     attributes.reset_is_clear_context();
 7957   }
 7958   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7959   emit_int8((unsigned char)0xEF);
 7960   emit_operand(dst, src, 0);
 7961 }
 7962 
 7963 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7964   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7965   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7966   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7967   attributes.set_is_evex_instruction();
 7968   attributes.set_embedded_opmask_register_specifier(mask);
 7969   if (merge) {
 7970     attributes.reset_is_clear_context();
 7971   }
 7972   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7973   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7974 }
 7975 
 7976 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7977   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7978   InstructionMark im(this);
 7979   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7981   attributes.set_is_evex_instruction();
 7982   attributes.set_embedded_opmask_register_specifier(mask);
 7983   if (merge) {
 7984     attributes.reset_is_clear_context();
 7985   }
 7986   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7987   emit_int8((unsigned char)0xEF);
 7988   emit_operand(dst, src, 0);
 7989 }
 7990 
 7991 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7992   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7993   InstructionMark im(this);
 7994   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7995   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7996   attributes.set_is_evex_instruction();
 7997   attributes.set_embedded_opmask_register_specifier(mask);
 7998   if (merge) {
 7999     attributes.reset_is_clear_context();
 8000   }
 8001   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8002   emit_int8((unsigned char)0xDB);
 8003   emit_operand(dst, src, 0);
 8004 }
 8005 
 8006 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8007   assert(VM_Version::supports_evex(), "");
 8008   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8009   attributes.set_is_evex_instruction();
 8010   attributes.set_embedded_opmask_register_specifier(mask);
 8011   if (merge) {
 8012     attributes.reset_is_clear_context();
 8013   }
 8014   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8015   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 8016 }
 8017 
 8018 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8019   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8020   InstructionMark im(this);
 8021   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8022   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8023   attributes.set_is_evex_instruction();
 8024   attributes.set_embedded_opmask_register_specifier(mask);
 8025   if (merge) {
 8026     attributes.reset_is_clear_context();
 8027   }
 8028   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8029   emit_int8((unsigned char)0xDB);
 8030   emit_operand(dst, src, 0);
 8031 }
 8032 
 8033 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8034   assert(VM_Version::supports_evex(), "");
 8035   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8036   attributes.set_is_evex_instruction();
 8037   attributes.set_embedded_opmask_register_specifier(mask);
 8038   if (merge) {
 8039     attributes.reset_is_clear_context();
 8040   }
 8041   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8042   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8043 }
 8044 
 8045 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8046   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8047   InstructionMark im(this);
 8048   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8049   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8050   attributes.set_is_evex_instruction();
 8051   attributes.set_embedded_opmask_register_specifier(mask);
 8052   if (merge) {
 8053     attributes.reset_is_clear_context();
 8054   }
 8055   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8056   emit_int8((unsigned char)0xEB);
 8057   emit_operand(dst, src, 0);
 8058 }
 8059 
 8060 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8061   assert(VM_Version::supports_evex(), "requires EVEX support");
 8062   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8063   attributes.set_is_evex_instruction();
 8064   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8065   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8066 }
 8067 
 8068 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8069   assert(VM_Version::supports_evex(), "requires EVEX support");
 8070   assert(dst != xnoreg, "sanity");
 8071   InstructionMark im(this);
 8072   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8073   attributes.set_is_evex_instruction();
 8074   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8075   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8076   emit_int8((unsigned char)0xEF);
 8077   emit_operand(dst, src, 0);
 8078 }
 8079 
 8080 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8081   assert(VM_Version::supports_evex(), "requires EVEX support");
 8082   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8084   attributes.set_is_evex_instruction();
 8085   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8086   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8087 }
 8088 
 8089 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8090   assert(VM_Version::supports_evex(), "requires EVEX support");
 8091   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8092   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8093   attributes.set_is_evex_instruction();
 8094   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8095   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8096 }
 8097 
 8098 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8099   assert(VM_Version::supports_evex(), "requires EVEX support");
 8100   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8102   attributes.set_is_evex_instruction();
 8103   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8104   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8105 }
 8106 
 8107 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8108   assert(VM_Version::supports_evex(), "requires EVEX support");
 8109   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8110   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8111   attributes.set_is_evex_instruction();
 8112   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8113   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8114 }
 8115 
 8116 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8117   assert(VM_Version::supports_evex(), "requires EVEX support");
 8118   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8120   attributes.set_is_evex_instruction();
 8121   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8122   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8123 }
 8124 
 8125 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8126   assert(VM_Version::supports_evex(), "requires EVEX support");
 8127   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8128   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8129   attributes.set_is_evex_instruction();
 8130   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8131   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8132 }
 8133 
 8134 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8135   assert(VM_Version::supports_evex(), "requires EVEX support");
 8136   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8137   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8138   attributes.set_is_evex_instruction();
 8139   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8140   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8141 }
 8142 
 8143 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8144   assert(VM_Version::supports_evex(), "requires EVEX support");
 8145   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8146   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8147   attributes.set_is_evex_instruction();
 8148   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8149   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8150 }
 8151 
 8152 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8153   assert(VM_Version::supports_avx512cd(), "");
 8154   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8155   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8156   attributes.set_is_evex_instruction();
 8157   attributes.set_embedded_opmask_register_specifier(mask);
 8158   if (merge) {
 8159     attributes.reset_is_clear_context();
 8160   }
 8161   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8162   emit_int16(0x44, (0xC0 | encode));
 8163 }
 8164 
 8165 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8166   assert(VM_Version::supports_avx512cd(), "");
 8167   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8168   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8169   attributes.set_is_evex_instruction();
 8170   attributes.set_embedded_opmask_register_specifier(mask);
 8171   if (merge) {
 8172     attributes.reset_is_clear_context();
 8173   }
 8174   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8175   emit_int16(0x44, (0xC0 | encode));
 8176 }
 8177 
 8178 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8179   assert(VM_Version::supports_evex(), "requires EVEX support");
 8180   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8181   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8182   attributes.set_is_evex_instruction();
 8183   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8184   emit_int8(0x25);
 8185   emit_int8((unsigned char)(0xC0 | encode));
 8186   emit_int8(imm8);
 8187 }
 8188 
 8189 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8190   assert(VM_Version::supports_evex(), "requires EVEX support");
 8191   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8192   assert(dst != xnoreg, "sanity");
 8193   InstructionMark im(this);
 8194   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8195   attributes.set_is_evex_instruction();
 8196   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8197   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8198   emit_int8(0x25);
 8199   emit_operand(dst, src3, 1);
 8200   emit_int8(imm8);
 8201 }
 8202 
 8203 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8204   assert(VM_Version::supports_evex(), "requires EVEX support");
 8205   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8206   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8207   attributes.set_is_evex_instruction();
 8208   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8209   emit_int8(0x25);
 8210   emit_int8((unsigned char)(0xC0 | encode));
 8211   emit_int8(imm8);
 8212 }
 8213 
 8214 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8215   assert(VM_Version::supports_evex(), "");
 8216   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8218   attributes.set_is_evex_instruction();
 8219   attributes.set_embedded_opmask_register_specifier(mask);
 8220   if (merge) {
 8221     attributes.reset_is_clear_context();
 8222   }
 8223   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8224   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8225 }
 8226 
 8227 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8228   assert(VM_Version::supports_evex(), "");
 8229   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8230   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8231   attributes.set_is_evex_instruction();
 8232   attributes.set_embedded_opmask_register_specifier(mask);
 8233   if (merge) {
 8234     attributes.reset_is_clear_context();
 8235   }
 8236   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8237   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8238 }
 8239 
 8240 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8241   assert(VM_Version::supports_avx512_vbmi2(), "");
 8242   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8243   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8244   attributes.set_is_evex_instruction();
 8245   attributes.set_embedded_opmask_register_specifier(mask);
 8246   if (merge) {
 8247     attributes.reset_is_clear_context();
 8248   }
 8249   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8250   emit_int16(0x62, (0xC0 | encode));
 8251 }
 8252 
 8253 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8254   assert(VM_Version::supports_avx512_vbmi2(), "");
 8255   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8256   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8257   attributes.set_is_evex_instruction();
 8258   attributes.set_embedded_opmask_register_specifier(mask);
 8259   if (merge) {
 8260     attributes.reset_is_clear_context();
 8261   }
 8262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8263   emit_int16(0x62, (0xC0 | encode));
 8264 }
 8265 
 8266 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8267   assert(VM_Version::supports_evex(), "");
 8268   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8269   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8270   attributes.set_is_evex_instruction();
 8271   attributes.set_embedded_opmask_register_specifier(mask);
 8272   if (merge) {
 8273     attributes.reset_is_clear_context();
 8274   }
 8275   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8276   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8277 }
 8278 
 8279 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8280   assert(VM_Version::supports_evex(), "");
 8281   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8282   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8283   attributes.set_is_evex_instruction();
 8284   attributes.set_embedded_opmask_register_specifier(mask);
 8285   if (merge) {
 8286     attributes.reset_is_clear_context();
 8287   }
 8288   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8289   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8290 }
 8291 
 8292 // vinserti forms
 8293 
 8294 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8295   assert(VM_Version::supports_avx2(), "");
 8296   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8297   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8298   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8299   // last byte:
 8300   // 0x00 - insert into lower 128 bits
 8301   // 0x01 - insert into upper 128 bits
 8302   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8303 }
 8304 
 8305 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8306   assert(VM_Version::supports_avx2(), "");
 8307   assert(dst != xnoreg, "sanity");
 8308   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8309   InstructionMark im(this);
 8310   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8311   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8312   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8313   emit_int8(0x38);
 8314   emit_operand(dst, src, 1);
 8315   // 0x00 - insert into lower 128 bits
 8316   // 0x01 - insert into upper 128 bits
 8317   emit_int8(imm8 & 0x01);
 8318 }
 8319 
 8320 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8321   assert(VM_Version::supports_evex(), "");
 8322   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8323   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8324   attributes.set_is_evex_instruction();
 8325   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8326   // imm8:
 8327   // 0x00 - insert into q0 128 bits (0..127)
 8328   // 0x01 - insert into q1 128 bits (128..255)
 8329   // 0x02 - insert into q2 128 bits (256..383)
 8330   // 0x03 - insert into q3 128 bits (384..511)
 8331   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8332 }
 8333 
 8334 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8335   assert(VM_Version::supports_evex(), "");
 8336   assert(dst != xnoreg, "sanity");
 8337   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8338   InstructionMark im(this);
 8339   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8340   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8341   attributes.set_is_evex_instruction();
 8342   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8343   emit_int8(0x18);
 8344   emit_operand(dst, src, 1);
 8345   // 0x00 - insert into q0 128 bits (0..127)
 8346   // 0x01 - insert into q1 128 bits (128..255)
 8347   // 0x02 - insert into q2 128 bits (256..383)
 8348   // 0x03 - insert into q3 128 bits (384..511)
 8349   emit_int8(imm8 & 0x03);
 8350 }
 8351 
 8352 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8353   assert(VM_Version::supports_evex(), "");
 8354   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8355   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8356   attributes.set_is_evex_instruction();
 8357   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8358   //imm8:
 8359   // 0x00 - insert into lower 256 bits
 8360   // 0x01 - insert into upper 256 bits
 8361   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8362 }
 8363 
 8364 
 8365 // vinsertf forms
 8366 
 8367 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8368   assert(VM_Version::supports_avx(), "");
 8369   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8370   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8371   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8372   // imm8:
 8373   // 0x00 - insert into lower 128 bits
 8374   // 0x01 - insert into upper 128 bits
 8375   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8376 }
 8377 
 8378 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8379   assert(VM_Version::supports_avx(), "");
 8380   assert(dst != xnoreg, "sanity");
 8381   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8382   InstructionMark im(this);
 8383   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8384   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8385   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8386   emit_int8(0x18);
 8387   emit_operand(dst, src, 1);
 8388   // 0x00 - insert into lower 128 bits
 8389   // 0x01 - insert into upper 128 bits
 8390   emit_int8(imm8 & 0x01);
 8391 }
 8392 
 8393 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8394   assert(VM_Version::supports_evex(), "");
 8395   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8396   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8397   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8398   // imm8:
 8399   // 0x00 - insert into q0 128 bits (0..127)
 8400   // 0x01 - insert into q1 128 bits (128..255)
 8401   // 0x02 - insert into q0 128 bits (256..383)
 8402   // 0x03 - insert into q1 128 bits (384..512)
 8403   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8404 }
 8405 
 8406 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8407   assert(VM_Version::supports_evex(), "");
 8408   assert(dst != xnoreg, "sanity");
 8409   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8410   InstructionMark im(this);
 8411   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8412   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8413   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8414   emit_int8(0x18);
 8415   emit_operand(dst, src, 1);
 8416   // 0x00 - insert into q0 128 bits (0..127)
 8417   // 0x01 - insert into q1 128 bits (128..255)
 8418   // 0x02 - insert into q0 128 bits (256..383)
 8419   // 0x03 - insert into q1 128 bits (384..512)
 8420   emit_int8(imm8 & 0x03);
 8421 }
 8422 
 8423 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8424   assert(VM_Version::supports_evex(), "");
 8425   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8426   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8427   attributes.set_is_evex_instruction();
 8428   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8429   // imm8:
 8430   // 0x00 - insert into lower 256 bits
 8431   // 0x01 - insert into upper 256 bits
 8432   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8433 }
 8434 
 8435 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8436   assert(VM_Version::supports_evex(), "");
 8437   assert(dst != xnoreg, "sanity");
 8438   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8439   InstructionMark im(this);
 8440   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8441   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8442   attributes.set_is_evex_instruction();
 8443   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8444   emit_int8(0x1A);
 8445   emit_operand(dst, src, 1);
 8446   // 0x00 - insert into lower 256 bits
 8447   // 0x01 - insert into upper 256 bits
 8448   emit_int8(imm8 & 0x01);
 8449 }
 8450 
 8451 
 8452 // vextracti forms
 8453 
 8454 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8455   assert(VM_Version::supports_avx2(), "");
 8456   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8457   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8458   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8459   // imm8:
 8460   // 0x00 - extract from lower 128 bits
 8461   // 0x01 - extract from upper 128 bits
 8462   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8463 }
 8464 
 8465 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8466   assert(VM_Version::supports_avx2(), "");
 8467   assert(src != xnoreg, "sanity");
 8468   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8469   InstructionMark im(this);
 8470   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8471   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8472   attributes.reset_is_clear_context();
 8473   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8474   emit_int8(0x39);
 8475   emit_operand(src, dst, 1);
 8476   // 0x00 - extract from lower 128 bits
 8477   // 0x01 - extract from upper 128 bits
 8478   emit_int8(imm8 & 0x01);
 8479 }
 8480 
 8481 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8482   assert(VM_Version::supports_evex(), "");
 8483   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8484   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8485   attributes.set_is_evex_instruction();
 8486   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8487   // imm8:
 8488   // 0x00 - extract from bits 127:0
 8489   // 0x01 - extract from bits 255:128
 8490   // 0x02 - extract from bits 383:256
 8491   // 0x03 - extract from bits 511:384
 8492   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8493 }
 8494 
 8495 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8496   assert(VM_Version::supports_evex(), "");
 8497   assert(src != xnoreg, "sanity");
 8498   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8499   InstructionMark im(this);
 8500   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8501   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8502   attributes.reset_is_clear_context();
 8503   attributes.set_is_evex_instruction();
 8504   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8505   emit_int8(0x39);
 8506   emit_operand(src, dst, 1);
 8507   // 0x00 - extract from bits 127:0
 8508   // 0x01 - extract from bits 255:128
 8509   // 0x02 - extract from bits 383:256
 8510   // 0x03 - extract from bits 511:384
 8511   emit_int8(imm8 & 0x03);
 8512 }
 8513 
 8514 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8515   assert(VM_Version::supports_avx512dq(), "");
 8516   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8517   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8518   attributes.set_is_evex_instruction();
 8519   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8520   // imm8:
 8521   // 0x00 - extract from bits 127:0
 8522   // 0x01 - extract from bits 255:128
 8523   // 0x02 - extract from bits 383:256
 8524   // 0x03 - extract from bits 511:384
 8525   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8526 }
 8527 
 8528 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8529   assert(VM_Version::supports_evex(), "");
 8530   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8531   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8532   attributes.set_is_evex_instruction();
 8533   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8534   // imm8:
 8535   // 0x00 - extract from lower 256 bits
 8536   // 0x01 - extract from upper 256 bits
 8537   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8538 }
 8539 
 8540 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8541   assert(VM_Version::supports_evex(), "");
 8542   assert(src != xnoreg, "sanity");
 8543   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8544   InstructionMark im(this);
 8545   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8546   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8547   attributes.reset_is_clear_context();
 8548   attributes.set_is_evex_instruction();
 8549   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8550   emit_int8(0x38);
 8551   emit_operand(src, dst, 1);
 8552   // 0x00 - extract from lower 256 bits
 8553   // 0x01 - extract from upper 256 bits
 8554   emit_int8(imm8 & 0x01);
 8555 }
 8556 // vextractf forms
 8557 
 8558 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8559   assert(VM_Version::supports_avx(), "");
 8560   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8561   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8562   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8563   // imm8:
 8564   // 0x00 - extract from lower 128 bits
 8565   // 0x01 - extract from upper 128 bits
 8566   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8567 }
 8568 
 8569 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8570   assert(VM_Version::supports_avx(), "");
 8571   assert(src != xnoreg, "sanity");
 8572   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8573   InstructionMark im(this);
 8574   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8575   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8576   attributes.reset_is_clear_context();
 8577   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8578   emit_int8(0x19);
 8579   emit_operand(src, dst, 1);
 8580   // 0x00 - extract from lower 128 bits
 8581   // 0x01 - extract from upper 128 bits
 8582   emit_int8(imm8 & 0x01);
 8583 }
 8584 
 8585 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8586   assert(VM_Version::supports_evex(), "");
 8587   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8588   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8589   attributes.set_is_evex_instruction();
 8590   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8591   // imm8:
 8592   // 0x00 - extract from bits 127:0
 8593   // 0x01 - extract from bits 255:128
 8594   // 0x02 - extract from bits 383:256
 8595   // 0x03 - extract from bits 511:384
 8596   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8597 }
 8598 
 8599 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8600   assert(VM_Version::supports_evex(), "");
 8601   assert(src != xnoreg, "sanity");
 8602   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8603   InstructionMark im(this);
 8604   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8605   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8606   attributes.reset_is_clear_context();
 8607   attributes.set_is_evex_instruction();
 8608   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8609   emit_int8(0x19);
 8610   emit_operand(src, dst, 1);
 8611   // 0x00 - extract from bits 127:0
 8612   // 0x01 - extract from bits 255:128
 8613   // 0x02 - extract from bits 383:256
 8614   // 0x03 - extract from bits 511:384
 8615   emit_int8(imm8 & 0x03);
 8616 }
 8617 
 8618 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8619   assert(VM_Version::supports_avx512dq(), "");
 8620   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8621   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8622   attributes.set_is_evex_instruction();
 8623   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8624   // imm8:
 8625   // 0x00 - extract from bits 127:0
 8626   // 0x01 - extract from bits 255:128
 8627   // 0x02 - extract from bits 383:256
 8628   // 0x03 - extract from bits 511:384
 8629   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8630 }
 8631 
 8632 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8633   assert(VM_Version::supports_evex(), "");
 8634   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8635   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8636   attributes.set_is_evex_instruction();
 8637   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8638   // imm8:
 8639   // 0x00 - extract from lower 256 bits
 8640   // 0x01 - extract from upper 256 bits
 8641   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8642 }
 8643 
 8644 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8645   assert(VM_Version::supports_evex(), "");
 8646   assert(src != xnoreg, "sanity");
 8647   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8648   InstructionMark im(this);
 8649   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8650   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8651   attributes.reset_is_clear_context();
 8652   attributes.set_is_evex_instruction();
 8653   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8654   emit_int8(0x1B);
 8655   emit_operand(src, dst, 1);
 8656   // 0x00 - extract from lower 256 bits
 8657   // 0x01 - extract from upper 256 bits
 8658   emit_int8(imm8 & 0x01);
 8659 }
 8660 
 8661 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8662 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8663   assert(VM_Version::supports_avx2(), "");
 8664   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8665   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8666   emit_int16(0x78, (0xC0 | encode));
 8667 }
 8668 
 8669 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8670   assert(VM_Version::supports_avx2(), "");
 8671   assert(dst != xnoreg, "sanity");
 8672   InstructionMark im(this);
 8673   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8674   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8675   // swap src<->dst for encoding
 8676   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8677   emit_int8(0x78);
 8678   emit_operand(dst, src, 0);
 8679 }
 8680 
 8681 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8682 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8683   assert(VM_Version::supports_avx2(), "");
 8684   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8685   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8686   emit_int16(0x79, (0xC0 | encode));
 8687 }
 8688 
 8689 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8690   assert(VM_Version::supports_avx2(), "");
 8691   assert(dst != xnoreg, "sanity");
 8692   InstructionMark im(this);
 8693   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8694   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8695   // swap src<->dst for encoding
 8696   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8697   emit_int8(0x79);
 8698   emit_operand(dst, src, 0);
 8699 }
 8700 
 8701 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8702   assert(UseAVX > 0, "requires some form of AVX");
 8703   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8705   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8706 }
 8707 
 8708 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8709   assert(UseAVX > 0, "requires some form of AVX");
 8710   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8711   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8712   emit_int16(0x69, (0xC0 | encode));
 8713 }
 8714 
 8715 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8716   assert(UseAVX > 0, "requires some form of AVX");
 8717   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8718   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8719   emit_int16(0x61, (0xC0 | encode));
 8720 }
 8721 
 8722 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8723   assert(UseAVX > 0, "requires some form of AVX");
 8724   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8725   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8726   emit_int16(0x6A, (0xC0 | encode));
 8727 }
 8728 
 8729 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8730   assert(UseAVX > 0, "requires some form of AVX");
 8731   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8732   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8733   emit_int16(0x62, (0xC0 | encode));
 8734 }
 8735 
 8736 // xmm/mem sourced byte/word/dword/qword replicate
 8737 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8738   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8739   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8740   attributes.set_is_evex_instruction();
 8741   attributes.set_embedded_opmask_register_specifier(mask);
 8742   if (merge) {
 8743     attributes.reset_is_clear_context();
 8744   }
 8745   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8746   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8747 }
 8748 
 8749 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8750   InstructionMark im(this);
 8751   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8752   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8753   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8754   attributes.set_is_evex_instruction();
 8755   attributes.set_embedded_opmask_register_specifier(mask);
 8756   if (merge) {
 8757     attributes.reset_is_clear_context();
 8758   }
 8759   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8760   emit_int8((unsigned char)0xFC);
 8761   emit_operand(dst, src, 0);
 8762 }
 8763 
 8764 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8765   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8766   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8767   attributes.set_is_evex_instruction();
 8768   attributes.set_embedded_opmask_register_specifier(mask);
 8769   if (merge) {
 8770     attributes.reset_is_clear_context();
 8771   }
 8772   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8773   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8774 }
 8775 
 8776 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8777   InstructionMark im(this);
 8778   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8779   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8780   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8781   attributes.set_is_evex_instruction();
 8782   attributes.set_embedded_opmask_register_specifier(mask);
 8783   if (merge) {
 8784     attributes.reset_is_clear_context();
 8785   }
 8786   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8787   emit_int8((unsigned char)0xFD);
 8788   emit_operand(dst, src, 0);
 8789 }
 8790 
 8791 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8792   assert(VM_Version::supports_evex(), "");
 8793   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8794   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8795   attributes.set_is_evex_instruction();
 8796   attributes.set_embedded_opmask_register_specifier(mask);
 8797   if (merge) {
 8798     attributes.reset_is_clear_context();
 8799   }
 8800   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8801   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8802 }
 8803 
 8804 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8805   InstructionMark im(this);
 8806   assert(VM_Version::supports_evex(), "");
 8807   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8808   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8809   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8810   attributes.set_is_evex_instruction();
 8811   attributes.set_embedded_opmask_register_specifier(mask);
 8812   if (merge) {
 8813     attributes.reset_is_clear_context();
 8814   }
 8815   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8816   emit_int8((unsigned char)0xFE);
 8817   emit_operand(dst, src, 0);
 8818 }
 8819 
 8820 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8821   assert(VM_Version::supports_evex(), "");
 8822   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8823   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8824   attributes.set_is_evex_instruction();
 8825   attributes.set_embedded_opmask_register_specifier(mask);
 8826   if (merge) {
 8827     attributes.reset_is_clear_context();
 8828   }
 8829   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8830   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8831 }
 8832 
 8833 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8834   InstructionMark im(this);
 8835   assert(VM_Version::supports_evex(), "");
 8836   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8837   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8838   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8839   attributes.set_is_evex_instruction();
 8840   attributes.set_embedded_opmask_register_specifier(mask);
 8841   if (merge) {
 8842     attributes.reset_is_clear_context();
 8843   }
 8844   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8845   emit_int8((unsigned char)0xD4);
 8846   emit_operand(dst, src, 0);
 8847 }
 8848 
 8849 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8850   assert(VM_Version::supports_evex(), "");
 8851   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8852   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8853   attributes.set_is_evex_instruction();
 8854   attributes.set_embedded_opmask_register_specifier(mask);
 8855   if (merge) {
 8856     attributes.reset_is_clear_context();
 8857   }
 8858   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8859   emit_int16(0x58, (0xC0 | encode));
 8860 }
 8861 
 8862 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8863   InstructionMark im(this);
 8864   assert(VM_Version::supports_evex(), "");
 8865   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8866   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8867   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8868   attributes.set_is_evex_instruction();
 8869   attributes.set_embedded_opmask_register_specifier(mask);
 8870   if (merge) {
 8871     attributes.reset_is_clear_context();
 8872   }
 8873   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8874   emit_int8(0x58);
 8875   emit_operand(dst, src, 0);
 8876 }
 8877 
 8878 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8879   assert(VM_Version::supports_evex(), "");
 8880   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8881   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8882   attributes.set_is_evex_instruction();
 8883   attributes.set_embedded_opmask_register_specifier(mask);
 8884   if (merge) {
 8885     attributes.reset_is_clear_context();
 8886   }
 8887   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8888   emit_int16(0x58, (0xC0 | encode));
 8889 }
 8890 
 8891 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8892   InstructionMark im(this);
 8893   assert(VM_Version::supports_evex(), "");
 8894   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8895   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8896   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8897   attributes.set_is_evex_instruction();
 8898   attributes.set_embedded_opmask_register_specifier(mask);
 8899   if (merge) {
 8900     attributes.reset_is_clear_context();
 8901   }
 8902   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8903   emit_int8(0x58);
 8904   emit_operand(dst, src, 0);
 8905 }
 8906 
 8907 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8908   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8909   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8910   attributes.set_is_evex_instruction();
 8911   attributes.set_embedded_opmask_register_specifier(mask);
 8912   if (merge) {
 8913     attributes.reset_is_clear_context();
 8914   }
 8915   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8916   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8917 }
 8918 
 8919 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8920   InstructionMark im(this);
 8921   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8922   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8923   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8924   attributes.set_is_evex_instruction();
 8925   attributes.set_embedded_opmask_register_specifier(mask);
 8926   if (merge) {
 8927     attributes.reset_is_clear_context();
 8928   }
 8929   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8930   emit_int8((unsigned char)0xF8);
 8931   emit_operand(dst, src, 0);
 8932 }
 8933 
 8934 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8935   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8936   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8937   attributes.set_is_evex_instruction();
 8938   attributes.set_embedded_opmask_register_specifier(mask);
 8939   if (merge) {
 8940     attributes.reset_is_clear_context();
 8941   }
 8942   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8943   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8944 }
 8945 
 8946 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8947   InstructionMark im(this);
 8948   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8949   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8950   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8951   attributes.set_is_evex_instruction();
 8952   attributes.set_embedded_opmask_register_specifier(mask);
 8953   if (merge) {
 8954     attributes.reset_is_clear_context();
 8955   }
 8956   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8957   emit_int8((unsigned char)0xF9);
 8958   emit_operand(dst, src, 0);
 8959 }
 8960 
 8961 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8962   assert(VM_Version::supports_evex(), "");
 8963   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8964   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8965   attributes.set_is_evex_instruction();
 8966   attributes.set_embedded_opmask_register_specifier(mask);
 8967   if (merge) {
 8968     attributes.reset_is_clear_context();
 8969   }
 8970   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8971   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8972 }
 8973 
 8974 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8975   InstructionMark im(this);
 8976   assert(VM_Version::supports_evex(), "");
 8977   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8978   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8979   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8980   attributes.set_is_evex_instruction();
 8981   attributes.set_embedded_opmask_register_specifier(mask);
 8982   if (merge) {
 8983     attributes.reset_is_clear_context();
 8984   }
 8985   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8986   emit_int8((unsigned char)0xFA);
 8987   emit_operand(dst, src, 0);
 8988 }
 8989 
 8990 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8991   assert(VM_Version::supports_evex(), "");
 8992   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8993   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8994   attributes.set_is_evex_instruction();
 8995   attributes.set_embedded_opmask_register_specifier(mask);
 8996   if (merge) {
 8997     attributes.reset_is_clear_context();
 8998   }
 8999   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9000   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 9001 }
 9002 
 9003 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9004   InstructionMark im(this);
 9005   assert(VM_Version::supports_evex(), "");
 9006   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9007   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9008   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9009   attributes.set_is_evex_instruction();
 9010   attributes.set_embedded_opmask_register_specifier(mask);
 9011   if (merge) {
 9012     attributes.reset_is_clear_context();
 9013   }
 9014   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9015   emit_int8((unsigned char)0xFB);
 9016   emit_operand(dst, src, 0);
 9017 }
 9018 
 9019 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9020   assert(VM_Version::supports_evex(), "");
 9021   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9023   attributes.set_is_evex_instruction();
 9024   attributes.set_embedded_opmask_register_specifier(mask);
 9025   if (merge) {
 9026     attributes.reset_is_clear_context();
 9027   }
 9028   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9029   emit_int16(0x5C, (0xC0 | encode));
 9030 }
 9031 
 9032 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9033   InstructionMark im(this);
 9034   assert(VM_Version::supports_evex(), "");
 9035   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9036   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9037   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9038   attributes.set_is_evex_instruction();
 9039   attributes.set_embedded_opmask_register_specifier(mask);
 9040   if (merge) {
 9041     attributes.reset_is_clear_context();
 9042   }
 9043   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9044   emit_int8(0x5C);
 9045   emit_operand(dst, src, 0);
 9046 }
 9047 
 9048 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9049   assert(VM_Version::supports_evex(), "");
 9050   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9051   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9052   attributes.set_is_evex_instruction();
 9053   attributes.set_embedded_opmask_register_specifier(mask);
 9054   if (merge) {
 9055     attributes.reset_is_clear_context();
 9056   }
 9057   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9058   emit_int16(0x5C, (0xC0 | encode));
 9059 }
 9060 
 9061 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9062   InstructionMark im(this);
 9063   assert(VM_Version::supports_evex(), "");
 9064   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9065   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9066   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9067   attributes.set_is_evex_instruction();
 9068   attributes.set_embedded_opmask_register_specifier(mask);
 9069   if (merge) {
 9070     attributes.reset_is_clear_context();
 9071   }
 9072   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9073   emit_int8(0x5C);
 9074   emit_operand(dst, src, 0);
 9075 }
 9076 
 9077 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9078   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9079   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9080   attributes.set_is_evex_instruction();
 9081   attributes.set_embedded_opmask_register_specifier(mask);
 9082   if (merge) {
 9083     attributes.reset_is_clear_context();
 9084   }
 9085   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9086   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 9087 }
 9088 
 9089 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9090   InstructionMark im(this);
 9091   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9092   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9093   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9094   attributes.set_is_evex_instruction();
 9095   attributes.set_embedded_opmask_register_specifier(mask);
 9096   if (merge) {
 9097     attributes.reset_is_clear_context();
 9098   }
 9099   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9100   emit_int8((unsigned char)0xD5);
 9101   emit_operand(dst, src, 0);
 9102 }
 9103 
 9104 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9105   assert(VM_Version::supports_evex(), "");
 9106   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9107   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9108   attributes.set_is_evex_instruction();
 9109   attributes.set_embedded_opmask_register_specifier(mask);
 9110   if (merge) {
 9111     attributes.reset_is_clear_context();
 9112   }
 9113   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9114   emit_int16(0x40, (0xC0 | encode));
 9115 }
 9116 
 9117 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9118   InstructionMark im(this);
 9119   assert(VM_Version::supports_evex(), "");
 9120   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9121   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9122   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9123   attributes.set_is_evex_instruction();
 9124   attributes.set_embedded_opmask_register_specifier(mask);
 9125   if (merge) {
 9126     attributes.reset_is_clear_context();
 9127   }
 9128   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9129   emit_int8(0x40);
 9130   emit_operand(dst, src, 0);
 9131 }
 9132 
 9133 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9134   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9135   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9136   attributes.set_is_evex_instruction();
 9137   attributes.set_embedded_opmask_register_specifier(mask);
 9138   if (merge) {
 9139     attributes.reset_is_clear_context();
 9140   }
 9141   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9142   emit_int16(0x40, (0xC0 | encode));
 9143 }
 9144 
 9145 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9146   InstructionMark im(this);
 9147   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9148   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9149   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9150   attributes.set_is_evex_instruction();
 9151   attributes.set_embedded_opmask_register_specifier(mask);
 9152   if (merge) {
 9153     attributes.reset_is_clear_context();
 9154   }
 9155   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9156   emit_int8(0x40);
 9157   emit_operand(dst, src, 0);
 9158 }
 9159 
 9160 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9161   assert(VM_Version::supports_evex(), "");
 9162   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9163   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9164   attributes.set_is_evex_instruction();
 9165   attributes.set_embedded_opmask_register_specifier(mask);
 9166   if (merge) {
 9167     attributes.reset_is_clear_context();
 9168   }
 9169   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9170   emit_int16(0x59, (0xC0 | encode));
 9171 }
 9172 
 9173 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9174   InstructionMark im(this);
 9175   assert(VM_Version::supports_evex(), "");
 9176   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9177   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9178   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9179   attributes.set_is_evex_instruction();
 9180   attributes.set_embedded_opmask_register_specifier(mask);
 9181   if (merge) {
 9182     attributes.reset_is_clear_context();
 9183   }
 9184   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9185   emit_int8(0x59);
 9186   emit_operand(dst, src, 0);
 9187 }
 9188 
 9189 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9190   assert(VM_Version::supports_evex(), "");
 9191   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9192   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9193   attributes.set_is_evex_instruction();
 9194   attributes.set_embedded_opmask_register_specifier(mask);
 9195   if (merge) {
 9196     attributes.reset_is_clear_context();
 9197   }
 9198   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9199   emit_int16(0x59, (0xC0 | encode));
 9200 }
 9201 
 9202 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9203   InstructionMark im(this);
 9204   assert(VM_Version::supports_evex(), "");
 9205   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9206   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9207   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9208   attributes.set_is_evex_instruction();
 9209   attributes.set_embedded_opmask_register_specifier(mask);
 9210   if (merge) {
 9211     attributes.reset_is_clear_context();
 9212   }
 9213   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9214   emit_int8(0x59);
 9215   emit_operand(dst, src, 0);
 9216 }
 9217 
 9218 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9219   assert(VM_Version::supports_evex(), "");
 9220   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9221   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9222   attributes.set_is_evex_instruction();
 9223   attributes.set_embedded_opmask_register_specifier(mask);
 9224   if (merge) {
 9225     attributes.reset_is_clear_context();
 9226   }
 9227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9228   emit_int16(0x51, (0xC0 | encode));
 9229 }
 9230 
 9231 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9232   InstructionMark im(this);
 9233   assert(VM_Version::supports_evex(), "");
 9234   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9235   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9236   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9237   attributes.set_is_evex_instruction();
 9238   attributes.set_embedded_opmask_register_specifier(mask);
 9239   if (merge) {
 9240     attributes.reset_is_clear_context();
 9241   }
 9242   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9243   emit_int8(0x51);
 9244   emit_operand(dst, src, 0);
 9245 }
 9246 
 9247 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9248   assert(VM_Version::supports_evex(), "");
 9249   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9250   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9251   attributes.set_is_evex_instruction();
 9252   attributes.set_embedded_opmask_register_specifier(mask);
 9253   if (merge) {
 9254     attributes.reset_is_clear_context();
 9255   }
 9256   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9257   emit_int16(0x51, (0xC0 | encode));
 9258 }
 9259 
 9260 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9261   InstructionMark im(this);
 9262   assert(VM_Version::supports_evex(), "");
 9263   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9264   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9265   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9266   attributes.set_is_evex_instruction();
 9267   attributes.set_embedded_opmask_register_specifier(mask);
 9268   if (merge) {
 9269     attributes.reset_is_clear_context();
 9270   }
 9271   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9272   emit_int8(0x51);
 9273   emit_operand(dst, src, 0);
 9274 }
 9275 
 9276 
 9277 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9278   assert(VM_Version::supports_evex(), "");
 9279   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9280   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9281   attributes.set_is_evex_instruction();
 9282   attributes.set_embedded_opmask_register_specifier(mask);
 9283   if (merge) {
 9284     attributes.reset_is_clear_context();
 9285   }
 9286   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9287   emit_int16(0x5E, (0xC0 | encode));
 9288 }
 9289 
 9290 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9291   InstructionMark im(this);
 9292   assert(VM_Version::supports_evex(), "");
 9293   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9294   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9295   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9296   attributes.set_is_evex_instruction();
 9297   attributes.set_embedded_opmask_register_specifier(mask);
 9298   if (merge) {
 9299     attributes.reset_is_clear_context();
 9300   }
 9301   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9302   emit_int8(0x5E);
 9303   emit_operand(dst, src, 0);
 9304 }
 9305 
 9306 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9307   assert(VM_Version::supports_evex(), "");
 9308   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9309   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9310   attributes.set_is_evex_instruction();
 9311   attributes.set_embedded_opmask_register_specifier(mask);
 9312   if (merge) {
 9313     attributes.reset_is_clear_context();
 9314   }
 9315   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9316   emit_int16(0x5E, (0xC0 | encode));
 9317 }
 9318 
 9319 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9320   InstructionMark im(this);
 9321   assert(VM_Version::supports_evex(), "");
 9322   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9323   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9324   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9325   attributes.set_is_evex_instruction();
 9326   attributes.set_embedded_opmask_register_specifier(mask);
 9327   if (merge) {
 9328     attributes.reset_is_clear_context();
 9329   }
 9330   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9331   emit_int8(0x5E);
 9332   emit_operand(dst, src, 0);
 9333 }
 9334 
 9335 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9336   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9337   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9338   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9339   attributes.set_is_evex_instruction();
 9340   attributes.set_embedded_opmask_register_specifier(mask);
 9341   if (merge) {
 9342     attributes.reset_is_clear_context();
 9343   }
 9344   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9345   emit_int16(0x1C, (0xC0 | encode));
 9346 }
 9347 
 9348 
 9349 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9350   InstructionMark im(this);
 9351   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9352   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9353   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9354   attributes.set_is_evex_instruction();
 9355   attributes.set_embedded_opmask_register_specifier(mask);
 9356   if (merge) {
 9357     attributes.reset_is_clear_context();
 9358   }
 9359   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9360   emit_int8(0x1C);
 9361   emit_operand(dst, src, 0);
 9362 }
 9363 
 9364 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9365   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9366   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9367   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9368   attributes.set_is_evex_instruction();
 9369   attributes.set_embedded_opmask_register_specifier(mask);
 9370   if (merge) {
 9371     attributes.reset_is_clear_context();
 9372   }
 9373   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9374   emit_int16(0x1D, (0xC0 | encode));
 9375 }
 9376 
 9377 
 9378 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9379   InstructionMark im(this);
 9380   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9381   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9382   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9383   attributes.set_is_evex_instruction();
 9384   attributes.set_embedded_opmask_register_specifier(mask);
 9385   if (merge) {
 9386     attributes.reset_is_clear_context();
 9387   }
 9388   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9389   emit_int8(0x1D);
 9390   emit_operand(dst, src, 0);
 9391 }
 9392 
 9393 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9394   assert(VM_Version::supports_evex(), "");
 9395   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9396   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9397   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9398   attributes.set_is_evex_instruction();
 9399   attributes.set_embedded_opmask_register_specifier(mask);
 9400   if (merge) {
 9401     attributes.reset_is_clear_context();
 9402   }
 9403   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9404   emit_int16(0x1E, (0xC0 | encode));
 9405 }
 9406 
 9407 
 9408 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9409   InstructionMark im(this);
 9410   assert(VM_Version::supports_evex(), "");
 9411   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9412   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9413   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9414   attributes.set_is_evex_instruction();
 9415   attributes.set_embedded_opmask_register_specifier(mask);
 9416   if (merge) {
 9417     attributes.reset_is_clear_context();
 9418   }
 9419   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9420   emit_int8(0x1E);
 9421   emit_operand(dst, src, 0);
 9422 }
 9423 
 9424 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9425   assert(VM_Version::supports_evex(), "");
 9426   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9427   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9428   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9429   attributes.set_is_evex_instruction();
 9430   attributes.set_embedded_opmask_register_specifier(mask);
 9431   if (merge) {
 9432     attributes.reset_is_clear_context();
 9433   }
 9434   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9435   emit_int16(0x1F, (0xC0 | encode));
 9436 }
 9437 
 9438 
 9439 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9440   InstructionMark im(this);
 9441   assert(VM_Version::supports_evex(), "");
 9442   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9443   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9444   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9445   attributes.set_is_evex_instruction();
 9446   attributes.set_embedded_opmask_register_specifier(mask);
 9447   if (merge) {
 9448     attributes.reset_is_clear_context();
 9449   }
 9450   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9451   emit_int8(0x1F);
 9452   emit_operand(dst, src, 0);
 9453 }
 9454 
 9455 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9456   assert(VM_Version::supports_evex(), "");
 9457   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9458   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9459   attributes.set_is_evex_instruction();
 9460   attributes.set_embedded_opmask_register_specifier(mask);
 9461   if (merge) {
 9462     attributes.reset_is_clear_context();
 9463   }
 9464   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9465   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9466 }
 9467 
 9468 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9469   InstructionMark im(this);
 9470   assert(VM_Version::supports_evex(), "");
 9471   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9472   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9473   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9474   attributes.set_is_evex_instruction();
 9475   attributes.set_embedded_opmask_register_specifier(mask);
 9476   if (merge) {
 9477     attributes.reset_is_clear_context();
 9478   }
 9479   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9480   emit_int8((unsigned char)0xA8);
 9481   emit_operand(dst, src, 0);
 9482 }
 9483 
 9484 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9485   assert(VM_Version::supports_evex(), "");
 9486   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9487   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9488   attributes.set_is_evex_instruction();
 9489   attributes.set_embedded_opmask_register_specifier(mask);
 9490   if (merge) {
 9491     attributes.reset_is_clear_context();
 9492   }
 9493   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9494   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9495 }
 9496 
 9497 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9498   InstructionMark im(this);
 9499   assert(VM_Version::supports_evex(), "");
 9500   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9501   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9502   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9503   attributes.set_is_evex_instruction();
 9504   attributes.set_embedded_opmask_register_specifier(mask);
 9505   if (merge) {
 9506     attributes.reset_is_clear_context();
 9507   }
 9508   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9509   emit_int8((unsigned char)0xA8);
 9510   emit_operand(dst, src, 0);
 9511 }
 9512 
 9513 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9514   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9515   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9516   attributes.set_is_evex_instruction();
 9517   attributes.set_embedded_opmask_register_specifier(mask);
 9518   if (merge) {
 9519     attributes.reset_is_clear_context();
 9520   }
 9521   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9522   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9523 }
 9524 
 9525 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9526   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9527   InstructionMark im(this);
 9528   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9529   attributes.set_is_evex_instruction();
 9530   attributes.set_embedded_opmask_register_specifier(mask);
 9531   if (merge) {
 9532     attributes.reset_is_clear_context();
 9533   }
 9534   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9535   emit_int8((unsigned char)0x8D);
 9536   emit_operand(dst, src, 0);
 9537 }
 9538 
 9539 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9540   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9541   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9542   attributes.set_is_evex_instruction();
 9543   attributes.set_embedded_opmask_register_specifier(mask);
 9544   if (merge) {
 9545     attributes.reset_is_clear_context();
 9546   }
 9547   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9548   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9549 }
 9550 
 9551 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9552   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9553   InstructionMark im(this);
 9554   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9555   attributes.set_is_evex_instruction();
 9556   attributes.set_embedded_opmask_register_specifier(mask);
 9557   if (merge) {
 9558     attributes.reset_is_clear_context();
 9559   }
 9560   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9561   emit_int8((unsigned char)0x8D);
 9562   emit_operand(dst, src, 0);
 9563 }
 9564 
 9565 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9566   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9567   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9568   attributes.set_is_evex_instruction();
 9569   attributes.set_embedded_opmask_register_specifier(mask);
 9570   if (merge) {
 9571     attributes.reset_is_clear_context();
 9572   }
 9573   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9574   emit_int16(0x36, (0xC0 | encode));
 9575 }
 9576 
 9577 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9578   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9579   InstructionMark im(this);
 9580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9581   attributes.set_is_evex_instruction();
 9582   attributes.set_embedded_opmask_register_specifier(mask);
 9583   if (merge) {
 9584     attributes.reset_is_clear_context();
 9585   }
 9586   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9587   emit_int8(0x36);
 9588   emit_operand(dst, src, 0);
 9589 }
 9590 
 9591 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9592   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9593   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9594   attributes.set_is_evex_instruction();
 9595   attributes.set_embedded_opmask_register_specifier(mask);
 9596   if (merge) {
 9597     attributes.reset_is_clear_context();
 9598   }
 9599   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9600   emit_int16(0x36, (0xC0 | encode));
 9601 }
 9602 
 9603 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9604   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9605   InstructionMark im(this);
 9606   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9607   attributes.set_is_evex_instruction();
 9608   attributes.set_embedded_opmask_register_specifier(mask);
 9609   if (merge) {
 9610     attributes.reset_is_clear_context();
 9611   }
 9612   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9613   emit_int8(0x36);
 9614   emit_operand(dst, src, 0);
 9615 }
 9616 
 9617 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9618   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9619   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9620   attributes.set_is_evex_instruction();
 9621   attributes.set_embedded_opmask_register_specifier(mask);
 9622   if (merge) {
 9623     attributes.reset_is_clear_context();
 9624   }
 9625   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9626   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9627 }
 9628 
 9629 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9630   assert(VM_Version::supports_evex(), "");
 9631   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9633   attributes.set_is_evex_instruction();
 9634   attributes.set_embedded_opmask_register_specifier(mask);
 9635   if (merge) {
 9636     attributes.reset_is_clear_context();
 9637   }
 9638   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9639   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9640 }
 9641 
 9642 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9643   assert(VM_Version::supports_evex(), "");
 9644   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9645   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9646   attributes.set_is_evex_instruction();
 9647   attributes.set_embedded_opmask_register_specifier(mask);
 9648   if (merge) {
 9649     attributes.reset_is_clear_context();
 9650   }
 9651   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9652   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9653 }
 9654 
 9655 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9656   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9657   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9658   attributes.set_is_evex_instruction();
 9659   attributes.set_embedded_opmask_register_specifier(mask);
 9660   if (merge) {
 9661     attributes.reset_is_clear_context();
 9662   }
 9663   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9664   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9665 }
 9666 
 9667 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9668   assert(VM_Version::supports_evex(), "");
 9669   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9670   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9671   attributes.set_is_evex_instruction();
 9672   attributes.set_embedded_opmask_register_specifier(mask);
 9673   if (merge) {
 9674     attributes.reset_is_clear_context();
 9675   }
 9676   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9677   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9678 }
 9679 
 9680 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9681   assert(VM_Version::supports_evex(), "");
 9682   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9683   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9684   attributes.set_is_evex_instruction();
 9685   attributes.set_embedded_opmask_register_specifier(mask);
 9686   if (merge) {
 9687     attributes.reset_is_clear_context();
 9688   }
 9689   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9690   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9691 }
 9692 
 9693 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9694   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9696   attributes.set_is_evex_instruction();
 9697   attributes.set_embedded_opmask_register_specifier(mask);
 9698   if (merge) {
 9699     attributes.reset_is_clear_context();
 9700   }
 9701   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9702   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9703 }
 9704 
 9705 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9706   assert(VM_Version::supports_evex(), "");
 9707   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9708   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9709   attributes.set_is_evex_instruction();
 9710   attributes.set_embedded_opmask_register_specifier(mask);
 9711   if (merge) {
 9712     attributes.reset_is_clear_context();
 9713   }
 9714   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9715   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9716 }
 9717 
 9718 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9719   assert(VM_Version::supports_evex(), "");
 9720   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9721   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9722   attributes.set_is_evex_instruction();
 9723   attributes.set_embedded_opmask_register_specifier(mask);
 9724   if (merge) {
 9725     attributes.reset_is_clear_context();
 9726   }
 9727   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9728   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9729 }
 9730 
 9731 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9732   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9733   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9734   attributes.set_is_evex_instruction();
 9735   attributes.set_embedded_opmask_register_specifier(mask);
 9736   if (merge) {
 9737     attributes.reset_is_clear_context();
 9738   }
 9739   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9740   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9741 }
 9742 
 9743 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9744   assert(VM_Version::supports_evex(), "");
 9745   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9746   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9747   attributes.set_is_evex_instruction();
 9748   attributes.set_embedded_opmask_register_specifier(mask);
 9749   if (merge) {
 9750     attributes.reset_is_clear_context();
 9751   }
 9752   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9753   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9754 }
 9755 
 9756 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9757   assert(VM_Version::supports_evex(), "");
 9758   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9759   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9760   attributes.set_is_evex_instruction();
 9761   attributes.set_embedded_opmask_register_specifier(mask);
 9762   if (merge) {
 9763     attributes.reset_is_clear_context();
 9764   }
 9765   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9766   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9767 }
 9768 
 9769 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9770   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9771   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9772   attributes.set_is_evex_instruction();
 9773   attributes.set_embedded_opmask_register_specifier(mask);
 9774   if (merge) {
 9775     attributes.reset_is_clear_context();
 9776   }
 9777   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9778   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9779 }
 9780 
 9781 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9782   assert(VM_Version::supports_evex(), "");
 9783   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9784   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9785   attributes.set_is_evex_instruction();
 9786   attributes.set_embedded_opmask_register_specifier(mask);
 9787   if (merge) {
 9788     attributes.reset_is_clear_context();
 9789   }
 9790   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9791   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9792 }
 9793 
 9794 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9795   assert(VM_Version::supports_evex(), "");
 9796   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9797   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9798   attributes.set_is_evex_instruction();
 9799   attributes.set_embedded_opmask_register_specifier(mask);
 9800   if (merge) {
 9801     attributes.reset_is_clear_context();
 9802   }
 9803   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9804   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9805 }
 9806 
 9807 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9808   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9809   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9810   attributes.set_is_evex_instruction();
 9811   attributes.set_embedded_opmask_register_specifier(mask);
 9812   if (merge) {
 9813     attributes.reset_is_clear_context();
 9814   }
 9815   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9816   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9817 }
 9818 
 9819 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9820   assert(VM_Version::supports_evex(), "");
 9821   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9822   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9823   attributes.set_is_evex_instruction();
 9824   attributes.set_embedded_opmask_register_specifier(mask);
 9825   if (merge) {
 9826     attributes.reset_is_clear_context();
 9827   }
 9828   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9829   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9830 }
 9831 
 9832 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9833   assert(VM_Version::supports_evex(), "");
 9834   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9835   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9836   attributes.set_is_evex_instruction();
 9837   attributes.set_embedded_opmask_register_specifier(mask);
 9838   if (merge) {
 9839     attributes.reset_is_clear_context();
 9840   }
 9841   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9842   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9843 }
 9844 
 9845 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9846   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9847   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9848   attributes.set_is_evex_instruction();
 9849   attributes.set_embedded_opmask_register_specifier(mask);
 9850   if (merge) {
 9851     attributes.reset_is_clear_context();
 9852   }
 9853   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9854   emit_int16(0x12, (0xC0 | encode));
 9855 }
 9856 
 9857 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9858   assert(VM_Version::supports_evex(), "");
 9859   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9860   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9861   attributes.set_is_evex_instruction();
 9862   attributes.set_embedded_opmask_register_specifier(mask);
 9863   if (merge) {
 9864     attributes.reset_is_clear_context();
 9865   }
 9866   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9867   emit_int16(0x47, (0xC0 | encode));
 9868 }
 9869 
 9870 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9871   assert(VM_Version::supports_evex(), "");
 9872   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9873   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9874   attributes.set_is_evex_instruction();
 9875   attributes.set_embedded_opmask_register_specifier(mask);
 9876   if (merge) {
 9877     attributes.reset_is_clear_context();
 9878   }
 9879   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9880   emit_int16(0x47, (0xC0 | encode));
 9881 }
 9882 
 9883 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9884   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9885   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9886   attributes.set_is_evex_instruction();
 9887   attributes.set_embedded_opmask_register_specifier(mask);
 9888   if (merge) {
 9889     attributes.reset_is_clear_context();
 9890   }
 9891   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9892   emit_int16(0x10, (0xC0 | encode));
 9893 }
 9894 
 9895 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9896   assert(VM_Version::supports_evex(), "");
 9897   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9898   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9899   attributes.set_is_evex_instruction();
 9900   attributes.set_embedded_opmask_register_specifier(mask);
 9901   if (merge) {
 9902     attributes.reset_is_clear_context();
 9903   }
 9904   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9905   emit_int16(0x45, (0xC0 | encode));
 9906 }
 9907 
 9908 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9909   assert(VM_Version::supports_evex(), "");
 9910   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9911   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9912   attributes.set_is_evex_instruction();
 9913   attributes.set_embedded_opmask_register_specifier(mask);
 9914   if (merge) {
 9915     attributes.reset_is_clear_context();
 9916   }
 9917   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9918   emit_int16(0x45, (0xC0 | encode));
 9919 }
 9920 
 9921 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9922   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9923   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9924   attributes.set_is_evex_instruction();
 9925   attributes.set_embedded_opmask_register_specifier(mask);
 9926   if (merge) {
 9927     attributes.reset_is_clear_context();
 9928   }
 9929   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9930   emit_int16(0x11, (0xC0 | encode));
 9931 }
 9932 
 9933 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9934   assert(VM_Version::supports_evex(), "");
 9935   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9936   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9937   attributes.set_is_evex_instruction();
 9938   attributes.set_embedded_opmask_register_specifier(mask);
 9939   if (merge) {
 9940     attributes.reset_is_clear_context();
 9941   }
 9942   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9943   emit_int16(0x46, (0xC0 | encode));
 9944 }
 9945 
 9946 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9947   assert(VM_Version::supports_evex(), "");
 9948   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9949   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9950   attributes.set_is_evex_instruction();
 9951   attributes.set_embedded_opmask_register_specifier(mask);
 9952   if (merge) {
 9953     attributes.reset_is_clear_context();
 9954   }
 9955   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9956   emit_int16(0x46, (0xC0 | encode));
 9957 }
 9958 
 9959 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9960   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9961   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9962   attributes.set_is_evex_instruction();
 9963   attributes.set_embedded_opmask_register_specifier(mask);
 9964   if (merge) {
 9965     attributes.reset_is_clear_context();
 9966   }
 9967   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9968   emit_int16(0x38, (0xC0 | encode));
 9969 }
 9970 
 9971 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9972   assert(VM_Version::supports_avx512bw(), "");
 9973   InstructionMark im(this);
 9974   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9975   attributes.set_is_evex_instruction();
 9976   attributes.set_embedded_opmask_register_specifier(mask);
 9977   if (merge) {
 9978     attributes.reset_is_clear_context();
 9979   }
 9980   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9981   emit_int8(0x38);
 9982   emit_operand(dst, src, 0);
 9983 }
 9984 
 9985 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9986   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9988   attributes.set_is_evex_instruction();
 9989   attributes.set_embedded_opmask_register_specifier(mask);
 9990   if (merge) {
 9991     attributes.reset_is_clear_context();
 9992   }
 9993   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9994   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9995 }
 9996 
 9997 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9998   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9999   InstructionMark im(this);
10000   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10001   attributes.set_is_evex_instruction();
10002   attributes.set_embedded_opmask_register_specifier(mask);
10003   if (merge) {
10004     attributes.reset_is_clear_context();
10005   }
10006   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10007   emit_int8((unsigned char)0xEA);
10008   emit_operand(dst, src, 0);
10009 }
10010 
10011 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10012   assert(VM_Version::supports_evex(), "");
10013   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10015   attributes.set_is_evex_instruction();
10016   attributes.set_embedded_opmask_register_specifier(mask);
10017   if (merge) {
10018     attributes.reset_is_clear_context();
10019   }
10020   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10021   emit_int16(0x39, (0xC0 | encode));
10022 }
10023 
10024 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10025   assert(VM_Version::supports_evex(), "");
10026   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10027   InstructionMark im(this);
10028   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10029   attributes.set_is_evex_instruction();
10030   attributes.set_embedded_opmask_register_specifier(mask);
10031   if (merge) {
10032     attributes.reset_is_clear_context();
10033   }
10034   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10035   emit_int8(0x39);
10036   emit_operand(dst, src, 0);
10037 }
10038 
10039 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10040   assert(VM_Version::supports_evex(), "");
10041   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10042   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10043   attributes.set_is_evex_instruction();
10044   attributes.set_embedded_opmask_register_specifier(mask);
10045   if (merge) {
10046     attributes.reset_is_clear_context();
10047   }
10048   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10049   emit_int16(0x39, (0xC0 | encode));
10050 }
10051 
10052 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10053   assert(VM_Version::supports_evex(), "");
10054   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10055   InstructionMark im(this);
10056   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10057   attributes.set_is_evex_instruction();
10058   attributes.set_embedded_opmask_register_specifier(mask);
10059   if (merge) {
10060     attributes.reset_is_clear_context();
10061   }
10062   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10063   emit_int8(0x39);
10064   emit_operand(dst, src, 0);
10065 }
10066 
10067 
10068 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10069   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10070   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10071   attributes.set_is_evex_instruction();
10072   attributes.set_embedded_opmask_register_specifier(mask);
10073   if (merge) {
10074     attributes.reset_is_clear_context();
10075   }
10076   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10077   emit_int16(0x3C, (0xC0 | encode));
10078 }
10079 
10080 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10081   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10082   InstructionMark im(this);
10083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10084   attributes.set_is_evex_instruction();
10085   attributes.set_embedded_opmask_register_specifier(mask);
10086   if (merge) {
10087     attributes.reset_is_clear_context();
10088   }
10089   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10090   emit_int8(0x3C);
10091   emit_operand(dst, src, 0);
10092 }
10093 
10094 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10095   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10096   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10097   attributes.set_is_evex_instruction();
10098   attributes.set_embedded_opmask_register_specifier(mask);
10099   if (merge) {
10100     attributes.reset_is_clear_context();
10101   }
10102   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10103   emit_int16((unsigned char)0xEE, (0xC0 | encode));
10104 }
10105 
10106 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10107   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10108   InstructionMark im(this);
10109   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10110   attributes.set_is_evex_instruction();
10111   attributes.set_embedded_opmask_register_specifier(mask);
10112   if (merge) {
10113     attributes.reset_is_clear_context();
10114   }
10115   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10116   emit_int8((unsigned char)0xEE);
10117   emit_operand(dst, src, 0);
10118 }
10119 
10120 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10121   assert(VM_Version::supports_evex(), "");
10122   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10124   attributes.set_is_evex_instruction();
10125   attributes.set_embedded_opmask_register_specifier(mask);
10126   if (merge) {
10127     attributes.reset_is_clear_context();
10128   }
10129   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10130   emit_int16(0x3D, (0xC0 | encode));
10131 }
10132 
10133 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10134   assert(VM_Version::supports_evex(), "");
10135   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10136   InstructionMark im(this);
10137   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10138   attributes.set_is_evex_instruction();
10139   attributes.set_embedded_opmask_register_specifier(mask);
10140   if (merge) {
10141     attributes.reset_is_clear_context();
10142   }
10143   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10144   emit_int8(0x3D);
10145   emit_operand(dst, src, 0);
10146 }
10147 
10148 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10149   assert(VM_Version::supports_evex(), "");
10150   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10151   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10152   attributes.set_is_evex_instruction();
10153   attributes.set_embedded_opmask_register_specifier(mask);
10154   if (merge) {
10155     attributes.reset_is_clear_context();
10156   }
10157   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10158   emit_int16(0x3D, (0xC0 | encode));
10159 }
10160 
10161 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10162   assert(VM_Version::supports_evex(), "");
10163   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10164   InstructionMark im(this);
10165   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10166   attributes.set_is_evex_instruction();
10167   attributes.set_embedded_opmask_register_specifier(mask);
10168   if (merge) {
10169     attributes.reset_is_clear_context();
10170   }
10171   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10172   emit_int8(0x3D);
10173   emit_operand(dst, src, 0);
10174 }
10175 
10176 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10177   assert(VM_Version::supports_evex(), "requires EVEX support");
10178   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10179   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10180   attributes.set_is_evex_instruction();
10181   attributes.set_embedded_opmask_register_specifier(mask);
10182   if (merge) {
10183     attributes.reset_is_clear_context();
10184   }
10185   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10186   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10187 }
10188 
10189 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10190   assert(VM_Version::supports_evex(), "requires EVEX support");
10191   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10192   assert(dst != xnoreg, "sanity");
10193   InstructionMark im(this);
10194   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10195   attributes.set_is_evex_instruction();
10196   attributes.set_embedded_opmask_register_specifier(mask);
10197   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10198   if (merge) {
10199     attributes.reset_is_clear_context();
10200   }
10201   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10202   emit_int8(0x25);
10203   emit_operand(dst, src3, 1);
10204   emit_int8(imm8);
10205 }
10206 
10207 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10208   assert(VM_Version::supports_evex(), "requires EVEX support");
10209   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10210   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10211   attributes.set_is_evex_instruction();
10212   attributes.set_embedded_opmask_register_specifier(mask);
10213   if (merge) {
10214     attributes.reset_is_clear_context();
10215   }
10216   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10217   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10218 }
10219 
10220 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10221   assert(VM_Version::supports_evex(), "requires EVEX support");
10222   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10223   assert(dst != xnoreg, "sanity");
10224   InstructionMark im(this);
10225   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10226   attributes.set_is_evex_instruction();
10227   attributes.set_embedded_opmask_register_specifier(mask);
10228   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10229   if (merge) {
10230     attributes.reset_is_clear_context();
10231   }
10232   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10233   emit_int8(0x25);
10234   emit_operand(dst, src3, 1);
10235   emit_int8(imm8);
10236 }
10237 
10238 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10239   assert(VM_Version::supports_gfni(), "");
10240   assert(VM_Version::supports_sse(), "");
10241   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10242   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10243   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10244 }
10245 
10246 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10247   assert(VM_Version::supports_gfni(), "requires GFNI support");
10248   assert(VM_Version::supports_sse(), "");
10249   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10250   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10251   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10252 }
10253 
10254 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10255 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10256   assert(UseAVX >= 2, "");
10257   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10258   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10259   emit_int16(0x58, (0xC0 | encode));
10260 }
10261 
10262 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10263   assert(VM_Version::supports_avx2(), "");
10264   assert(dst != xnoreg, "sanity");
10265   InstructionMark im(this);
10266   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10267   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10268   // swap src<->dst for encoding
10269   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10270   emit_int8(0x58);
10271   emit_operand(dst, src, 0);
10272 }
10273 
10274 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10275 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10276   assert(VM_Version::supports_avx2(), "");
10277   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10278   attributes.set_rex_vex_w_reverted();
10279   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10280   emit_int16(0x59, (0xC0 | encode));
10281 }
10282 
10283 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10284   assert(VM_Version::supports_avx2(), "");
10285   assert(dst != xnoreg, "sanity");
10286   InstructionMark im(this);
10287   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10288   attributes.set_rex_vex_w_reverted();
10289   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10290   // swap src<->dst for encoding
10291   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10292   emit_int8(0x59);
10293   emit_operand(dst, src, 0);
10294 }
10295 
10296 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10297   assert(vector_len != Assembler::AVX_128bit, "");
10298   assert(VM_Version::supports_evex(), "");
10299   assert(dst != xnoreg, "sanity");
10300   InstructionMark im(this);
10301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10302   attributes.set_rex_vex_w_reverted();
10303   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10304   // swap src<->dst for encoding
10305   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10306   emit_int8(0x5A);
10307   emit_operand(dst, src, 0);
10308 }
10309 
10310 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10311   assert(vector_len != Assembler::AVX_128bit, "");
10312   assert(VM_Version::supports_avx512dq(), "");
10313   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10314   attributes.set_rex_vex_w_reverted();
10315   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10316   emit_int16(0x5A, (0xC0 | encode));
10317 }
10318 
10319 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10320   assert(vector_len != Assembler::AVX_128bit, "");
10321   assert(VM_Version::supports_avx512dq(), "");
10322   assert(dst != xnoreg, "sanity");
10323   InstructionMark im(this);
10324   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10325   attributes.set_rex_vex_w_reverted();
10326   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10327   // swap src<->dst for encoding
10328   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10329   emit_int8(0x5A);
10330   emit_operand(dst, src, 0);
10331 }
10332 
10333 // scalar single/double precision replicate
10334 
10335 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10336 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10337   assert(VM_Version::supports_avx2(), "");
10338   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10339   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10340   emit_int16(0x18, (0xC0 | encode));
10341 }
10342 
10343 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10344   assert(VM_Version::supports_avx(), "");
10345   assert(dst != xnoreg, "sanity");
10346   InstructionMark im(this);
10347   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10348   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10349   // swap src<->dst for encoding
10350   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10351   emit_int8(0x18);
10352   emit_operand(dst, src, 0);
10353 }
10354 
10355 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10356 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10357   assert(VM_Version::supports_avx2(), "");
10358   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10359   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10360   attributes.set_rex_vex_w_reverted();
10361   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10362   emit_int16(0x19, (0xC0 | encode));
10363 }
10364 
10365 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10366   assert(VM_Version::supports_avx(), "");
10367   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10368   assert(dst != xnoreg, "sanity");
10369   InstructionMark im(this);
10370   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10371   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10372   attributes.set_rex_vex_w_reverted();
10373   // swap src<->dst for encoding
10374   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10375   emit_int8(0x19);
10376   emit_operand(dst, src, 0);
10377 }
10378 
10379 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10380   assert(VM_Version::supports_avx(), "");
10381   assert(vector_len == AVX_256bit, "");
10382   assert(dst != xnoreg, "sanity");
10383   InstructionMark im(this);
10384   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10385   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10386   // swap src<->dst for encoding
10387   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10388   emit_int8(0x1A);
10389   emit_operand(dst, src, 0);
10390 }
10391 
10392 // gpr source broadcast forms
10393 
10394 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10395 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10396   assert(VM_Version::supports_avx512bw(), "");
10397   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10398   attributes.set_is_evex_instruction();
10399   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10400   emit_int16(0x7A, (0xC0 | encode));
10401 }
10402 
10403 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10404 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10405   assert(VM_Version::supports_avx512bw(), "");
10406   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10407   attributes.set_is_evex_instruction();
10408   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10409   emit_int16(0x7B, (0xC0 | encode));
10410 }
10411 
10412 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10413 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10414   assert(VM_Version::supports_evex(), "");
10415   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10416   attributes.set_is_evex_instruction();
10417   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10418   emit_int16(0x7C, (0xC0 | encode));
10419 }
10420 
10421 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10422 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10423   assert(VM_Version::supports_evex(), "");
10424   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10425   attributes.set_is_evex_instruction();
10426   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10427   emit_int16(0x7C, (0xC0 | encode));
10428 }
10429 
10430 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10431   assert(VM_Version::supports_avx2(), "");
10432   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10433   assert(dst != xnoreg, "sanity");
10434   assert(src.isxmmindex(),"expected to be xmm index");
10435   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10436   InstructionMark im(this);
10437   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10438   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10439   emit_int8((unsigned char)0x90);
10440   emit_operand(dst, src, 0);
10441 }
10442 
10443 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10444   assert(VM_Version::supports_avx2(), "");
10445   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10446   assert(dst != xnoreg, "sanity");
10447   assert(src.isxmmindex(),"expected to be xmm index");
10448   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10449   InstructionMark im(this);
10450   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10451   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10452   emit_int8((unsigned char)0x90);
10453   emit_operand(dst, src, 0);
10454 }
10455 
10456 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10457   assert(VM_Version::supports_avx2(), "");
10458   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10459   assert(dst != xnoreg, "sanity");
10460   assert(src.isxmmindex(),"expected to be xmm index");
10461   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10462   InstructionMark im(this);
10463   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10464   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10465   emit_int8((unsigned char)0x92);
10466   emit_operand(dst, src, 0);
10467 }
10468 
10469 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10470   assert(VM_Version::supports_avx2(), "");
10471   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10472   assert(dst != xnoreg, "sanity");
10473   assert(src.isxmmindex(),"expected to be xmm index");
10474   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10475   InstructionMark im(this);
10476   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10477   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10478   emit_int8((unsigned char)0x92);
10479   emit_operand(dst, src, 0);
10480 }
10481 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10482   assert(VM_Version::supports_evex(), "");
10483   assert(dst != xnoreg, "sanity");
10484   assert(src.isxmmindex(),"expected to be xmm index");
10485   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10486   assert(mask != k0, "instruction will #UD if mask is in k0");
10487   InstructionMark im(this);
10488   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10489   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10490   attributes.reset_is_clear_context();
10491   attributes.set_embedded_opmask_register_specifier(mask);
10492   attributes.set_is_evex_instruction();
10493   // swap src<->dst for encoding
10494   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10495   emit_int8((unsigned char)0x90);
10496   emit_operand(dst, src, 0);
10497 }
10498 
10499 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10500   assert(VM_Version::supports_evex(), "");
10501   assert(dst != xnoreg, "sanity");
10502   assert(src.isxmmindex(),"expected to be xmm index");
10503   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10504   assert(mask != k0, "instruction will #UD if mask is in k0");
10505   InstructionMark im(this);
10506   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10507   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10508   attributes.reset_is_clear_context();
10509   attributes.set_embedded_opmask_register_specifier(mask);
10510   attributes.set_is_evex_instruction();
10511   // swap src<->dst for encoding
10512   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10513   emit_int8((unsigned char)0x90);
10514   emit_operand(dst, src, 0);
10515 }
10516 
10517 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10518   assert(VM_Version::supports_evex(), "");
10519   assert(dst != xnoreg, "sanity");
10520   assert(src.isxmmindex(),"expected to be xmm index");
10521   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10522   assert(mask != k0, "instruction will #UD if mask is in k0");
10523   InstructionMark im(this);
10524   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10526   attributes.reset_is_clear_context();
10527   attributes.set_embedded_opmask_register_specifier(mask);
10528   attributes.set_is_evex_instruction();
10529   // swap src<->dst for encoding
10530   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10531   emit_int8((unsigned char)0x92);
10532   emit_operand(dst, src, 0);
10533 }
10534 
10535 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10536   assert(VM_Version::supports_evex(), "");
10537   assert(dst != xnoreg, "sanity");
10538   assert(src.isxmmindex(),"expected to be xmm index");
10539   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10540   assert(mask != k0, "instruction will #UD if mask is in k0");
10541   InstructionMark im(this);
10542   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10543   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10544   attributes.reset_is_clear_context();
10545   attributes.set_embedded_opmask_register_specifier(mask);
10546   attributes.set_is_evex_instruction();
10547   // swap src<->dst for encoding
10548   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10549   emit_int8((unsigned char)0x92);
10550   emit_operand(dst, src, 0);
10551 }
10552 
10553 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10554   assert(VM_Version::supports_evex(), "");
10555   assert(mask != k0, "instruction will #UD if mask is in k0");
10556   InstructionMark im(this);
10557   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10558   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10559   attributes.reset_is_clear_context();
10560   attributes.set_embedded_opmask_register_specifier(mask);
10561   attributes.set_is_evex_instruction();
10562   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10563   emit_int8((unsigned char)0xA0);
10564   emit_operand(src, dst, 0);
10565 }
10566 
10567 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10568   assert(VM_Version::supports_evex(), "");
10569   assert(mask != k0, "instruction will #UD if mask is in k0");
10570   InstructionMark im(this);
10571   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10572   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10573   attributes.reset_is_clear_context();
10574   attributes.set_embedded_opmask_register_specifier(mask);
10575   attributes.set_is_evex_instruction();
10576   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10577   emit_int8((unsigned char)0xA0);
10578   emit_operand(src, dst, 0);
10579 }
10580 
10581 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10582   assert(VM_Version::supports_evex(), "");
10583   assert(mask != k0, "instruction will #UD if mask is in k0");
10584   InstructionMark im(this);
10585   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10586   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10587   attributes.reset_is_clear_context();
10588   attributes.set_embedded_opmask_register_specifier(mask);
10589   attributes.set_is_evex_instruction();
10590   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10591   emit_int8((unsigned char)0xA2);
10592   emit_operand(src, dst, 0);
10593 }
10594 
10595 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10596   assert(VM_Version::supports_evex(), "");
10597   assert(mask != k0, "instruction will #UD if mask is in k0");
10598   InstructionMark im(this);
10599   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10600   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10601   attributes.reset_is_clear_context();
10602   attributes.set_embedded_opmask_register_specifier(mask);
10603   attributes.set_is_evex_instruction();
10604   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10605   emit_int8((unsigned char)0xA2);
10606   emit_operand(src, dst, 0);
10607 }
10608 // Carry-Less Multiplication Quadword
10609 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10610   assert(VM_Version::supports_clmul(), "");
10611   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10612   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10613   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10614 }
10615 
10616 // Carry-Less Multiplication Quadword
10617 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10618   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10619   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10620   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10621   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10622 }
10623 
10624 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10625   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10626   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10627   attributes.set_is_evex_instruction();
10628   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10629   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10630 }
10631 
10632 void Assembler::vzeroupper_uncached() {
10633   if (VM_Version::supports_vzeroupper()) {
10634     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10635     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10636     emit_int8(0x77);
10637   }
10638 }
10639 
10640 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10641   // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10642   assert(VM_Version::supports_evex(), "");
10643   assert(VM_Version::supports_avx512dq(), "");
10644   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10645   attributes.set_is_evex_instruction();
10646   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10647   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10648 }
10649 
10650 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10651   // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10652   assert(VM_Version::supports_evex(), "");
10653   assert(VM_Version::supports_avx512dq(), "");
10654   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10655   attributes.set_is_evex_instruction();
10656   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10657   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10658 }
10659 
10660 void Assembler::fld_x(Address adr) {
10661   InstructionMark im(this);
10662   emit_int8((unsigned char)0xDB);
10663   emit_operand32(rbp, adr, 0);
10664 }
10665 
10666 void Assembler::fstp_x(Address adr) {
10667   InstructionMark im(this);
10668   emit_int8((unsigned char)0xDB);
10669   emit_operand32(rdi, adr, 0);
10670 }
10671 
10672 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10673   assert(reg->encoding() < 8, "no extended registers");
10674   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10675   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10676 }
10677 
10678 #ifndef _LP64
10679 // 32bit only pieces of the assembler
10680 
10681 void Assembler::emms() {
10682   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10683   emit_int16(0x0F, 0x77);
10684 }
10685 
10686 void Assembler::vzeroupper() {
10687   vzeroupper_uncached();
10688 }
10689 
10690 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10691   // NO PREFIX AS NEVER 64BIT
10692   InstructionMark im(this);
10693   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10694   emit_data(imm32, rspec, 0);
10695 }
10696 
10697 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10698   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10699   InstructionMark im(this);
10700   emit_int8((unsigned char)0x81);
10701   emit_operand(rdi, src1, 4);
10702   emit_data(imm32, rspec, 0);
10703 }
10704 
10705 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10706 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10707 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10708 void Assembler::cmpxchg8(Address adr) {
10709   InstructionMark im(this);
10710   emit_int16(0x0F, (unsigned char)0xC7);
10711   emit_operand(rcx, adr, 0);
10712 }
10713 
10714 void Assembler::decl(Register dst) {
10715   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10716  emit_int8(0x48 | dst->encoding());
10717 }
10718 
10719 // 64bit doesn't use the x87
10720 
10721 void Assembler::emit_farith(int b1, int b2, int i) {
10722   assert(isByte(b1) && isByte(b2), "wrong opcode");
10723   assert(0 <= i &&  i < 8, "illegal stack offset");
10724   emit_int16(b1, b2 + i);
10725 }
10726 
10727 void Assembler::fabs() {
10728   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10729 }
10730 
10731 void Assembler::fadd(int i) {
10732   emit_farith(0xD8, 0xC0, i);
10733 }
10734 
10735 void Assembler::fadd_d(Address src) {
10736   InstructionMark im(this);
10737   emit_int8((unsigned char)0xDC);
10738   emit_operand32(rax, src, 0);
10739 }
10740 
10741 void Assembler::fadd_s(Address src) {
10742   InstructionMark im(this);
10743   emit_int8((unsigned char)0xD8);
10744   emit_operand32(rax, src, 0);
10745 }
10746 
10747 void Assembler::fadda(int i) {
10748   emit_farith(0xDC, 0xC0, i);
10749 }
10750 
10751 void Assembler::faddp(int i) {
10752   emit_farith(0xDE, 0xC0, i);
10753 }
10754 
10755 void Assembler::fchs() {
10756   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10757 }
10758 
10759 void Assembler::fcom(int i) {
10760   emit_farith(0xD8, 0xD0, i);
10761 }
10762 
10763 void Assembler::fcomp(int i) {
10764   emit_farith(0xD8, 0xD8, i);
10765 }
10766 
10767 void Assembler::fcomp_d(Address src) {
10768   InstructionMark im(this);
10769   emit_int8((unsigned char)0xDC);
10770   emit_operand32(rbx, src, 0);
10771 }
10772 
10773 void Assembler::fcomp_s(Address src) {
10774   InstructionMark im(this);
10775   emit_int8((unsigned char)0xD8);
10776   emit_operand32(rbx, src, 0);
10777 }
10778 
10779 void Assembler::fcompp() {
10780   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10781 }
10782 
10783 void Assembler::fcos() {
10784   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10785 }
10786 
10787 void Assembler::fdecstp() {
10788   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10789 }
10790 
10791 void Assembler::fdiv(int i) {
10792   emit_farith(0xD8, 0xF0, i);
10793 }
10794 
10795 void Assembler::fdiv_d(Address src) {
10796   InstructionMark im(this);
10797   emit_int8((unsigned char)0xDC);
10798   emit_operand32(rsi, src, 0);
10799 }
10800 
10801 void Assembler::fdiv_s(Address src) {
10802   InstructionMark im(this);
10803   emit_int8((unsigned char)0xD8);
10804   emit_operand32(rsi, src, 0);
10805 }
10806 
10807 void Assembler::fdiva(int i) {
10808   emit_farith(0xDC, 0xF8, i);
10809 }
10810 
10811 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10812 //       is erroneous for some of the floating-point instructions below.
10813 
10814 void Assembler::fdivp(int i) {
10815   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10816 }
10817 
10818 void Assembler::fdivr(int i) {
10819   emit_farith(0xD8, 0xF8, i);
10820 }
10821 
10822 void Assembler::fdivr_d(Address src) {
10823   InstructionMark im(this);
10824   emit_int8((unsigned char)0xDC);
10825   emit_operand32(rdi, src, 0);
10826 }
10827 
10828 void Assembler::fdivr_s(Address src) {
10829   InstructionMark im(this);
10830   emit_int8((unsigned char)0xD8);
10831   emit_operand32(rdi, src, 0);
10832 }
10833 
10834 void Assembler::fdivra(int i) {
10835   emit_farith(0xDC, 0xF0, i);
10836 }
10837 
10838 void Assembler::fdivrp(int i) {
10839   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10840 }
10841 
10842 void Assembler::ffree(int i) {
10843   emit_farith(0xDD, 0xC0, i);
10844 }
10845 
10846 void Assembler::fild_d(Address adr) {
10847   InstructionMark im(this);
10848   emit_int8((unsigned char)0xDF);
10849   emit_operand32(rbp, adr, 0);
10850 }
10851 
10852 void Assembler::fild_s(Address adr) {
10853   InstructionMark im(this);
10854   emit_int8((unsigned char)0xDB);
10855   emit_operand32(rax, adr, 0);
10856 }
10857 
10858 void Assembler::fincstp() {
10859   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10860 }
10861 
10862 void Assembler::finit() {
10863   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10864 }
10865 
10866 void Assembler::fist_s(Address adr) {
10867   InstructionMark im(this);
10868   emit_int8((unsigned char)0xDB);
10869   emit_operand32(rdx, adr, 0);
10870 }
10871 
10872 void Assembler::fistp_d(Address adr) {
10873   InstructionMark im(this);
10874   emit_int8((unsigned char)0xDF);
10875   emit_operand32(rdi, adr, 0);
10876 }
10877 
10878 void Assembler::fistp_s(Address adr) {
10879   InstructionMark im(this);
10880   emit_int8((unsigned char)0xDB);
10881   emit_operand32(rbx, adr, 0);
10882 }
10883 
10884 void Assembler::fld1() {
10885   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10886 }
10887 
10888 void Assembler::fld_d(Address adr) {
10889   InstructionMark im(this);
10890   emit_int8((unsigned char)0xDD);
10891   emit_operand32(rax, adr, 0);
10892 }
10893 
10894 void Assembler::fld_s(Address adr) {
10895   InstructionMark im(this);
10896   emit_int8((unsigned char)0xD9);
10897   emit_operand32(rax, adr, 0);
10898 }
10899 
10900 
10901 void Assembler::fld_s(int index) {
10902   emit_farith(0xD9, 0xC0, index);
10903 }
10904 
10905 void Assembler::fldcw(Address src) {
10906   InstructionMark im(this);
10907   emit_int8((unsigned char)0xD9);
10908   emit_operand32(rbp, src, 0);
10909 }
10910 
10911 void Assembler::fldenv(Address src) {
10912   InstructionMark im(this);
10913   emit_int8((unsigned char)0xD9);
10914   emit_operand32(rsp, src, 0);
10915 }
10916 
10917 void Assembler::fldlg2() {
10918   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10919 }
10920 
10921 void Assembler::fldln2() {
10922   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10923 }
10924 
10925 void Assembler::fldz() {
10926   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10927 }
10928 
10929 void Assembler::flog() {
10930   fldln2();
10931   fxch();
10932   fyl2x();
10933 }
10934 
10935 void Assembler::flog10() {
10936   fldlg2();
10937   fxch();
10938   fyl2x();
10939 }
10940 
10941 void Assembler::fmul(int i) {
10942   emit_farith(0xD8, 0xC8, i);
10943 }
10944 
10945 void Assembler::fmul_d(Address src) {
10946   InstructionMark im(this);
10947   emit_int8((unsigned char)0xDC);
10948   emit_operand32(rcx, src, 0);
10949 }
10950 
10951 void Assembler::fmul_s(Address src) {
10952   InstructionMark im(this);
10953   emit_int8((unsigned char)0xD8);
10954   emit_operand32(rcx, src, 0);
10955 }
10956 
10957 void Assembler::fmula(int i) {
10958   emit_farith(0xDC, 0xC8, i);
10959 }
10960 
10961 void Assembler::fmulp(int i) {
10962   emit_farith(0xDE, 0xC8, i);
10963 }
10964 
10965 void Assembler::fnsave(Address dst) {
10966   InstructionMark im(this);
10967   emit_int8((unsigned char)0xDD);
10968   emit_operand32(rsi, dst, 0);
10969 }
10970 
10971 void Assembler::fnstcw(Address src) {
10972   InstructionMark im(this);
10973   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10974   emit_operand32(rdi, src, 0);
10975 }
10976 
10977 void Assembler::fnstsw_ax() {
10978   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10979 }
10980 
10981 void Assembler::fprem() {
10982   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10983 }
10984 
10985 void Assembler::fprem1() {
10986   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10987 }
10988 
10989 void Assembler::frstor(Address src) {
10990   InstructionMark im(this);
10991   emit_int8((unsigned char)0xDD);
10992   emit_operand32(rsp, src, 0);
10993 }
10994 
10995 void Assembler::fsin() {
10996   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10997 }
10998 
10999 void Assembler::fsqrt() {
11000   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11001 }
11002 
11003 void Assembler::fst_d(Address adr) {
11004   InstructionMark im(this);
11005   emit_int8((unsigned char)0xDD);
11006   emit_operand32(rdx, adr, 0);
11007 }
11008 
11009 void Assembler::fst_s(Address adr) {
11010   InstructionMark im(this);
11011   emit_int8((unsigned char)0xD9);
11012   emit_operand32(rdx, adr, 0);
11013 }
11014 
11015 void Assembler::fstp_d(Address adr) {
11016   InstructionMark im(this);
11017   emit_int8((unsigned char)0xDD);
11018   emit_operand32(rbx, adr, 0);
11019 }
11020 
11021 void Assembler::fstp_d(int index) {
11022   emit_farith(0xDD, 0xD8, index);
11023 }
11024 
11025 void Assembler::fstp_s(Address adr) {
11026   InstructionMark im(this);
11027   emit_int8((unsigned char)0xD9);
11028   emit_operand32(rbx, adr, 0);
11029 }
11030 
11031 void Assembler::fsub(int i) {
11032   emit_farith(0xD8, 0xE0, i);
11033 }
11034 
11035 void Assembler::fsub_d(Address src) {
11036   InstructionMark im(this);
11037   emit_int8((unsigned char)0xDC);
11038   emit_operand32(rsp, src, 0);
11039 }
11040 
11041 void Assembler::fsub_s(Address src) {
11042   InstructionMark im(this);
11043   emit_int8((unsigned char)0xD8);
11044   emit_operand32(rsp, src, 0);
11045 }
11046 
11047 void Assembler::fsuba(int i) {
11048   emit_farith(0xDC, 0xE8, i);
11049 }
11050 
11051 void Assembler::fsubp(int i) {
11052   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11053 }
11054 
11055 void Assembler::fsubr(int i) {
11056   emit_farith(0xD8, 0xE8, i);
11057 }
11058 
11059 void Assembler::fsubr_d(Address src) {
11060   InstructionMark im(this);
11061   emit_int8((unsigned char)0xDC);
11062   emit_operand32(rbp, src, 0);
11063 }
11064 
11065 void Assembler::fsubr_s(Address src) {
11066   InstructionMark im(this);
11067   emit_int8((unsigned char)0xD8);
11068   emit_operand32(rbp, src, 0);
11069 }
11070 
11071 void Assembler::fsubra(int i) {
11072   emit_farith(0xDC, 0xE0, i);
11073 }
11074 
11075 void Assembler::fsubrp(int i) {
11076   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11077 }
11078 
11079 void Assembler::ftan() {
11080   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11081 }
11082 
11083 void Assembler::ftst() {
11084   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11085 }
11086 
11087 void Assembler::fucomi(int i) {
11088   // make sure the instruction is supported (introduced for P6, together with cmov)
11089   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11090   emit_farith(0xDB, 0xE8, i);
11091 }
11092 
11093 void Assembler::fucomip(int i) {
11094   // make sure the instruction is supported (introduced for P6, together with cmov)
11095   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11096   emit_farith(0xDF, 0xE8, i);
11097 }
11098 
11099 void Assembler::fwait() {
11100   emit_int8((unsigned char)0x9B);
11101 }
11102 
11103 void Assembler::fxch(int i) {
11104   emit_farith(0xD9, 0xC8, i);
11105 }
11106 
11107 void Assembler::fyl2x() {
11108   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11109 }
11110 
11111 void Assembler::frndint() {
11112   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11113 }
11114 
11115 void Assembler::f2xm1() {
11116   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11117 }
11118 
11119 void Assembler::fldl2e() {
11120   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11121 }
11122 #endif // !_LP64
11123 
11124 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11125 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11126 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11127 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
11128 
11129 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11130 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11131   if (pre > 0) {
11132     emit_int8(simd_pre[pre]);
11133   }
11134   if (rex_w) {
11135     prefixq(adr, xreg);
11136   } else {
11137     prefix(adr, xreg);
11138   }
11139   if (opc > 0) {
11140     emit_int8(0x0F);
11141     int opc2 = simd_opc[opc];
11142     if (opc2 > 0) {
11143       emit_int8(opc2);
11144     }
11145   }
11146 }
11147 
11148 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11149   if (pre > 0) {
11150     emit_int8(simd_pre[pre]);
11151   }
11152   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11153   if (opc > 0) {
11154     emit_int8(0x0F);
11155     int opc2 = simd_opc[opc];
11156     if (opc2 > 0) {
11157       emit_int8(opc2);
11158     }
11159   }
11160   return encode;
11161 }
11162 
11163 
11164 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11165   int vector_len = _attributes->get_vector_len();
11166   bool vex_w = _attributes->is_rex_vex_w();
11167   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11168     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11169     byte1 = (~byte1) & 0xE0;
11170     byte1 |= opc;
11171 
11172     int byte2 = ((~nds_enc) & 0xf) << 3;
11173     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11174 
11175     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11176   } else {
11177     int byte1 = vex_r ? VEX_R : 0;
11178     byte1 = (~byte1) & 0x80;
11179     byte1 |= ((~nds_enc) & 0xf) << 3;
11180     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11181     emit_int16((unsigned char)VEX_2bytes, byte1);
11182   }
11183 }
11184 
11185 // This is a 4 byte encoding
11186 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11187   // EVEX 0x62 prefix
11188   // byte1 = EVEX_4bytes;
11189 
11190   bool vex_w = _attributes->is_rex_vex_w();
11191   int evex_encoding = (vex_w ? VEX_W : 0);
11192   // EVEX.b is not currently used for broadcast of single element or data rounding modes
11193   _attributes->set_evex_encoding(evex_encoding);
11194 
11195   // P0: byte 2, initialized to RXBR`00mm
11196   // instead of not'd
11197   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11198   byte2 = (~byte2) & 0xF0;
11199   // confine opc opcode extensions in mm bits to lower two bits
11200   // of form {0F, 0F_38, 0F_3A}
11201   byte2 |= opc;
11202 
11203   // P1: byte 3 as Wvvvv1pp
11204   int byte3 = ((~nds_enc) & 0xf) << 3;
11205   // p[10] is always 1
11206   byte3 |= EVEX_F;
11207   byte3 |= (vex_w & 1) << 7;
11208   // confine pre opcode extensions in pp bits to lower two bits
11209   // of form {66, F3, F2}
11210   byte3 |= pre;
11211 
11212   // P2: byte 4 as zL'Lbv'aaa
11213   // kregs are implemented in the low 3 bits as aaa
11214   int byte4 = (_attributes->is_no_reg_mask()) ?
11215               0 :
11216               _attributes->get_embedded_opmask_register_specifier();
11217   // EVEX.v` for extending EVEX.vvvv or VIDX
11218   byte4 |= (evex_v ? 0: EVEX_V);
11219   // third EXEC.b for broadcast actions
11220   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11221   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11222   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11223   // last is EVEX.z for zero/merge actions
11224   if (_attributes->is_no_reg_mask() == false &&
11225       _attributes->get_embedded_opmask_register_specifier() != 0) {
11226     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11227   }
11228 
11229   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11230 }
11231 
11232 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11233   bool vex_r = (xreg_enc & 8) == 8;
11234   bool vex_b = adr.base_needs_rex();
11235   bool vex_x;
11236   if (adr.isxmmindex()) {
11237     vex_x = adr.xmmindex_needs_rex();
11238   } else {
11239     vex_x = adr.index_needs_rex();
11240   }
11241   set_attributes(attributes);
11242   attributes->set_current_assembler(this);
11243 
11244   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11245   // is allowed in legacy mode and has resources which will fit in it.
11246   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11247   if (!attributes->is_legacy_mode()) {
11248     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11249       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11250           attributes->set_is_legacy_mode();
11251       }
11252     }
11253   }
11254 
11255   if (UseAVX > 2) {
11256     assert(((!attributes->uses_vl()) ||
11257             (attributes->get_vector_len() == AVX_512bit) ||
11258             (!_legacy_mode_vl) ||
11259             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11260     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11261   }
11262 
11263   clear_managed();
11264   if (UseAVX > 2 && !attributes->is_legacy_mode())
11265   {
11266     bool evex_r = (xreg_enc >= 16);
11267     bool evex_v;
11268     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11269     if (adr.isxmmindex())  {
11270       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11271     } else {
11272       evex_v = (nds_enc >= 16);
11273     }
11274     attributes->set_is_evex_instruction();
11275     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11276   } else {
11277     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11278       attributes->set_rex_vex_w(false);
11279     }
11280     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11281   }
11282 }
11283 
11284 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11285   bool vex_r = (dst_enc & 8) == 8;
11286   bool vex_b = (src_enc & 8) == 8;
11287   bool vex_x = false;
11288   set_attributes(attributes);
11289   attributes->set_current_assembler(this);
11290 
11291   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11292   // is allowed in legacy mode and has resources which will fit in it.
11293   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11294   if (!attributes->is_legacy_mode()) {
11295     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11296       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11297           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11298           attributes->set_is_legacy_mode();
11299       }
11300     }
11301   }
11302 
11303   if (UseAVX > 2) {
11304     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11305     // Instruction with uses_vl true are vector instructions
11306     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11307     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11308     // Rest all should have legacy_mode set as true
11309     assert(((!attributes->uses_vl()) ||
11310             (attributes->get_vector_len() == AVX_512bit) ||
11311             (!_legacy_mode_vl) ||
11312             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11313     // Instruction with legacy_mode true should have dst, nds and src < 15
11314     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11315   }
11316 
11317   clear_managed();
11318   if (UseAVX > 2 && !attributes->is_legacy_mode())
11319   {
11320     bool evex_r = (dst_enc >= 16);
11321     bool evex_v = (nds_enc >= 16);
11322     // can use vex_x as bank extender on rm encoding
11323     vex_x = (src_enc >= 16);
11324     attributes->set_is_evex_instruction();
11325     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11326   } else {
11327     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11328       attributes->set_rex_vex_w(false);
11329     }
11330     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11331   }
11332 
11333   // return modrm byte components for operands
11334   return (((dst_enc & 7) << 3) | (src_enc & 7));
11335 }
11336 
11337 
11338 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11339                             VexOpcode opc, InstructionAttr *attributes) {
11340   if (UseAVX > 0) {
11341     int xreg_enc = xreg->encoding();
11342     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11343     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11344   } else {
11345     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11346     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11347   }
11348 }
11349 
11350 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11351                                       VexOpcode opc, InstructionAttr *attributes) {
11352   int dst_enc = dst->encoding();
11353   int src_enc = src->encoding();
11354   if (UseAVX > 0) {
11355     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11356     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11357   } else {
11358     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11359     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11360   }
11361 }
11362 
11363 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11364   assert(VM_Version::supports_avx(), "");
11365   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11366   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11367   emit_int16(0x5F, (0xC0 | encode));
11368 }
11369 
11370 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11371   assert(VM_Version::supports_avx(), "");
11372   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11373   attributes.set_rex_vex_w_reverted();
11374   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11375   emit_int16(0x5F, (0xC0 | encode));
11376 }
11377 
11378 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11379   assert(VM_Version::supports_avx(), "");
11380   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11381   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11382   emit_int16(0x5D, (0xC0 | encode));
11383 }
11384 
11385 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11386   assert(VM_Version::supports_avx(), "");
11387   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11388   attributes.set_rex_vex_w_reverted();
11389   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11390   emit_int16(0x5D, (0xC0 | encode));
11391 }
11392 
11393 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11394   assert(VM_Version::supports_avx(), "");
11395   assert(vector_len <= AVX_256bit, "");
11396   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11397   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11398   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11399 }
11400 
11401 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11402   assert(VM_Version::supports_avx(), "");
11403   assert(vector_len <= AVX_256bit, "");
11404   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11405   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11406   int src2_enc = src2->encoding();
11407   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11408 }
11409 
11410 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11411   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11412   assert(vector_len <= AVX_256bit, "");
11413   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11414   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11415   int src2_enc = src2->encoding();
11416   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11417 }
11418 
11419 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11420   assert(VM_Version::supports_avx2(), "");
11421   assert(vector_len <= AVX_256bit, "");
11422   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11423   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11424   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11425 }
11426 
11427 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11428   assert(VM_Version::supports_avx(), "");
11429   assert(vector_len <= AVX_256bit, "");
11430   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11431   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11432   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11433 }
11434 
11435 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11436                         ComparisonPredicateFP comparison, int vector_len) {
11437   assert(VM_Version::supports_evex(), "");
11438   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11439   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11440   attributes.set_is_evex_instruction();
11441   attributes.set_embedded_opmask_register_specifier(mask);
11442   attributes.reset_is_clear_context();
11443   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11444   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11445 }
11446 
11447 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11448                         ComparisonPredicateFP comparison, int vector_len) {
11449   assert(VM_Version::supports_evex(), "");
11450   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11451   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11452   attributes.set_is_evex_instruction();
11453   attributes.set_embedded_opmask_register_specifier(mask);
11454   attributes.reset_is_clear_context();
11455   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11456   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11457 }
11458 
11459 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11460   assert(VM_Version::supports_sse4_1(), "");
11461   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11462   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11463   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11464   emit_int16(0x14, (0xC0 | encode));
11465 }
11466 
11467 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11468   assert(VM_Version::supports_sse4_1(), "");
11469   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11471   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11472   emit_int16(0x15, (0xC0 | encode));
11473 }
11474 
11475 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11476   assert(VM_Version::supports_sse4_1(), "");
11477   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11478   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11479   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11480   emit_int16(0x10, (0xC0 | encode));
11481 }
11482 
11483 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11484   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11486   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11487   int src2_enc = src2->encoding();
11488   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11489 }
11490 
11491 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11493   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11494   emit_int24(0x0C, (0xC0 | encode), imm8);
11495 }
11496 
11497 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11498   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11499   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11500   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11501   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11502   emit_int16(0x64, (0xC0 | encode));
11503 }
11504 
11505 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11506   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11507   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11508   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11509   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11510   emit_int16(0x65, (0xC0 | encode));
11511 }
11512 
11513 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11514   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11515   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11516   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11517   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11518   emit_int16(0x66, (0xC0 | encode));
11519 }
11520 
11521 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11522   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11523   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11525   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11526   emit_int16(0x37, (0xC0 | encode));
11527 }
11528 
11529 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11530                         int comparison, bool is_signed, int vector_len) {
11531   assert(VM_Version::supports_evex(), "");
11532   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11533   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11534   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11535   attributes.set_is_evex_instruction();
11536   attributes.set_embedded_opmask_register_specifier(mask);
11537   attributes.reset_is_clear_context();
11538   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11539   int opcode = is_signed ? 0x1F : 0x1E;
11540   emit_int24(opcode, (0xC0 | encode), comparison);
11541 }
11542 
11543 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11544                         int comparison, bool is_signed, int vector_len) {
11545   assert(VM_Version::supports_evex(), "");
11546   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11547   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11548   InstructionMark im(this);
11549   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11550   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11551   attributes.set_is_evex_instruction();
11552   attributes.set_embedded_opmask_register_specifier(mask);
11553   attributes.reset_is_clear_context();
11554   int dst_enc = kdst->encoding();
11555   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11556   int opcode = is_signed ? 0x1F : 0x1E;
11557   emit_int8((unsigned char)opcode);
11558   emit_operand(as_Register(dst_enc), src, 1);
11559   emit_int8((unsigned char)comparison);
11560 }
11561 
11562 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11563                         int comparison, bool is_signed, int vector_len) {
11564   assert(VM_Version::supports_evex(), "");
11565   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11566   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11567   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11568   attributes.set_is_evex_instruction();
11569   attributes.set_embedded_opmask_register_specifier(mask);
11570   attributes.reset_is_clear_context();
11571   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11572   int opcode = is_signed ? 0x1F : 0x1E;
11573   emit_int24(opcode, (0xC0 | encode), comparison);
11574 }
11575 
11576 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11577                         int comparison, bool is_signed, int vector_len) {
11578   assert(VM_Version::supports_evex(), "");
11579   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11580   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11581   InstructionMark im(this);
11582   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11583   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11584   attributes.set_is_evex_instruction();
11585   attributes.set_embedded_opmask_register_specifier(mask);
11586   attributes.reset_is_clear_context();
11587   int dst_enc = kdst->encoding();
11588   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11589   int opcode = is_signed ? 0x1F : 0x1E;
11590   emit_int8((unsigned char)opcode);
11591   emit_operand(as_Register(dst_enc), src, 1);
11592   emit_int8((unsigned char)comparison);
11593 }
11594 
11595 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11596                         int comparison, bool is_signed, int vector_len) {
11597   assert(VM_Version::supports_evex(), "");
11598   assert(VM_Version::supports_avx512bw(), "");
11599   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11600   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11601   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11602   attributes.set_is_evex_instruction();
11603   attributes.set_embedded_opmask_register_specifier(mask);
11604   attributes.reset_is_clear_context();
11605   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11606   int opcode = is_signed ? 0x3F : 0x3E;
11607   emit_int24(opcode, (0xC0 | encode), comparison);
11608 }
11609 
11610 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11611                         int comparison, bool is_signed, int vector_len) {
11612   assert(VM_Version::supports_evex(), "");
11613   assert(VM_Version::supports_avx512bw(), "");
11614   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11615   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11616   InstructionMark im(this);
11617   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11618   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11619   attributes.set_is_evex_instruction();
11620   attributes.set_embedded_opmask_register_specifier(mask);
11621   attributes.reset_is_clear_context();
11622   int dst_enc = kdst->encoding();
11623   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11624   int opcode = is_signed ? 0x3F : 0x3E;
11625   emit_int8((unsigned char)opcode);
11626   emit_operand(as_Register(dst_enc), src, 1);
11627   emit_int8((unsigned char)comparison);
11628 }
11629 
11630 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11631                         int comparison, bool is_signed, int vector_len) {
11632   assert(VM_Version::supports_evex(), "");
11633   assert(VM_Version::supports_avx512bw(), "");
11634   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11635   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11636   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11637   attributes.set_is_evex_instruction();
11638   attributes.set_embedded_opmask_register_specifier(mask);
11639   attributes.reset_is_clear_context();
11640   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11641   int opcode = is_signed ? 0x3F : 0x3E;
11642   emit_int24(opcode, (0xC0 | encode), comparison);
11643 }
11644 
11645 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11646                         int comparison, bool is_signed, int vector_len) {
11647   assert(VM_Version::supports_evex(), "");
11648   assert(VM_Version::supports_avx512bw(), "");
11649   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11650   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11651   InstructionMark im(this);
11652   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11653   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11654   attributes.set_is_evex_instruction();
11655   attributes.set_embedded_opmask_register_specifier(mask);
11656   attributes.reset_is_clear_context();
11657   int dst_enc = kdst->encoding();
11658   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11659   int opcode = is_signed ? 0x3F : 0x3E;
11660   emit_int8((unsigned char)opcode);
11661   emit_operand(as_Register(dst_enc), src, 1);
11662   emit_int8((unsigned char)comparison);
11663 }
11664 
11665 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11666   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11667   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11668   attributes.set_is_evex_instruction();
11669   attributes.set_embedded_opmask_register_specifier(mask);
11670   if (merge) {
11671     attributes.reset_is_clear_context();
11672   }
11673   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11674   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11675 }
11676 
11677 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11678   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11679   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11680   attributes.set_is_evex_instruction();
11681   attributes.set_embedded_opmask_register_specifier(mask);
11682   if (merge) {
11683     attributes.reset_is_clear_context();
11684   }
11685   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11686   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11687 }
11688 
11689 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11690   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11691   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11692   attributes.set_is_evex_instruction();
11693   attributes.set_embedded_opmask_register_specifier(mask);
11694   if (merge) {
11695     attributes.reset_is_clear_context();
11696   }
11697   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11698   emit_int16(0x14, (0xC0 | encode));
11699 }
11700 
11701 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11702   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11703   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11704   attributes.set_is_evex_instruction();
11705   attributes.set_embedded_opmask_register_specifier(mask);
11706   if (merge) {
11707     attributes.reset_is_clear_context();
11708   }
11709   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11710   emit_int16(0x14, (0xC0 | encode));
11711 }
11712 
11713 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11714   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11715   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11716   attributes.set_is_evex_instruction();
11717   attributes.set_embedded_opmask_register_specifier(mask);
11718   if (merge) {
11719     attributes.reset_is_clear_context();
11720   }
11721   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11722   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11723 }
11724 
11725 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11726   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11727   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11728   attributes.set_is_evex_instruction();
11729   attributes.set_embedded_opmask_register_specifier(mask);
11730   if (merge) {
11731     attributes.reset_is_clear_context();
11732   }
11733   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11734   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11735 }
11736 
11737 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11738   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11739   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11740   attributes.set_is_evex_instruction();
11741   attributes.set_embedded_opmask_register_specifier(mask);
11742   if (merge) {
11743     attributes.reset_is_clear_context();
11744   }
11745   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11746   emit_int16(0x15, (0xC0 | encode));
11747 }
11748 
11749 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11750   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11751   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11752   attributes.set_is_evex_instruction();
11753   attributes.set_embedded_opmask_register_specifier(mask);
11754   if (merge) {
11755     attributes.reset_is_clear_context();
11756   }
11757   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11758   emit_int16(0x15, (0xC0 | encode));
11759 }
11760 
11761 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11762   assert(VM_Version::supports_avx(), "");
11763   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11764   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11765   int mask_enc = mask->encoding();
11766   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11767 }
11768 
11769 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11770   assert(VM_Version::supports_evex(), "");
11771   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11772   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11773   attributes.set_is_evex_instruction();
11774   attributes.set_embedded_opmask_register_specifier(mask);
11775   if (merge) {
11776     attributes.reset_is_clear_context();
11777   }
11778   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11779   emit_int16(0x65, (0xC0 | encode));
11780 }
11781 
11782 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11783   assert(VM_Version::supports_evex(), "");
11784   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11785   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11786   attributes.set_is_evex_instruction();
11787   attributes.set_embedded_opmask_register_specifier(mask);
11788   if (merge) {
11789     attributes.reset_is_clear_context();
11790   }
11791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11792   emit_int16(0x65, (0xC0 | encode));
11793 }
11794 
11795 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11796   assert(VM_Version::supports_evex(), "");
11797   assert(VM_Version::supports_avx512bw(), "");
11798   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11799   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11800   attributes.set_is_evex_instruction();
11801   attributes.set_embedded_opmask_register_specifier(mask);
11802   if (merge) {
11803     attributes.reset_is_clear_context();
11804   }
11805   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11806   emit_int16(0x66, (0xC0 | encode));
11807 }
11808 
11809 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11810   assert(VM_Version::supports_evex(), "");
11811   assert(VM_Version::supports_avx512bw(), "");
11812   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11813   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11814   attributes.set_is_evex_instruction();
11815   attributes.set_embedded_opmask_register_specifier(mask);
11816   if (merge) {
11817     attributes.reset_is_clear_context();
11818   }
11819   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11820   emit_int16(0x66, (0xC0 | encode));
11821 }
11822 
11823 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11824   assert(VM_Version::supports_evex(), "");
11825   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11827   attributes.set_is_evex_instruction();
11828   attributes.set_embedded_opmask_register_specifier(mask);
11829   if (merge) {
11830     attributes.reset_is_clear_context();
11831   }
11832   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11833   emit_int16(0x64, (0xC0 | encode));
11834 }
11835 
11836 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11837   assert(VM_Version::supports_evex(), "");
11838   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11839   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11840   attributes.set_is_evex_instruction();
11841   attributes.set_embedded_opmask_register_specifier(mask);
11842   if (merge) {
11843     attributes.reset_is_clear_context();
11844   }
11845   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11846   emit_int16(0x64, (0xC0 | encode));
11847 }
11848 
11849 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11850   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11851   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11852   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11853   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11854 }
11855 
11856 void Assembler::pextl(Register dst, Register src1, Register src2) {
11857   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11858   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11859   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11860   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11861 }
11862 
11863 void Assembler::pdepl(Register dst, Register src1, Register src2) {
11864   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11865   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11866   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11867   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11868 }
11869 
11870 void Assembler::pextq(Register dst, Register src1, Register src2) {
11871   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11872   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11873   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11874   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11875 }
11876 
11877 void Assembler::pdepq(Register dst, Register src1, Register src2) {
11878   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11879   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11880   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11881   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11882 }
11883 
11884 void Assembler::pextl(Register dst, Register src1, Address src2) {
11885   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11886   InstructionMark im(this);
11887   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11888   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11889   emit_int8((unsigned char)0xF5);
11890   emit_operand(dst, src2, 0);
11891 }
11892 
11893 void Assembler::pdepl(Register dst, Register src1, Address src2) {
11894   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11895   InstructionMark im(this);
11896   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11897   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11898   emit_int8((unsigned char)0xF5);
11899   emit_operand(dst, src2, 0);
11900 }
11901 
11902 void Assembler::pextq(Register dst, Register src1, Address src2) {
11903   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11904   InstructionMark im(this);
11905   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11906   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11907   emit_int8((unsigned char)0xF5);
11908   emit_operand(dst, src2, 0);
11909 }
11910 
11911 void Assembler::pdepq(Register dst, Register src1, Address src2) {
11912   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11913   InstructionMark im(this);
11914   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11915   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11916   emit_int8((unsigned char)0xF5);
11917   emit_operand(dst, src2, 0);
11918 }
11919 
11920 void Assembler::sarxl(Register dst, Register src1, Register src2) {
11921   assert(VM_Version::supports_bmi2(), "");
11922   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11923   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11924   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11925 }
11926 
11927 void Assembler::sarxl(Register dst, Address src1, Register src2) {
11928   assert(VM_Version::supports_bmi2(), "");
11929   InstructionMark im(this);
11930   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11931   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11932   emit_int8((unsigned char)0xF7);
11933   emit_operand(dst, src1, 0);
11934 }
11935 
11936 void Assembler::sarxq(Register dst, Register src1, Register src2) {
11937   assert(VM_Version::supports_bmi2(), "");
11938   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11939   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11940   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11941 }
11942 
11943 void Assembler::sarxq(Register dst, Address src1, Register src2) {
11944   assert(VM_Version::supports_bmi2(), "");
11945   InstructionMark im(this);
11946   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11947   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11948   emit_int8((unsigned char)0xF7);
11949   emit_operand(dst, src1, 0);
11950 }
11951 
11952 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11953   assert(VM_Version::supports_bmi2(), "");
11954   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11955   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11956   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11957 }
11958 
11959 void Assembler::shlxl(Register dst, Address src1, Register src2) {
11960   assert(VM_Version::supports_bmi2(), "");
11961   InstructionMark im(this);
11962   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11963   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11964   emit_int8((unsigned char)0xF7);
11965   emit_operand(dst, src1, 0);
11966 }
11967 
11968 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11969   assert(VM_Version::supports_bmi2(), "");
11970   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11971   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11972   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11973 }
11974 
11975 void Assembler::shlxq(Register dst, Address src1, Register src2) {
11976   assert(VM_Version::supports_bmi2(), "");
11977   InstructionMark im(this);
11978   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11979   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11980   emit_int8((unsigned char)0xF7);
11981   emit_operand(dst, src1, 0);
11982 }
11983 
11984 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11985   assert(VM_Version::supports_bmi2(), "");
11986   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11987   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11988   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11989 }
11990 
11991 void Assembler::shrxl(Register dst, Address src1, Register src2) {
11992   assert(VM_Version::supports_bmi2(), "");
11993   InstructionMark im(this);
11994   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11995   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11996   emit_int8((unsigned char)0xF7);
11997   emit_operand(dst, src1, 0);
11998 }
11999 
12000 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12001   assert(VM_Version::supports_bmi2(), "");
12002   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12003   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12004   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12005 }
12006 
12007 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12008   assert(VM_Version::supports_bmi2(), "");
12009   InstructionMark im(this);
12010   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12011   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12012   emit_int8((unsigned char)0xF7);
12013   emit_operand(dst, src1, 0);
12014 }
12015 
12016 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12017   assert(VM_Version::supports_avx512vldq(), "");
12018   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12019   attributes.set_is_evex_instruction();
12020   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12021   emit_int16(0x39, (0xC0 | encode));
12022 }
12023 
12024 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12025   assert(VM_Version::supports_avx512vldq(), "");
12026   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12027   attributes.set_is_evex_instruction();
12028   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12029   emit_int16(0x39, (0xC0 | encode));
12030 }
12031 
12032 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12033   assert(VM_Version::supports_avx512vlbw(), "");
12034   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12035   attributes.set_is_evex_instruction();
12036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12037   emit_int16(0x29, (0xC0 | encode));
12038 }
12039 
12040 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12041   assert(VM_Version::supports_avx512vlbw(), "");
12042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12043   attributes.set_is_evex_instruction();
12044   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12045   emit_int16(0x29, (0xC0 | encode));
12046 }
12047 
12048 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12049   assert(VM_Version::supports_avx512vldq(), "");
12050   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12051   attributes.set_is_evex_instruction();
12052   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12053   emit_int16(0x38, (0xC0 | encode));
12054 }
12055 
12056 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12057   assert(VM_Version::supports_avx512vldq(), "");
12058   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12059   attributes.set_is_evex_instruction();
12060   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12061   emit_int16(0x38, (0xC0 | encode));
12062 }
12063 
12064 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12065   assert(VM_Version::supports_avx512vlbw(), "");
12066   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12067   attributes.set_is_evex_instruction();
12068   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12069   emit_int16(0x28, (0xC0 | encode));
12070 }
12071 
12072 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12073   assert(VM_Version::supports_avx512vlbw(), "");
12074   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12075   attributes.set_is_evex_instruction();
12076   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12077   emit_int16(0x28, (0xC0 | encode));
12078 }
12079 
12080 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12081   assert(VM_Version::supports_avx512_vbmi2(), "");
12082   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12084   attributes.set_embedded_opmask_register_specifier(mask);
12085   attributes.set_is_evex_instruction();
12086   if (merge) {
12087     attributes.reset_is_clear_context();
12088   }
12089   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12090   emit_int16((unsigned char)0x63, (0xC0 | encode));
12091 }
12092 
12093 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12094   assert(VM_Version::supports_avx512_vbmi2(), "");
12095   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12096   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12097   attributes.set_embedded_opmask_register_specifier(mask);
12098   attributes.set_is_evex_instruction();
12099   if (merge) {
12100     attributes.reset_is_clear_context();
12101   }
12102   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12103   emit_int16((unsigned char)0x63, (0xC0 | encode));
12104 }
12105 
12106 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12107   assert(VM_Version::supports_evex(), "");
12108   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12109   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12110   attributes.set_embedded_opmask_register_specifier(mask);
12111   attributes.set_is_evex_instruction();
12112   if (merge) {
12113     attributes.reset_is_clear_context();
12114   }
12115   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12116   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12117 }
12118 
12119 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12120   assert(VM_Version::supports_evex(), "");
12121   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12122   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12123   attributes.set_embedded_opmask_register_specifier(mask);
12124   attributes.set_is_evex_instruction();
12125   if (merge) {
12126     attributes.reset_is_clear_context();
12127   }
12128   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12129   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12130 }
12131 
12132 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12133   assert(VM_Version::supports_evex(), "");
12134   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12135   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12136   attributes.set_embedded_opmask_register_specifier(mask);
12137   attributes.set_is_evex_instruction();
12138   if (merge) {
12139     attributes.reset_is_clear_context();
12140   }
12141   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12142   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12143 }
12144 
12145 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12146   assert(VM_Version::supports_evex(), "");
12147   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12148   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12149   attributes.set_embedded_opmask_register_specifier(mask);
12150   attributes.set_is_evex_instruction();
12151   if (merge) {
12152     attributes.reset_is_clear_context();
12153   }
12154   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12155   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12156 }
12157 
12158 #ifndef _LP64
12159 
12160 void Assembler::incl(Register dst) {
12161   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12162   emit_int8(0x40 | dst->encoding());
12163 }
12164 
12165 void Assembler::lea(Register dst, Address src) {
12166   leal(dst, src);
12167 }
12168 
12169 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12170   InstructionMark im(this);
12171   emit_int8((unsigned char)0xC7);
12172   emit_operand(rax, dst, 4);
12173   emit_data((int)imm32, rspec, 0);
12174 }
12175 
12176 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12177   InstructionMark im(this);
12178   int encode = prefix_and_encode(dst->encoding());
12179   emit_int8((0xB8 | encode));
12180   emit_data((int)imm32, rspec, 0);
12181 }
12182 
12183 void Assembler::popa() { // 32bit
12184   emit_int8(0x61);
12185 }
12186 
12187 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12188   InstructionMark im(this);
12189   emit_int8(0x68);
12190   emit_data(imm32, rspec, 0);
12191 }
12192 
12193 void Assembler::pusha() { // 32bit
12194   emit_int8(0x60);
12195 }
12196 
12197 void Assembler::set_byte_if_not_zero(Register dst) {
12198   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12199 }
12200 
12201 #else // LP64
12202 
12203 // 64bit only pieces of the assembler
12204 
12205 void Assembler::set_byte_if_not_zero(Register dst) {
12206   int enc = prefix_and_encode(dst->encoding(), true);
12207   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12208 }
12209 
12210 // This should only be used by 64bit instructions that can use rip-relative
12211 // it cannot be used by instructions that want an immediate value.
12212 
12213 // Determine whether an address is always reachable in rip-relative addressing mode
12214 // when accessed from the code cache.
12215 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12216   switch (reloc_type) {
12217     // This should be rip-relative and easily reachable.
12218     case relocInfo::internal_word_type: {
12219       return true;
12220     }
12221     // This should be rip-relative within the code cache and easily
12222     // reachable until we get huge code caches. (At which point
12223     // IC code is going to have issues).
12224     case relocInfo::virtual_call_type:
12225     case relocInfo::opt_virtual_call_type:
12226     case relocInfo::static_call_type:
12227     case relocInfo::static_stub_type: {
12228       return true;
12229     }
12230     case relocInfo::runtime_call_type:
12231     case relocInfo::external_word_type:
12232     case relocInfo::poll_return_type: // these are really external_word but need special
12233     case relocInfo::poll_type: {      // relocs to identify them
12234       return CodeCache::contains(target);
12235     }
12236     default: {
12237       return false;
12238     }
12239   }
12240 }
12241 
12242 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12243 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12244   if (is_always_reachable(target, reloc_type)) {
12245     return true;
12246   }
12247   switch (reloc_type) {
12248     // None will force a 64bit literal to the code stream. Likely a placeholder
12249     // for something that will be patched later and we need to certain it will
12250     // always be reachable.
12251     case relocInfo::none: {
12252       return false;
12253     }
12254     case relocInfo::runtime_call_type:
12255     case relocInfo::external_word_type:
12256     case relocInfo::poll_return_type: // these are really external_word but need special
12257     case relocInfo::poll_type: {      // relocs to identify them
12258       assert(!CodeCache::contains(target), "always reachable");
12259       if (ForceUnreachable) {
12260         return false; // stress the correction code
12261       }
12262       // For external_word_type/runtime_call_type if it is reachable from where we
12263       // are now (possibly a temp buffer) and where we might end up
12264       // anywhere in the code cache then we are always reachable.
12265       // This would have to change if we ever save/restore shared code to be more pessimistic.
12266       // Code buffer has to be allocated in the code cache, so check against
12267       // code cache boundaries cover that case.
12268       //
12269       // In rip-relative addressing mode, an effective address is formed by adding displacement
12270       // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12271       // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12272       // to account for that.
12273       return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12274              Assembler::is_simm32(target - CodeCache::high_bound());
12275     }
12276     default: {
12277       return false;
12278     }
12279   }
12280 }
12281 
12282 bool Assembler::reachable(AddressLiteral adr) {
12283   assert(CodeCache::contains(pc()), "required");
12284   if (adr.is_lval()) {
12285     return false;
12286   }
12287   return is_reachable(adr.target(), adr.reloc());
12288 }
12289 
12290 bool Assembler::always_reachable(AddressLiteral adr) {
12291   assert(CodeCache::contains(pc()), "required");
12292   if (adr.is_lval()) {
12293     return false;
12294   }
12295   return is_always_reachable(adr.target(), adr.reloc());
12296 }
12297 
12298 void Assembler::emit_data64(jlong data,
12299                             relocInfo::relocType rtype,
12300                             int format) {
12301   if (rtype == relocInfo::none) {
12302     emit_int64(data);
12303   } else {
12304     emit_data64(data, Relocation::spec_simple(rtype), format);
12305   }
12306 }
12307 
12308 void Assembler::emit_data64(jlong data,
12309                             RelocationHolder const& rspec,
12310                             int format) {
12311   assert(imm_operand == 0, "default format must be immediate in this file");
12312   assert(imm_operand == format, "must be immediate");
12313   assert(inst_mark() != NULL, "must be inside InstructionMark");
12314   // Do not use AbstractAssembler::relocate, which is not intended for
12315   // embedded words.  Instead, relocate to the enclosing instruction.
12316   code_section()->relocate(inst_mark(), rspec, format);
12317 #ifdef ASSERT
12318   check_relocation(rspec, format);
12319 #endif
12320   emit_int64(data);
12321 }
12322 
12323 void Assembler::prefix(Register reg) {
12324   if (reg->encoding() >= 8) {
12325     prefix(REX_B);
12326   }
12327 }
12328 
12329 void Assembler::prefix(Register dst, Register src, Prefix p) {
12330   if (src->encoding() >= 8) {
12331     p = (Prefix)(p | REX_B);
12332   }
12333   if (dst->encoding() >= 8) {
12334     p = (Prefix)(p | REX_R);
12335   }
12336   if (p != Prefix_EMPTY) {
12337     // do not generate an empty prefix
12338     prefix(p);
12339   }
12340 }
12341 
12342 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12343   if (adr.base_needs_rex()) {
12344     if (adr.index_needs_rex()) {
12345       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12346     } else {
12347       prefix(REX_B);
12348     }
12349   } else {
12350     if (adr.index_needs_rex()) {
12351       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12352     }
12353   }
12354   if (dst->encoding() >= 8) {
12355     p = (Prefix)(p | REX_R);
12356   }
12357   if (p != Prefix_EMPTY) {
12358     // do not generate an empty prefix
12359     prefix(p);
12360   }
12361 }
12362 
12363 void Assembler::prefix(Address adr) {
12364   if (adr.base_needs_rex()) {
12365     if (adr.index_needs_rex()) {
12366       prefix(REX_XB);
12367     } else {
12368       prefix(REX_B);
12369     }
12370   } else {
12371     if (adr.index_needs_rex()) {
12372       prefix(REX_X);
12373     }
12374   }
12375 }
12376 
12377 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12378   if (reg->encoding() < 8) {
12379     if (adr.base_needs_rex()) {
12380       if (adr.index_needs_rex()) {
12381         prefix(REX_XB);
12382       } else {
12383         prefix(REX_B);
12384       }
12385     } else {
12386       if (adr.index_needs_rex()) {
12387         prefix(REX_X);
12388       } else if (byteinst && reg->encoding() >= 4) {
12389         prefix(REX);
12390       }
12391     }
12392   } else {
12393     if (adr.base_needs_rex()) {
12394       if (adr.index_needs_rex()) {
12395         prefix(REX_RXB);
12396       } else {
12397         prefix(REX_RB);
12398       }
12399     } else {
12400       if (adr.index_needs_rex()) {
12401         prefix(REX_RX);
12402       } else {
12403         prefix(REX_R);
12404       }
12405     }
12406   }
12407 }
12408 
12409 void Assembler::prefix(Address adr, XMMRegister reg) {
12410   if (reg->encoding() < 8) {
12411     if (adr.base_needs_rex()) {
12412       if (adr.index_needs_rex()) {
12413         prefix(REX_XB);
12414       } else {
12415         prefix(REX_B);
12416       }
12417     } else {
12418       if (adr.index_needs_rex()) {
12419         prefix(REX_X);
12420       }
12421     }
12422   } else {
12423     if (adr.base_needs_rex()) {
12424       if (adr.index_needs_rex()) {
12425         prefix(REX_RXB);
12426       } else {
12427         prefix(REX_RB);
12428       }
12429     } else {
12430       if (adr.index_needs_rex()) {
12431         prefix(REX_RX);
12432       } else {
12433         prefix(REX_R);
12434       }
12435     }
12436   }
12437 }
12438 
12439 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12440   if (reg_enc >= 8) {
12441     prefix(REX_B);
12442     reg_enc -= 8;
12443   } else if (byteinst && reg_enc >= 4) {
12444     prefix(REX);
12445   }
12446   return reg_enc;
12447 }
12448 
12449 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12450   if (dst_enc < 8) {
12451     if (src_enc >= 8) {
12452       prefix(REX_B);
12453       src_enc -= 8;
12454     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12455       prefix(REX);
12456     }
12457   } else {
12458     if (src_enc < 8) {
12459       prefix(REX_R);
12460     } else {
12461       prefix(REX_RB);
12462       src_enc -= 8;
12463     }
12464     dst_enc -= 8;
12465   }
12466   return dst_enc << 3 | src_enc;
12467 }
12468 
12469 int8_t Assembler::get_prefixq(Address adr) {
12470   int8_t prfx = get_prefixq(adr, rax);
12471   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12472   return prfx;
12473 }
12474 
12475 int8_t Assembler::get_prefixq(Address adr, Register src) {
12476   int8_t prfx = (int8_t)(REX_W +
12477                          ((int)adr.base_needs_rex()) +
12478                          ((int)adr.index_needs_rex() << 1) +
12479                          ((int)(src->encoding() >= 8) << 2));
12480 #ifdef ASSERT
12481   if (src->encoding() < 8) {
12482     if (adr.base_needs_rex()) {
12483       if (adr.index_needs_rex()) {
12484         assert(prfx == REX_WXB, "must be");
12485       } else {
12486         assert(prfx == REX_WB, "must be");
12487       }
12488     } else {
12489       if (adr.index_needs_rex()) {
12490         assert(prfx == REX_WX, "must be");
12491       } else {
12492         assert(prfx == REX_W, "must be");
12493       }
12494     }
12495   } else {
12496     if (adr.base_needs_rex()) {
12497       if (adr.index_needs_rex()) {
12498         assert(prfx == REX_WRXB, "must be");
12499       } else {
12500         assert(prfx == REX_WRB, "must be");
12501       }
12502     } else {
12503       if (adr.index_needs_rex()) {
12504         assert(prfx == REX_WRX, "must be");
12505       } else {
12506         assert(prfx == REX_WR, "must be");
12507       }
12508     }
12509   }
12510 #endif
12511   return prfx;
12512 }
12513 
12514 void Assembler::prefixq(Address adr) {
12515   emit_int8(get_prefixq(adr));
12516 }
12517 
12518 void Assembler::prefixq(Address adr, Register src) {
12519   emit_int8(get_prefixq(adr, src));
12520 }
12521 
12522 void Assembler::prefixq(Address adr, XMMRegister src) {
12523   if (src->encoding() < 8) {
12524     if (adr.base_needs_rex()) {
12525       if (adr.index_needs_rex()) {
12526         prefix(REX_WXB);
12527       } else {
12528         prefix(REX_WB);
12529       }
12530     } else {
12531       if (adr.index_needs_rex()) {
12532         prefix(REX_WX);
12533       } else {
12534         prefix(REX_W);
12535       }
12536     }
12537   } else {
12538     if (adr.base_needs_rex()) {
12539       if (adr.index_needs_rex()) {
12540         prefix(REX_WRXB);
12541       } else {
12542         prefix(REX_WRB);
12543       }
12544     } else {
12545       if (adr.index_needs_rex()) {
12546         prefix(REX_WRX);
12547       } else {
12548         prefix(REX_WR);
12549       }
12550     }
12551   }
12552 }
12553 
12554 int Assembler::prefixq_and_encode(int reg_enc) {
12555   if (reg_enc < 8) {
12556     prefix(REX_W);
12557   } else {
12558     prefix(REX_WB);
12559     reg_enc -= 8;
12560   }
12561   return reg_enc;
12562 }
12563 
12564 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12565   if (dst_enc < 8) {
12566     if (src_enc < 8) {
12567       prefix(REX_W);
12568     } else {
12569       prefix(REX_WB);
12570       src_enc -= 8;
12571     }
12572   } else {
12573     if (src_enc < 8) {
12574       prefix(REX_WR);
12575     } else {
12576       prefix(REX_WRB);
12577       src_enc -= 8;
12578     }
12579     dst_enc -= 8;
12580   }
12581   return dst_enc << 3 | src_enc;
12582 }
12583 
12584 void Assembler::adcq(Register dst, int32_t imm32) {
12585   (void) prefixq_and_encode(dst->encoding());
12586   emit_arith(0x81, 0xD0, dst, imm32);
12587 }
12588 
12589 void Assembler::adcq(Register dst, Address src) {
12590   InstructionMark im(this);
12591   emit_int16(get_prefixq(src, dst), 0x13);
12592   emit_operand(dst, src, 0);
12593 }
12594 
12595 void Assembler::adcq(Register dst, Register src) {
12596   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12597   emit_arith(0x13, 0xC0, dst, src);
12598 }
12599 
12600 void Assembler::addq(Address dst, int32_t imm32) {
12601   InstructionMark im(this);
12602   prefixq(dst);
12603   emit_arith_operand(0x81, rax, dst, imm32);
12604 }
12605 
12606 void Assembler::addq(Address dst, Register src) {
12607   InstructionMark im(this);
12608   emit_int16(get_prefixq(dst, src), 0x01);
12609   emit_operand(src, dst, 0);
12610 }
12611 
12612 void Assembler::addq(Register dst, int32_t imm32) {
12613   (void) prefixq_and_encode(dst->encoding());
12614   emit_arith(0x81, 0xC0, dst, imm32);
12615 }
12616 
12617 void Assembler::addq(Register dst, Address src) {
12618   InstructionMark im(this);
12619   emit_int16(get_prefixq(src, dst), 0x03);
12620   emit_operand(dst, src, 0);
12621 }
12622 
12623 void Assembler::addq(Register dst, Register src) {
12624   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12625   emit_arith(0x03, 0xC0, dst, src);
12626 }
12627 
12628 void Assembler::adcxq(Register dst, Register src) {
12629   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12630   emit_int8(0x66);
12631   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12632   emit_int32(0x0F,
12633              0x38,
12634              (unsigned char)0xF6,
12635              (0xC0 | encode));
12636 }
12637 
12638 void Assembler::adoxq(Register dst, Register src) {
12639   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12640   emit_int8((unsigned char)0xF3);
12641   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12642   emit_int32(0x0F,
12643              0x38,
12644              (unsigned char)0xF6,
12645              (0xC0 | encode));
12646 }
12647 
12648 void Assembler::andq(Address dst, int32_t imm32) {
12649   InstructionMark im(this);
12650   prefixq(dst);
12651   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12652 }
12653 
12654 void Assembler::andq(Register dst, int32_t imm32) {
12655   (void) prefixq_and_encode(dst->encoding());
12656   emit_arith(0x81, 0xE0, dst, imm32);
12657 }
12658 
12659 void Assembler::andq(Register dst, Address src) {
12660   InstructionMark im(this);
12661   emit_int16(get_prefixq(src, dst), 0x23);
12662   emit_operand(dst, src, 0);
12663 }
12664 
12665 void Assembler::andq(Register dst, Register src) {
12666   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12667   emit_arith(0x23, 0xC0, dst, src);
12668 }
12669 
12670 void Assembler::andq(Address dst, Register src) {
12671   InstructionMark im(this);
12672   emit_int16(get_prefixq(dst, src), 0x21);
12673   emit_operand(src, dst, 0);
12674 }
12675 
12676 void Assembler::andnq(Register dst, Register src1, Register src2) {
12677   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12678   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12679   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12680   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12681 }
12682 
12683 void Assembler::andnq(Register dst, Register src1, Address src2) {
12684   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12685   InstructionMark im(this);
12686   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12687   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12688   emit_int8((unsigned char)0xF2);
12689   emit_operand(dst, src2, 0);
12690 }
12691 
12692 void Assembler::bsfq(Register dst, Register src) {
12693   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12694   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12695 }
12696 
12697 void Assembler::bsrq(Register dst, Register src) {
12698   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12699   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12700 }
12701 
12702 void Assembler::bswapq(Register reg) {
12703   int encode = prefixq_and_encode(reg->encoding());
12704   emit_int16(0x0F, (0xC8 | encode));
12705 }
12706 
12707 void Assembler::blsiq(Register dst, Register src) {
12708   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12709   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12710   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12711   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12712 }
12713 
12714 void Assembler::blsiq(Register dst, Address src) {
12715   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12716   InstructionMark im(this);
12717   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12718   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12719   emit_int8((unsigned char)0xF3);
12720   emit_operand(rbx, src, 0);
12721 }
12722 
12723 void Assembler::blsmskq(Register dst, Register src) {
12724   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12725   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12726   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12727   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12728 }
12729 
12730 void Assembler::blsmskq(Register dst, Address src) {
12731   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12732   InstructionMark im(this);
12733   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12734   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12735   emit_int8((unsigned char)0xF3);
12736   emit_operand(rdx, src, 0);
12737 }
12738 
12739 void Assembler::blsrq(Register dst, Register src) {
12740   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12741   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12742   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12743   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12744 }
12745 
12746 void Assembler::blsrq(Register dst, Address src) {
12747   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12748   InstructionMark im(this);
12749   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12750   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12751   emit_int8((unsigned char)0xF3);
12752   emit_operand(rcx, src, 0);
12753 }
12754 
12755 void Assembler::cdqq() {
12756   emit_int16(REX_W, (unsigned char)0x99);
12757 }
12758 
12759 void Assembler::clflush(Address adr) {
12760   assert(VM_Version::supports_clflush(), "should do");
12761   prefix(adr);
12762   emit_int16(0x0F, (unsigned char)0xAE);
12763   emit_operand(rdi, adr, 0);
12764 }
12765 
12766 void Assembler::clflushopt(Address adr) {
12767   assert(VM_Version::supports_clflushopt(), "should do!");
12768   // adr should be base reg only with no index or offset
12769   assert(adr.index() == noreg, "index should be noreg");
12770   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12771   assert(adr.disp() == 0, "displacement should be 0");
12772   // instruction prefix is 0x66
12773   emit_int8(0x66);
12774   prefix(adr);
12775   // opcode family is 0x0F 0xAE
12776   emit_int16(0x0F, (unsigned char)0xAE);
12777   // extended opcode byte is 7 == rdi
12778   emit_operand(rdi, adr, 0);
12779 }
12780 
12781 void Assembler::clwb(Address adr) {
12782   assert(VM_Version::supports_clwb(), "should do!");
12783   // adr should be base reg only with no index or offset
12784   assert(adr.index() == noreg, "index should be noreg");
12785   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12786   assert(adr.disp() == 0, "displacement should be 0");
12787   // instruction prefix is 0x66
12788   emit_int8(0x66);
12789   prefix(adr);
12790   // opcode family is 0x0f 0xAE
12791   emit_int16(0x0F, (unsigned char)0xAE);
12792   // extended opcode byte is 6 == rsi
12793   emit_operand(rsi, adr, 0);
12794 }
12795 
12796 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12797   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12798   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12799 }
12800 
12801 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12802   InstructionMark im(this);
12803   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12804   emit_operand(dst, src, 0);
12805 }
12806 
12807 void Assembler::cmpq(Address dst, int32_t imm32) {
12808   InstructionMark im(this);
12809   prefixq(dst);
12810   emit_arith_operand(0x81, as_Register(7), dst, imm32);
12811 }
12812 
12813 void Assembler::cmpq(Register dst, int32_t imm32) {
12814   (void) prefixq_and_encode(dst->encoding());
12815   emit_arith(0x81, 0xF8, dst, imm32);
12816 }
12817 
12818 void Assembler::cmpq(Address dst, Register src) {
12819   InstructionMark im(this);
12820   emit_int16(get_prefixq(dst, src), 0x39);
12821   emit_operand(src, dst, 0);
12822 }
12823 
12824 void Assembler::cmpq(Register dst, Register src) {
12825   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12826   emit_arith(0x3B, 0xC0, dst, src);
12827 }
12828 
12829 void Assembler::cmpq(Register dst, Address src) {
12830   InstructionMark im(this);
12831   emit_int16(get_prefixq(src, dst), 0x3B);
12832   emit_operand(dst, src, 0);
12833 }
12834 
12835 void Assembler::cmpxchgq(Register reg, Address adr) {
12836   InstructionMark im(this);
12837   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12838   emit_operand(reg, adr, 0);
12839 }
12840 
12841 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12842   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12843   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12844   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12845   emit_int16(0x2A, (0xC0 | encode));
12846 }
12847 
12848 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12849   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12850   InstructionMark im(this);
12851   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12852   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12853   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12854   emit_int8(0x2A);
12855   emit_operand(dst, src, 0);
12856 }
12857 
12858 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12859   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12860   InstructionMark im(this);
12861   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12862   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12863   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12864   emit_int8(0x2A);
12865   emit_operand(dst, src, 0);
12866 }
12867 
12868 void Assembler::cvttsd2siq(Register dst, Address src) {
12869   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12870   // F2 REX.W 0F 2C /r
12871   // CVTTSD2SI r64, xmm1/m64
12872   InstructionMark im(this);
12873   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12874   emit_operand(dst, src, 0);
12875 }
12876 
12877 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12878   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12879   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12880   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12881   emit_int16(0x2C, (0xC0 | encode));
12882 }
12883 
12884 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12885   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12886   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12887   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12888   emit_int16(0x2D, (0xC0 | encode));
12889 }
12890 
12891 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12892   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12893   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12894   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12895   emit_int16(0x2C, (0xC0 | encode));
12896 }
12897 
12898 void Assembler::decl(Register dst) {
12899   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12900   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12901   int encode = prefix_and_encode(dst->encoding());
12902   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12903 }
12904 
12905 void Assembler::decq(Register dst) {
12906   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12907   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12908   int encode = prefixq_and_encode(dst->encoding());
12909   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12910 }
12911 
12912 void Assembler::decq(Address dst) {
12913   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12914   InstructionMark im(this);
12915   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12916   emit_operand(rcx, dst, 0);
12917 }
12918 
12919 void Assembler::fxrstor(Address src) {
12920   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12921   emit_operand(as_Register(1), src, 0);
12922 }
12923 
12924 void Assembler::xrstor(Address src) {
12925   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12926   emit_operand(as_Register(5), src, 0);
12927 }
12928 
12929 void Assembler::fxsave(Address dst) {
12930   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12931   emit_operand(as_Register(0), dst, 0);
12932 }
12933 
12934 void Assembler::xsave(Address dst) {
12935   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12936   emit_operand(as_Register(4), dst, 0);
12937 }
12938 
12939 void Assembler::idivq(Register src) {
12940   int encode = prefixq_and_encode(src->encoding());
12941   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12942 }
12943 
12944 void Assembler::divq(Register src) {
12945   int encode = prefixq_and_encode(src->encoding());
12946   emit_int16((unsigned char)0xF7, (0xF0 | encode));
12947 }
12948 
12949 void Assembler::imulq(Register dst, Register src) {
12950   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12951   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12952 }
12953 
12954 void Assembler::imulq(Register src) {
12955   int encode = prefixq_and_encode(src->encoding());
12956   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12957 }
12958 
12959 void Assembler::imulq(Register dst, Address src, int32_t value) {
12960   InstructionMark im(this);
12961   prefixq(src, dst);
12962   if (is8bit(value)) {
12963     emit_int8((unsigned char)0x6B);
12964     emit_operand(dst, src, 1);
12965     emit_int8(value);
12966   } else {
12967     emit_int8((unsigned char)0x69);
12968     emit_operand(dst, src, 4);
12969     emit_int32(value);
12970   }
12971 }
12972 
12973 void Assembler::imulq(Register dst, Register src, int value) {
12974   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12975   if (is8bit(value)) {
12976     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12977   } else {
12978     emit_int16(0x69, (0xC0 | encode));
12979     emit_int32(value);
12980   }
12981 }
12982 
12983 void Assembler::imulq(Register dst, Address src) {
12984   InstructionMark im(this);
12985   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12986   emit_operand(dst, src, 0);
12987 }
12988 
12989 void Assembler::incl(Register dst) {
12990   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12991   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12992   int encode = prefix_and_encode(dst->encoding());
12993   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12994 }
12995 
12996 void Assembler::incq(Register dst) {
12997   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12998   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12999   int encode = prefixq_and_encode(dst->encoding());
13000   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13001 }
13002 
13003 void Assembler::incq(Address dst) {
13004   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13005   InstructionMark im(this);
13006   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13007   emit_operand(rax, dst, 0);
13008 }
13009 
13010 void Assembler::lea(Register dst, Address src) {
13011   leaq(dst, src);
13012 }
13013 
13014 void Assembler::leaq(Register dst, Address src) {
13015   InstructionMark im(this);
13016   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13017   emit_operand(dst, src, 0);
13018 }
13019 
13020 void Assembler::mov64(Register dst, int64_t imm64) {
13021   InstructionMark im(this);
13022   int encode = prefixq_and_encode(dst->encoding());
13023   emit_int8(0xB8 | encode);
13024   emit_int64(imm64);
13025 }
13026 
13027 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13028   InstructionMark im(this);
13029   int encode = prefixq_and_encode(dst->encoding());
13030   emit_int8(0xB8 | encode);
13031   emit_data64(imm64, rtype, format);
13032 }
13033 
13034 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13035   InstructionMark im(this);
13036   int encode = prefixq_and_encode(dst->encoding());
13037   emit_int8(0xB8 | encode);
13038   emit_data64(imm64, rspec);
13039 }
13040 
13041 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13042   InstructionMark im(this);
13043   int encode = prefix_and_encode(dst->encoding());
13044   emit_int8(0xB8 | encode);
13045   emit_data((int)imm32, rspec, narrow_oop_operand);
13046 }
13047 
13048 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
13049   InstructionMark im(this);
13050   prefix(dst);
13051   emit_int8((unsigned char)0xC7);
13052   emit_operand(rax, dst, 4);
13053   emit_data((int)imm32, rspec, narrow_oop_operand);
13054 }
13055 
13056 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13057   InstructionMark im(this);
13058   int encode = prefix_and_encode(src1->encoding());
13059   emit_int16((unsigned char)0x81, (0xF8 | encode));
13060   emit_data((int)imm32, rspec, narrow_oop_operand);
13061 }
13062 
13063 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13064   InstructionMark im(this);
13065   prefix(src1);
13066   emit_int8((unsigned char)0x81);
13067   emit_operand(rax, src1, 4);
13068   emit_data((int)imm32, rspec, narrow_oop_operand);
13069 }
13070 
13071 void Assembler::lzcntq(Register dst, Register src) {
13072   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13073   emit_int8((unsigned char)0xF3);
13074   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13075   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13076 }
13077 
13078 void Assembler::lzcntq(Register dst, Address src) {
13079   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13080   InstructionMark im(this);
13081   emit_int8((unsigned char)0xF3);
13082   prefixq(src, dst);
13083   emit_int16(0x0F, (unsigned char)0xBD);
13084   emit_operand(dst, src, 0);
13085 }
13086 
13087 void Assembler::movdq(XMMRegister dst, Register src) {
13088   // table D-1 says MMX/SSE2
13089   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13090   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13091   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13092   emit_int16(0x6E, (0xC0 | encode));
13093 }
13094 
13095 void Assembler::movdq(Register dst, XMMRegister src) {
13096   // table D-1 says MMX/SSE2
13097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13098   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13099   // swap src/dst to get correct prefix
13100   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13101   emit_int16(0x7E,
13102              (0xC0 | encode));
13103 }
13104 
13105 void Assembler::movq(Register dst, Register src) {
13106   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13107   emit_int16((unsigned char)0x8B,
13108              (0xC0 | encode));
13109 }
13110 
13111 void Assembler::movq(Register dst, Address src) {
13112   InstructionMark im(this);
13113   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13114   emit_operand(dst, src, 0);
13115 }
13116 
13117 void Assembler::movq(Address dst, Register src) {
13118   InstructionMark im(this);
13119   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13120   emit_operand(src, dst, 0);
13121 }
13122 
13123 void Assembler::movq(Address dst, int32_t imm32) {
13124   InstructionMark im(this);
13125   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13126   emit_operand(as_Register(0), dst, 4);
13127   emit_int32(imm32);
13128 }
13129 
13130 void Assembler::movq(Register dst, int32_t imm32) {
13131   int encode = prefixq_and_encode(dst->encoding());
13132   emit_int16((unsigned char)0xC7, (0xC0 | encode));
13133   emit_int32(imm32);
13134 }
13135 
13136 void Assembler::movsbq(Register dst, Address src) {
13137   InstructionMark im(this);
13138   emit_int24(get_prefixq(src, dst),
13139              0x0F,
13140              (unsigned char)0xBE);
13141   emit_operand(dst, src, 0);
13142 }
13143 
13144 void Assembler::movsbq(Register dst, Register src) {
13145   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13146   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13147 }
13148 
13149 void Assembler::movslq(Register dst, int32_t imm32) {
13150   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
13151   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
13152   // as a result we shouldn't use until tested at runtime...
13153   ShouldNotReachHere();
13154   InstructionMark im(this);
13155   int encode = prefixq_and_encode(dst->encoding());
13156   emit_int8(0xC7 | encode);
13157   emit_int32(imm32);
13158 }
13159 
13160 void Assembler::movslq(Address dst, int32_t imm32) {
13161   assert(is_simm32(imm32), "lost bits");
13162   InstructionMark im(this);
13163   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13164   emit_operand(rax, dst, 4);
13165   emit_int32(imm32);
13166 }
13167 
13168 void Assembler::movslq(Register dst, Address src) {
13169   InstructionMark im(this);
13170   emit_int16(get_prefixq(src, dst), 0x63);
13171   emit_operand(dst, src, 0);
13172 }
13173 
13174 void Assembler::movslq(Register dst, Register src) {
13175   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13176   emit_int16(0x63, (0xC0 | encode));
13177 }
13178 
13179 void Assembler::movswq(Register dst, Address src) {
13180   InstructionMark im(this);
13181   emit_int24(get_prefixq(src, dst),
13182              0x0F,
13183              (unsigned char)0xBF);
13184   emit_operand(dst, src, 0);
13185 }
13186 
13187 void Assembler::movswq(Register dst, Register src) {
13188   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13189   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13190 }
13191 
13192 void Assembler::movzbq(Register dst, Address src) {
13193   InstructionMark im(this);
13194   emit_int24(get_prefixq(src, dst),
13195              0x0F,
13196              (unsigned char)0xB6);
13197   emit_operand(dst, src, 0);
13198 }
13199 
13200 void Assembler::movzbq(Register dst, Register src) {
13201   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13202   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13203 }
13204 
13205 void Assembler::movzwq(Register dst, Address src) {
13206   InstructionMark im(this);
13207   emit_int24(get_prefixq(src, dst),
13208              0x0F,
13209              (unsigned char)0xB7);
13210   emit_operand(dst, src, 0);
13211 }
13212 
13213 void Assembler::movzwq(Register dst, Register src) {
13214   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13215   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13216 }
13217 
13218 void Assembler::mulq(Address src) {
13219   InstructionMark im(this);
13220   emit_int16(get_prefixq(src), (unsigned char)0xF7);
13221   emit_operand(rsp, src, 0);
13222 }
13223 
13224 void Assembler::mulq(Register src) {
13225   int encode = prefixq_and_encode(src->encoding());
13226   emit_int16((unsigned char)0xF7, (0xE0 | encode));
13227 }
13228 
13229 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13230   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13231   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13232   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13233   emit_int16((unsigned char)0xF6, (0xC0 | encode));
13234 }
13235 
13236 void Assembler::negq(Register dst) {
13237   int encode = prefixq_and_encode(dst->encoding());
13238   emit_int16((unsigned char)0xF7, (0xD8 | encode));
13239 }
13240 
13241 void Assembler::negq(Address dst) {
13242   InstructionMark im(this);
13243   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13244   emit_operand(as_Register(3), dst, 0);
13245 }
13246 
13247 void Assembler::notq(Register dst) {
13248   int encode = prefixq_and_encode(dst->encoding());
13249   emit_int16((unsigned char)0xF7, (0xD0 | encode));
13250 }
13251 
13252 void Assembler::btsq(Address dst, int imm8) {
13253   assert(isByte(imm8), "not a byte");
13254   InstructionMark im(this);
13255   emit_int24(get_prefixq(dst),
13256              0x0F,
13257              (unsigned char)0xBA);
13258   emit_operand(rbp /* 5 */, dst, 1);
13259   emit_int8(imm8);
13260 }
13261 
13262 void Assembler::btrq(Address dst, int imm8) {
13263   assert(isByte(imm8), "not a byte");
13264   InstructionMark im(this);
13265   emit_int24(get_prefixq(dst),
13266              0x0F,
13267              (unsigned char)0xBA);
13268   emit_operand(rsi /* 6 */, dst, 1);
13269   emit_int8(imm8);
13270 }
13271 
13272 void Assembler::orq(Address dst, int32_t imm32) {
13273   InstructionMark im(this);
13274   prefixq(dst);
13275   emit_arith_operand(0x81, as_Register(1), dst, imm32);
13276 }
13277 
13278 void Assembler::orq(Address dst, Register src) {
13279   InstructionMark im(this);
13280   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13281   emit_operand(src, dst, 0);
13282 }
13283 
13284 void Assembler::orq(Register dst, int32_t imm32) {
13285   (void) prefixq_and_encode(dst->encoding());
13286   emit_arith(0x81, 0xC8, dst, imm32);
13287 }
13288 
13289 void Assembler::orq(Register dst, Address src) {
13290   InstructionMark im(this);
13291   emit_int16(get_prefixq(src, dst), 0x0B);
13292   emit_operand(dst, src, 0);
13293 }
13294 
13295 void Assembler::orq(Register dst, Register src) {
13296   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13297   emit_arith(0x0B, 0xC0, dst, src);
13298 }
13299 
13300 void Assembler::popcntq(Register dst, Address src) {
13301   assert(VM_Version::supports_popcnt(), "must support");
13302   InstructionMark im(this);
13303   emit_int32((unsigned char)0xF3,
13304              get_prefixq(src, dst),
13305              0x0F,
13306              (unsigned char)0xB8);
13307   emit_operand(dst, src, 0);
13308 }
13309 
13310 void Assembler::popcntq(Register dst, Register src) {
13311   assert(VM_Version::supports_popcnt(), "must support");
13312   emit_int8((unsigned char)0xF3);
13313   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13314   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13315 }
13316 
13317 void Assembler::popq(Address dst) {
13318   InstructionMark im(this);
13319   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13320   emit_operand(rax, dst, 0);
13321 }
13322 
13323 void Assembler::popq(Register dst) {
13324   emit_int8((unsigned char)0x58 | dst->encoding());
13325 }
13326 
13327 // Precomputable: popa, pusha, vzeroupper
13328 
13329 // The result of these routines are invariant from one invocation to another
13330 // invocation for the duration of a run. Caching the result on bootstrap
13331 // and copying it out on subsequent invocations can thus be beneficial
13332 static bool     precomputed = false;
13333 
13334 static u_char* popa_code  = NULL;
13335 static int     popa_len   = 0;
13336 
13337 static u_char* pusha_code = NULL;
13338 static int     pusha_len  = 0;
13339 
13340 static u_char* vzup_code  = NULL;
13341 static int     vzup_len   = 0;
13342 
13343 void Assembler::precompute_instructions() {
13344   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13345   guarantee(!precomputed, "only once");
13346   precomputed = true;
13347   ResourceMark rm;
13348 
13349   // Make a temporary buffer big enough for the routines we're capturing
13350   int size = 256;
13351   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13352   CodeBuffer buffer((address)tmp_code, size);
13353   MacroAssembler masm(&buffer);
13354 
13355   address begin_popa  = masm.code_section()->end();
13356   masm.popa_uncached();
13357   address end_popa    = masm.code_section()->end();
13358   masm.pusha_uncached();
13359   address end_pusha   = masm.code_section()->end();
13360   masm.vzeroupper_uncached();
13361   address end_vzup    = masm.code_section()->end();
13362 
13363   // Save the instructions to permanent buffers.
13364   popa_len = (int)(end_popa - begin_popa);
13365   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13366   memcpy(popa_code, begin_popa, popa_len);
13367 
13368   pusha_len = (int)(end_pusha - end_popa);
13369   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13370   memcpy(pusha_code, end_popa, pusha_len);
13371 
13372   vzup_len = (int)(end_vzup - end_pusha);
13373   if (vzup_len > 0) {
13374     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13375     memcpy(vzup_code, end_pusha, vzup_len);
13376   } else {
13377     vzup_code = pusha_code; // dummy
13378   }
13379 
13380   assert(masm.code()->total_oop_size() == 0 &&
13381          masm.code()->total_metadata_size() == 0 &&
13382          masm.code()->total_relocation_size() == 0,
13383          "pre-computed code can't reference oops, metadata or contain relocations");
13384 }
13385 
13386 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13387   assert(src != NULL, "code to copy must have been pre-computed");
13388   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13389   address end = code_section->end();
13390   memcpy(end, src, src_len);
13391   code_section->set_end(end + src_len);
13392 }
13393 
13394 void Assembler::popa() { // 64bit
13395   emit_copy(code_section(), popa_code, popa_len);
13396 }
13397 
13398 void Assembler::popa_uncached() { // 64bit
13399   movq(r15, Address(rsp, 0));
13400   movq(r14, Address(rsp, wordSize));
13401   movq(r13, Address(rsp, 2 * wordSize));
13402   movq(r12, Address(rsp, 3 * wordSize));
13403   movq(r11, Address(rsp, 4 * wordSize));
13404   movq(r10, Address(rsp, 5 * wordSize));
13405   movq(r9,  Address(rsp, 6 * wordSize));
13406   movq(r8,  Address(rsp, 7 * wordSize));
13407   movq(rdi, Address(rsp, 8 * wordSize));
13408   movq(rsi, Address(rsp, 9 * wordSize));
13409   movq(rbp, Address(rsp, 10 * wordSize));
13410   // Skip rsp as it is restored automatically to the value
13411   // before the corresponding pusha when popa is done.
13412   movq(rbx, Address(rsp, 12 * wordSize));
13413   movq(rdx, Address(rsp, 13 * wordSize));
13414   movq(rcx, Address(rsp, 14 * wordSize));
13415   movq(rax, Address(rsp, 15 * wordSize));
13416 
13417   addq(rsp, 16 * wordSize);
13418 }
13419 
13420 // Does not actually store the value of rsp on the stack.
13421 // The slot for rsp just contains an arbitrary value.
13422 void Assembler::pusha() { // 64bit
13423   emit_copy(code_section(), pusha_code, pusha_len);
13424 }
13425 
13426 // Does not actually store the value of rsp on the stack.
13427 // The slot for rsp just contains an arbitrary value.
13428 void Assembler::pusha_uncached() { // 64bit
13429   subq(rsp, 16 * wordSize);
13430 
13431   movq(Address(rsp, 15 * wordSize), rax);
13432   movq(Address(rsp, 14 * wordSize), rcx);
13433   movq(Address(rsp, 13 * wordSize), rdx);
13434   movq(Address(rsp, 12 * wordSize), rbx);
13435   // Skip rsp as the value is normally not used. There are a few places where
13436   // the original value of rsp needs to be known but that can be computed
13437   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13438   movq(Address(rsp, 10 * wordSize), rbp);
13439   movq(Address(rsp, 9 * wordSize), rsi);
13440   movq(Address(rsp, 8 * wordSize), rdi);
13441   movq(Address(rsp, 7 * wordSize), r8);
13442   movq(Address(rsp, 6 * wordSize), r9);
13443   movq(Address(rsp, 5 * wordSize), r10);
13444   movq(Address(rsp, 4 * wordSize), r11);
13445   movq(Address(rsp, 3 * wordSize), r12);
13446   movq(Address(rsp, 2 * wordSize), r13);
13447   movq(Address(rsp, wordSize), r14);
13448   movq(Address(rsp, 0), r15);
13449 }
13450 
13451 void Assembler::vzeroupper() {
13452   emit_copy(code_section(), vzup_code, vzup_len);
13453 }
13454 
13455 void Assembler::pushq(Address src) {
13456   InstructionMark im(this);
13457   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13458   emit_operand(rsi, src, 0);
13459 }
13460 
13461 void Assembler::rclq(Register dst, int imm8) {
13462   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13463   int encode = prefixq_and_encode(dst->encoding());
13464   if (imm8 == 1) {
13465     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13466   } else {
13467     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13468   }
13469 }
13470 
13471 void Assembler::rcrq(Register dst, int imm8) {
13472   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13473   int encode = prefixq_and_encode(dst->encoding());
13474   if (imm8 == 1) {
13475     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13476   } else {
13477     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13478   }
13479 }
13480 
13481 void Assembler::rorxl(Register dst, Register src, int imm8) {
13482   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13483   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13484   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13485   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13486 }
13487 
13488 void Assembler::rorxl(Register dst, Address src, int imm8) {
13489   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13490   InstructionMark im(this);
13491   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13492   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13493   emit_int8((unsigned char)0xF0);
13494   emit_operand(dst, src, 1);
13495   emit_int8(imm8);
13496 }
13497 
13498 void Assembler::rorxq(Register dst, Register src, int imm8) {
13499   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13500   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13501   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13502   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13503 }
13504 
13505 void Assembler::rorxq(Register dst, Address src, int imm8) {
13506   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13507   InstructionMark im(this);
13508   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13509   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13510   emit_int8((unsigned char)0xF0);
13511   emit_operand(dst, src, 1);
13512   emit_int8(imm8);
13513 }
13514 
13515 #ifdef _LP64
13516 void Assembler::salq(Address dst, int imm8) {
13517   InstructionMark im(this);
13518   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13519   if (imm8 == 1) {
13520     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13521     emit_operand(as_Register(4), dst, 0);
13522   }
13523   else {
13524     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13525     emit_operand(as_Register(4), dst, 1);
13526     emit_int8(imm8);
13527   }
13528 }
13529 
13530 void Assembler::salq(Address dst) {
13531   InstructionMark im(this);
13532   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13533   emit_operand(as_Register(4), dst, 0);
13534 }
13535 
13536 void Assembler::salq(Register dst, int imm8) {
13537   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13538   int encode = prefixq_and_encode(dst->encoding());
13539   if (imm8 == 1) {
13540     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13541   } else {
13542     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13543   }
13544 }
13545 
13546 void Assembler::salq(Register dst) {
13547   int encode = prefixq_and_encode(dst->encoding());
13548   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13549 }
13550 
13551 void Assembler::sarq(Address dst, int imm8) {
13552   InstructionMark im(this);
13553   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13554   if (imm8 == 1) {
13555     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13556     emit_operand(as_Register(7), dst, 0);
13557   }
13558   else {
13559     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13560     emit_operand(as_Register(7), dst, 1);
13561     emit_int8(imm8);
13562   }
13563 }
13564 
13565 void Assembler::sarq(Address dst) {
13566   InstructionMark im(this);
13567   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13568   emit_operand(as_Register(7), dst, 0);
13569 }
13570 
13571 void Assembler::sarq(Register dst, int imm8) {
13572   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13573   int encode = prefixq_and_encode(dst->encoding());
13574   if (imm8 == 1) {
13575     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13576   } else {
13577     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13578   }
13579 }
13580 
13581 void Assembler::sarq(Register dst) {
13582   int encode = prefixq_and_encode(dst->encoding());
13583   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13584 }
13585 #endif
13586 
13587 void Assembler::sbbq(Address dst, int32_t imm32) {
13588   InstructionMark im(this);
13589   prefixq(dst);
13590   emit_arith_operand(0x81, rbx, dst, imm32);
13591 }
13592 
13593 void Assembler::sbbq(Register dst, int32_t imm32) {
13594   (void) prefixq_and_encode(dst->encoding());
13595   emit_arith(0x81, 0xD8, dst, imm32);
13596 }
13597 
13598 void Assembler::sbbq(Register dst, Address src) {
13599   InstructionMark im(this);
13600   emit_int16(get_prefixq(src, dst), 0x1B);
13601   emit_operand(dst, src, 0);
13602 }
13603 
13604 void Assembler::sbbq(Register dst, Register src) {
13605   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13606   emit_arith(0x1B, 0xC0, dst, src);
13607 }
13608 
13609 void Assembler::shlq(Register dst, int imm8) {
13610   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13611   int encode = prefixq_and_encode(dst->encoding());
13612   if (imm8 == 1) {
13613     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13614   } else {
13615     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13616   }
13617 }
13618 
13619 void Assembler::shlq(Register dst) {
13620   int encode = prefixq_and_encode(dst->encoding());
13621   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13622 }
13623 
13624 void Assembler::shrq(Register dst, int imm8) {
13625   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13626   int encode = prefixq_and_encode(dst->encoding());
13627   if (imm8 == 1) {
13628     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13629   }
13630   else {
13631     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13632   }
13633 }
13634 
13635 void Assembler::shrq(Register dst) {
13636   int encode = prefixq_and_encode(dst->encoding());
13637   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13638 }
13639 
13640 void Assembler::shrq(Address dst) {
13641   InstructionMark im(this);
13642   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13643   emit_operand(as_Register(5), dst, 0);
13644 }
13645 
13646 void Assembler::shrq(Address dst, int imm8) {
13647   InstructionMark im(this);
13648   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13649   if (imm8 == 1) {
13650     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13651     emit_operand(as_Register(5), dst, 0);
13652   }
13653   else {
13654     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13655     emit_operand(as_Register(5), dst, 1);
13656     emit_int8(imm8);
13657   }
13658 }
13659 
13660 void Assembler::subq(Address dst, int32_t imm32) {
13661   InstructionMark im(this);
13662   prefixq(dst);
13663   emit_arith_operand(0x81, rbp, dst, imm32);
13664 }
13665 
13666 void Assembler::subq(Address dst, Register src) {
13667   InstructionMark im(this);
13668   emit_int16(get_prefixq(dst, src), 0x29);
13669   emit_operand(src, dst, 0);
13670 }
13671 
13672 void Assembler::subq(Register dst, int32_t imm32) {
13673   (void) prefixq_and_encode(dst->encoding());
13674   emit_arith(0x81, 0xE8, dst, imm32);
13675 }
13676 
13677 // Force generation of a 4 byte immediate value even if it fits into 8bit
13678 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13679   (void) prefixq_and_encode(dst->encoding());
13680   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13681 }
13682 
13683 void Assembler::subq(Register dst, Address src) {
13684   InstructionMark im(this);
13685   emit_int16(get_prefixq(src, dst), 0x2B);
13686   emit_operand(dst, src, 0);
13687 }
13688 
13689 void Assembler::subq(Register dst, Register src) {
13690   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13691   emit_arith(0x2B, 0xC0, dst, src);
13692 }
13693 
13694 void Assembler::testq(Address dst, int32_t imm32) {
13695   InstructionMark im(this);
13696   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13697   emit_operand(as_Register(0), dst, 4);
13698   emit_int32(imm32);
13699 }
13700 
13701 void Assembler::testq(Register dst, int32_t imm32) {
13702   // not using emit_arith because test
13703   // doesn't support sign-extension of
13704   // 8bit operands
13705   if (dst == rax) {
13706     prefix(REX_W);
13707     emit_int8((unsigned char)0xA9);
13708     emit_int32(imm32);
13709   } else {
13710     int encode = dst->encoding();
13711     encode = prefixq_and_encode(encode);
13712     emit_int16((unsigned char)0xF7, (0xC0 | encode));
13713     emit_int32(imm32);
13714   }
13715 }
13716 
13717 void Assembler::testq(Register dst, Register src) {
13718   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13719   emit_arith(0x85, 0xC0, dst, src);
13720 }
13721 
13722 void Assembler::testq(Register dst, Address src) {
13723   InstructionMark im(this);
13724   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13725   emit_operand(dst, src, 0);
13726 }
13727 
13728 void Assembler::xaddq(Address dst, Register src) {
13729   InstructionMark im(this);
13730   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13731   emit_operand(src, dst, 0);
13732 }
13733 
13734 void Assembler::xchgq(Register dst, Address src) {
13735   InstructionMark im(this);
13736   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13737   emit_operand(dst, src, 0);
13738 }
13739 
13740 void Assembler::xchgq(Register dst, Register src) {
13741   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13742   emit_int16((unsigned char)0x87, (0xc0 | encode));
13743 }
13744 
13745 void Assembler::xorq(Register dst, Register src) {
13746   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13747   emit_arith(0x33, 0xC0, dst, src);
13748 }
13749 
13750 void Assembler::xorq(Register dst, Address src) {
13751   InstructionMark im(this);
13752   emit_int16(get_prefixq(src, dst), 0x33);
13753   emit_operand(dst, src, 0);
13754 }
13755 
13756 void Assembler::xorq(Register dst, int32_t imm32) {
13757   (void) prefixq_and_encode(dst->encoding());
13758   emit_arith(0x81, 0xF0, dst, imm32);
13759 }
13760 
13761 void Assembler::xorq(Address dst, int32_t imm32) {
13762   InstructionMark im(this);
13763   prefixq(dst);
13764   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13765 }
13766 
13767 void Assembler::xorq(Address dst, Register src) {
13768   InstructionMark im(this);
13769   emit_int16(get_prefixq(dst, src), 0x31);
13770   emit_operand(src, dst, 0);
13771 }
13772 
13773 #endif // !LP64
13774 
13775 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13776   if (VM_Version::supports_evex()) {
13777     _tuple_type = tuple_type;
13778     _input_size_in_bits = input_size_in_bits;
13779   }
13780 }