1 /*
    2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec = RelocationHolder::none;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert(op1 == 0x81, "Unexpected opcode");
  304   if (is8bit(imm32)) {
  305     emit_int24(op1 | 0x02,        // set sign bit
  306                op2 | encode(dst),
  307                imm32 & 0xFF);
  308   } else if (dst == rax) {
  309     switch (op2) {
  310       case 0xD0: emit_int8(0x15); break; // adc
  311       case 0xC0: emit_int8(0x05); break; // add
  312       case 0xE0: emit_int8(0x25); break; // and
  313       case 0xF8: emit_int8(0x3D); break; // cmp
  314       case 0xC8: emit_int8(0x0D); break; // or
  315       case 0xD8: emit_int8(0x1D); break; // sbb
  316       case 0xE8: emit_int8(0x2D); break; // sub
  317       case 0xF0: emit_int8(0x35); break; // xor
  318       default: ShouldNotReachHere();
  319     }
  320     emit_int32(imm32);
  321   } else {
  322     emit_int16(op1, (op2 | encode(dst)));
  323     emit_int32(imm32);
  324   }
  325 }
  326 
  327 // Force generation of a 4 byte immediate value even if it fits into 8bit
  328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  329   assert(isByte(op1) && isByte(op2), "wrong opcode");
  330   assert((op1 & 0x01) == 1, "should be 32bit operation");
  331   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  332   emit_int16(op1, (op2 | encode(dst)));
  333   emit_int32(imm32);
  334 }
  335 
  336 // immediate-to-memory forms
  337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  338   assert((op1 & 0x01) == 1, "should be 32bit operation");
  339   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  340   if (is8bit(imm32)) {
  341     emit_int8(op1 | 0x02); // set sign bit
  342     emit_operand(rm, adr, 1);
  343     emit_int8(imm32 & 0xFF);
  344   } else {
  345     emit_int8(op1);
  346     emit_operand(rm, adr, 4);
  347     emit_int32(imm32);
  348   }
  349 }
  350 
  351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
  352   assert(op1 == 0x81, "unexpected opcode");
  353   emit_int8(op1);
  354   emit_operand(rm, adr, 4);
  355   emit_int32(imm32);
  356 }
  357 
  358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  359   assert(isByte(op1) && isByte(op2), "wrong opcode");
  360   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  361 }
  362 
  363 
  364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  365                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  366   int mod_idx = 0;
  367   // We will test if the displacement fits the compressed format and if so
  368   // apply the compression to the displacement iff the result is8bit.
  369   if (VM_Version::supports_evex() && is_evex_inst) {
  370     switch (cur_tuple_type) {
  371     case EVEX_FV:
  372       if ((cur_encoding & VEX_W) == VEX_W) {
  373         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  374       } else {
  375         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  376       }
  377       break;
  378 
  379     case EVEX_HV:
  380       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  381       break;
  382 
  383     case EVEX_FVM:
  384       break;
  385 
  386     case EVEX_T1S:
  387       switch (in_size_in_bits) {
  388       case EVEX_8bit:
  389         break;
  390 
  391       case EVEX_16bit:
  392         mod_idx = 1;
  393         break;
  394 
  395       case EVEX_32bit:
  396         mod_idx = 2;
  397         break;
  398 
  399       case EVEX_64bit:
  400         mod_idx = 3;
  401         break;
  402       }
  403       break;
  404 
  405     case EVEX_T1F:
  406     case EVEX_T2:
  407     case EVEX_T4:
  408       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  409       break;
  410 
  411     case EVEX_T8:
  412       break;
  413 
  414     case EVEX_HVM:
  415       break;
  416 
  417     case EVEX_QVM:
  418       break;
  419 
  420     case EVEX_OVM:
  421       break;
  422 
  423     case EVEX_M128:
  424       break;
  425 
  426     case EVEX_DUP:
  427       break;
  428 
  429     default:
  430       assert(0, "no valid evex tuple_table entry");
  431       break;
  432     }
  433 
  434     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  435       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  436       if ((disp % disp_factor) == 0) {
  437         int new_disp = disp / disp_factor;
  438         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  439           disp = new_disp;
  440         }
  441       } else {
  442         return false;
  443       }
  444     }
  445   }
  446   return (-0x80 <= disp && disp < 0x80);
  447 }
  448 
  449 
  450 bool Assembler::emit_compressed_disp_byte(int &disp) {
  451   int mod_idx = 0;
  452   // We will test if the displacement fits the compressed format and if so
  453   // apply the compression to the displacement iff the result is8bit.
  454   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  455     int evex_encoding = _attributes->get_evex_encoding();
  456     int tuple_type = _attributes->get_tuple_type();
  457     switch (tuple_type) {
  458     case EVEX_FV:
  459       if ((evex_encoding & VEX_W) == VEX_W) {
  460         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  461       } else {
  462         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  463       }
  464       break;
  465 
  466     case EVEX_HV:
  467       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  468       break;
  469 
  470     case EVEX_FVM:
  471       break;
  472 
  473     case EVEX_T1S:
  474       switch (_attributes->get_input_size()) {
  475       case EVEX_8bit:
  476         break;
  477 
  478       case EVEX_16bit:
  479         mod_idx = 1;
  480         break;
  481 
  482       case EVEX_32bit:
  483         mod_idx = 2;
  484         break;
  485 
  486       case EVEX_64bit:
  487         mod_idx = 3;
  488         break;
  489       }
  490       break;
  491 
  492     case EVEX_T1F:
  493     case EVEX_T2:
  494     case EVEX_T4:
  495       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  496       break;
  497 
  498     case EVEX_T8:
  499       break;
  500 
  501     case EVEX_HVM:
  502       break;
  503 
  504     case EVEX_QVM:
  505       break;
  506 
  507     case EVEX_OVM:
  508       break;
  509 
  510     case EVEX_M128:
  511       break;
  512 
  513     case EVEX_DUP:
  514       break;
  515 
  516     default:
  517       assert(0, "no valid evex tuple_table entry");
  518       break;
  519     }
  520 
  521     int vector_len = _attributes->get_vector_len();
  522     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  523       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  524       if ((disp % disp_factor) == 0) {
  525         int new_disp = disp / disp_factor;
  526         if (is8bit(new_disp)) {
  527           disp = new_disp;
  528         }
  529       } else {
  530         return false;
  531       }
  532     }
  533   }
  534   return is8bit(disp);
  535 }
  536 
  537 static bool is_valid_encoding(int reg_enc) {
  538   return reg_enc >= 0;
  539 }
  540 
  541 static int raw_encode(Register reg) {
  542   assert(reg == noreg || reg->is_valid(), "sanity");
  543   int reg_enc = reg->raw_encoding();
  544   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  545   return reg_enc;
  546 }
  547 
  548 static int raw_encode(XMMRegister xmmreg) {
  549   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  550   int xmmreg_enc = xmmreg->raw_encoding();
  551   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  552   return xmmreg_enc;
  553 }
  554 
  555 static int raw_encode(KRegister kreg) {
  556   assert(kreg == knoreg || kreg->is_valid(), "sanity");
  557   int kreg_enc = kreg->raw_encoding();
  558   assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
  559   return kreg_enc;
  560 }
  561 
  562 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  563   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  564 }
  565 
  566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  567   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  568 }
  569 
  570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  571   assert((mod & 3) != 0b11, "forbidden");
  572   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  573   emit_int8(modrm);
  574 }
  575 
  576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  577                                         int disp) {
  578   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  579   emit_int16(modrm, disp & 0xFF);
  580 }
  581 
  582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  583                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  584   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  585   int sib = sib_encoding(scale, index_enc, base_enc);
  586   emit_int16(modrm, sib);
  587 }
  588 
  589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  590                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  591                                             int disp) {
  592   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  593   int sib = sib_encoding(scale, index_enc, base_enc);
  594   emit_int24(modrm, sib, disp & 0xFF);
  595 }
  596 
  597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  598                                     Address::ScaleFactor scale, int disp,
  599                                     RelocationHolder const& rspec,
  600                                     int post_addr_length) {
  601   bool no_relocation = (rspec.type() == relocInfo::none);
  602 
  603   if (is_valid_encoding(base_enc)) {
  604     if (is_valid_encoding(index_enc)) {
  605       assert(scale != Address::no_scale, "inconsistent address");
  606       // [base + index*scale + disp]
  607       if (disp == 0 && no_relocation &&
  608           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  609         // [base + index*scale]
  610         // [00 reg 100][ss index base]
  611         emit_modrm_sib(0b00, reg_enc, 0b100,
  612                        scale, index_enc, base_enc);
  613       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  614         // [base + index*scale + imm8]
  615         // [01 reg 100][ss index base] imm8
  616         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  617                              scale, index_enc, base_enc,
  618                              disp);
  619       } else {
  620         // [base + index*scale + disp32]
  621         // [10 reg 100][ss index base] disp32
  622         emit_modrm_sib(0b10, reg_enc, 0b100,
  623                        scale, index_enc, base_enc);
  624         emit_data(disp, rspec, disp32_operand);
  625       }
  626     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  627       // [rsp + disp]
  628       if (disp == 0 && no_relocation) {
  629         // [rsp]
  630         // [00 reg 100][00 100 100]
  631         emit_modrm_sib(0b00, reg_enc, 0b100,
  632                        Address::times_1, 0b100, 0b100);
  633       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  634         // [rsp + imm8]
  635         // [01 reg 100][00 100 100] disp8
  636         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  637                              Address::times_1, 0b100, 0b100,
  638                              disp);
  639       } else {
  640         // [rsp + imm32]
  641         // [10 reg 100][00 100 100] disp32
  642         emit_modrm_sib(0b10, reg_enc, 0b100,
  643                        Address::times_1, 0b100, 0b100);
  644         emit_data(disp, rspec, disp32_operand);
  645       }
  646     } else {
  647       // [base + disp]
  648       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  649       if (disp == 0 && no_relocation &&
  650           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  651         // [base]
  652         // [00 reg base]
  653         emit_modrm(0, reg_enc, base_enc);
  654       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  655         // [base + disp8]
  656         // [01 reg base] disp8
  657         emit_modrm_disp8(0b01, reg_enc, base_enc,
  658                          disp);
  659       } else {
  660         // [base + disp32]
  661         // [10 reg base] disp32
  662         emit_modrm(0b10, reg_enc, base_enc);
  663         emit_data(disp, rspec, disp32_operand);
  664       }
  665     }
  666   } else {
  667     if (is_valid_encoding(index_enc)) {
  668       assert(scale != Address::no_scale, "inconsistent address");
  669       // base == noreg
  670       // [index*scale + disp]
  671       // [00 reg 100][ss index 101] disp32
  672       emit_modrm_sib(0b00, reg_enc, 0b100,
  673                      scale, index_enc, 0b101 /* no base */);
  674       emit_data(disp, rspec, disp32_operand);
  675     } else if (!no_relocation) {
  676       // base == noreg, index == noreg
  677       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  678       // [00 reg 101] disp32
  679 
  680       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  681       // Note that the RIP-rel. correction applies to the generated
  682       // disp field, but _not_ to the target address in the rspec.
  683 
  684       // disp was created by converting the target address minus the pc
  685       // at the start of the instruction. That needs more correction here.
  686       // intptr_t disp = target - next_ip;
  687       assert(inst_mark() != NULL, "must be inside InstructionMark");
  688       address next_ip = pc() + sizeof(int32_t) + post_addr_length;
  689       int64_t adjusted = disp;
  690       // Do rip-rel adjustment for 64bit
  691       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  692       assert(is_simm32(adjusted),
  693              "must be 32bit offset (RIP relative address)");
  694       emit_data((int32_t) adjusted, rspec, disp32_operand);
  695 
  696     } else {
  697       // base == noreg, index == noreg, no_relocation == true
  698       // 32bit never did this, did everything as the rip-rel/disp code above
  699       // [disp] ABSOLUTE
  700       // [00 reg 100][00 100 101] disp32
  701       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  702                      Address::times_1, 0b100, 0b101);
  703       emit_data(disp, rspec, disp32_operand);
  704     }
  705   }
  706 }
  707 
  708 void Assembler::emit_operand(Register reg, Register base, Register index,
  709                              Address::ScaleFactor scale, int disp,
  710                              RelocationHolder const& rspec,
  711                              int post_addr_length) {
  712   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  713   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  714                       scale, disp, rspec, post_addr_length);
  715 
  716 }
  717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  718                              Address::ScaleFactor scale, int disp,
  719                              RelocationHolder const& rspec,
  720                              int post_addr_length) {
  721   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  722   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  723   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  724                       scale, disp, rspec, post_addr_length);
  725 }
  726 
  727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  728                              Address::ScaleFactor scale, int disp,
  729                              RelocationHolder const& rspec,
  730                              int post_addr_length) {
  731   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  732   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  733   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  734                       scale, disp, rspec, post_addr_length);
  735 }
  736 
  737 void Assembler::emit_operand(KRegister kreg, Address adr,
  738                              int post_addr_length) {
  739   emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
  740                adr._rspec,
  741                post_addr_length);
  742 }
  743 
  744 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
  745                              Address::ScaleFactor scale, int disp,
  746                              RelocationHolder const& rspec,
  747                              int post_addr_length) {
  748   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  749   emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
  750                       scale, disp, rspec, post_addr_length);
  751 }
  752 
  753 // Secret local extension to Assembler::WhichOperand:
  754 #define end_pc_operand (_WhichOperand_limit)
  755 
  756 address Assembler::locate_operand(address inst, WhichOperand which) {
  757   // Decode the given instruction, and return the address of
  758   // an embedded 32-bit operand word.
  759 
  760   // If "which" is disp32_operand, selects the displacement portion
  761   // of an effective address specifier.
  762   // If "which" is imm64_operand, selects the trailing immediate constant.
  763   // If "which" is call32_operand, selects the displacement of a call or jump.
  764   // Caller is responsible for ensuring that there is such an operand,
  765   // and that it is 32/64 bits wide.
  766 
  767   // If "which" is end_pc_operand, find the end of the instruction.
  768 
  769   address ip = inst;
  770   bool is_64bit = false;
  771 
  772   debug_only(bool has_disp32 = false);
  773   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  774 
  775   again_after_prefix:
  776   switch (0xFF & *ip++) {
  777 
  778   // These convenience macros generate groups of "case" labels for the switch.
  779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  781              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  782 #define REP16(x) REP8((x)+0): \
  783               case REP8((x)+8)
  784 
  785   case CS_segment:
  786   case SS_segment:
  787   case DS_segment:
  788   case ES_segment:
  789   case FS_segment:
  790   case GS_segment:
  791     // Seems dubious
  792     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  793     assert(ip == inst+1, "only one prefix allowed");
  794     goto again_after_prefix;
  795 
  796   case 0x67:
  797   case REX:
  798   case REX_B:
  799   case REX_X:
  800   case REX_XB:
  801   case REX_R:
  802   case REX_RB:
  803   case REX_RX:
  804   case REX_RXB:
  805     NOT_LP64(assert(false, "64bit prefixes"));
  806     goto again_after_prefix;
  807 
  808   case REX_W:
  809   case REX_WB:
  810   case REX_WX:
  811   case REX_WXB:
  812   case REX_WR:
  813   case REX_WRB:
  814   case REX_WRX:
  815   case REX_WRXB:
  816     NOT_LP64(assert(false, "64bit prefixes"));
  817     is_64bit = true;
  818     goto again_after_prefix;
  819 
  820   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  821   case 0x88: // movb a, r
  822   case 0x89: // movl a, r
  823   case 0x8A: // movb r, a
  824   case 0x8B: // movl r, a
  825   case 0x8F: // popl a
  826     debug_only(has_disp32 = true);
  827     break;
  828 
  829   case 0x68: // pushq #32
  830     if (which == end_pc_operand) {
  831       return ip + 4;
  832     }
  833     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  834     return ip;                  // not produced by emit_operand
  835 
  836   case 0x66: // movw ... (size prefix)
  837     again_after_size_prefix2:
  838     switch (0xFF & *ip++) {
  839     case REX:
  840     case REX_B:
  841     case REX_X:
  842     case REX_XB:
  843     case REX_R:
  844     case REX_RB:
  845     case REX_RX:
  846     case REX_RXB:
  847     case REX_W:
  848     case REX_WB:
  849     case REX_WX:
  850     case REX_WXB:
  851     case REX_WR:
  852     case REX_WRB:
  853     case REX_WRX:
  854     case REX_WRXB:
  855       NOT_LP64(assert(false, "64bit prefix found"));
  856       goto again_after_size_prefix2;
  857     case 0x8B: // movw r, a
  858     case 0x89: // movw a, r
  859       debug_only(has_disp32 = true);
  860       break;
  861     case 0xC7: // movw a, #16
  862       debug_only(has_disp32 = true);
  863       tail_size = 2;  // the imm16
  864       break;
  865     case 0x0F: // several SSE/SSE2 variants
  866       ip--;    // reparse the 0x0F
  867       goto again_after_prefix;
  868     default:
  869       ShouldNotReachHere();
  870     }
  871     break;
  872 
  873   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  874     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  875     // these asserts are somewhat nonsensical
  876 #ifndef _LP64
  877     assert(which == imm_operand || which == disp32_operand,
  878            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  879 #else
  880     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  881            which == narrow_oop_operand && !is_64bit,
  882            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  883 #endif // _LP64
  884     return ip;
  885 
  886   case 0x69: // imul r, a, #32
  887   case 0xC7: // movl a, #32(oop?)
  888     tail_size = 4;
  889     debug_only(has_disp32 = true); // has both kinds of operands!
  890     break;
  891 
  892   case 0x0F: // movx..., etc.
  893     switch (0xFF & *ip++) {
  894     case 0x3A: // pcmpestri
  895       tail_size = 1;
  896     case 0x38: // ptest, pmovzxbw
  897       ip++; // skip opcode
  898       debug_only(has_disp32 = true); // has both kinds of operands!
  899       break;
  900 
  901     case 0x70: // pshufd r, r/a, #8
  902       debug_only(has_disp32 = true); // has both kinds of operands!
  903     case 0x73: // psrldq r, #8
  904       tail_size = 1;
  905       break;
  906 
  907     case 0x10: // movups
  908     case 0x11: // movups
  909     case 0x12: // movlps
  910     case 0x28: // movaps
  911     case 0x2E: // ucomiss
  912     case 0x2F: // comiss
  913     case 0x54: // andps
  914     case 0x55: // andnps
  915     case 0x56: // orps
  916     case 0x57: // xorps
  917     case 0x58: // addpd
  918     case 0x59: // mulpd
  919     case 0x6E: // movd
  920     case 0x7E: // movd
  921     case 0x6F: // movdq
  922     case 0x7F: // movdq
  923     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  924     case 0xFE: // paddd
  925       debug_only(has_disp32 = true);
  926       break;
  927 
  928     case 0xAD: // shrd r, a, %cl
  929     case 0xAF: // imul r, a
  930     case 0xBE: // movsbl r, a (movsxb)
  931     case 0xBF: // movswl r, a (movsxw)
  932     case 0xB6: // movzbl r, a (movzxb)
  933     case 0xB7: // movzwl r, a (movzxw)
  934     case REP16(0x40): // cmovl cc, r, a
  935     case 0xB0: // cmpxchgb
  936     case 0xB1: // cmpxchg
  937     case 0xC1: // xaddl
  938     case 0xC7: // cmpxchg8
  939     case REP16(0x90): // setcc a
  940       debug_only(has_disp32 = true);
  941       // fall out of the switch to decode the address
  942       break;
  943 
  944     case 0xC4: // pinsrw r, a, #8
  945       debug_only(has_disp32 = true);
  946     case 0xC5: // pextrw r, r, #8
  947       tail_size = 1;  // the imm8
  948       break;
  949 
  950     case 0xAC: // shrd r, a, #8
  951       debug_only(has_disp32 = true);
  952       tail_size = 1;  // the imm8
  953       break;
  954 
  955     case REP16(0x80): // jcc rdisp32
  956       if (which == end_pc_operand)  return ip + 4;
  957       assert(which == call32_operand, "jcc has no disp32 or imm");
  958       return ip;
  959     default:
  960       ShouldNotReachHere();
  961     }
  962     break;
  963 
  964   case 0x81: // addl a, #32; addl r, #32
  965     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  966     // on 32bit in the case of cmpl, the imm might be an oop
  967     tail_size = 4;
  968     debug_only(has_disp32 = true); // has both kinds of operands!
  969     break;
  970 
  971   case 0x83: // addl a, #8; addl r, #8
  972     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  973     debug_only(has_disp32 = true); // has both kinds of operands!
  974     tail_size = 1;
  975     break;
  976 
  977   case 0x15: // adc rax, #32
  978   case 0x05: // add rax, #32
  979   case 0x25: // and rax, #32
  980   case 0x3D: // cmp rax, #32
  981   case 0x0D: // or  rax, #32
  982   case 0x1D: // sbb rax, #32
  983   case 0x2D: // sub rax, #32
  984   case 0x35: // xor rax, #32
  985     return which == end_pc_operand ? ip + 4 : ip;
  986 
  987   case 0x9B:
  988     switch (0xFF & *ip++) {
  989     case 0xD9: // fnstcw a
  990       debug_only(has_disp32 = true);
  991       break;
  992     default:
  993       ShouldNotReachHere();
  994     }
  995     break;
  996 
  997   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  998   case REP4(0x10): // adc...
  999   case REP4(0x20): // and...
 1000   case REP4(0x30): // xor...
 1001   case REP4(0x08): // or...
 1002   case REP4(0x18): // sbb...
 1003   case REP4(0x28): // sub...
 1004   case 0xF7: // mull a
 1005   case 0x8D: // lea r, a
 1006   case 0x87: // xchg r, a
 1007   case REP4(0x38): // cmp...
 1008   case 0x85: // test r, a
 1009     debug_only(has_disp32 = true); // has both kinds of operands!
 1010     break;
 1011 
 1012   case 0xA8: // testb rax, #8
 1013     return which == end_pc_operand ? ip + 1 : ip;
 1014   case 0xA9: // testl/testq rax, #32
 1015     return which == end_pc_operand ? ip + 4 : ip;
 1016 
 1017   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 1018   case 0xC6: // movb a, #8
 1019   case 0x80: // cmpb a, #8
 1020   case 0x6B: // imul r, a, #8
 1021     debug_only(has_disp32 = true); // has both kinds of operands!
 1022     tail_size = 1; // the imm8
 1023     break;
 1024 
 1025   case 0xC4: // VEX_3bytes
 1026   case 0xC5: // VEX_2bytes
 1027     assert((UseAVX > 0), "shouldn't have VEX prefix");
 1028     assert(ip == inst+1, "no prefixes allowed");
 1029     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 1030     // but they have prefix 0x0F and processed when 0x0F processed above.
 1031     //
 1032     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1033     // instructions (these instructions are not supported in 64-bit mode).
 1034     // To distinguish them bits [7:6] are set in the VEX second byte since
 1035     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1036     // those VEX bits REX and vvvv bits are inverted.
 1037     //
 1038     // Fortunately C2 doesn't generate these instructions so we don't need
 1039     // to check for them in product version.
 1040 
 1041     // Check second byte
 1042     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1043 
 1044     int vex_opcode;
 1045     // First byte
 1046     if ((0xFF & *inst) == VEX_3bytes) {
 1047       vex_opcode = VEX_OPCODE_MASK & *ip;
 1048       ip++; // third byte
 1049       is_64bit = ((VEX_W & *ip) == VEX_W);
 1050     } else {
 1051       vex_opcode = VEX_OPCODE_0F;
 1052     }
 1053     ip++; // opcode
 1054     // To find the end of instruction (which == end_pc_operand).
 1055     switch (vex_opcode) {
 1056       case VEX_OPCODE_0F:
 1057         switch (0xFF & *ip) {
 1058         case 0x70: // pshufd r, r/a, #8
 1059         case 0x71: // ps[rl|ra|ll]w r, #8
 1060         case 0x72: // ps[rl|ra|ll]d r, #8
 1061         case 0x73: // ps[rl|ra|ll]q r, #8
 1062         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1063         case 0xC4: // pinsrw r, r, r/a, #8
 1064         case 0xC5: // pextrw r/a, r, #8
 1065         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1066           tail_size = 1;  // the imm8
 1067           break;
 1068         }
 1069         break;
 1070       case VEX_OPCODE_0F_3A:
 1071         tail_size = 1;
 1072         break;
 1073     }
 1074     ip++; // skip opcode
 1075     debug_only(has_disp32 = true); // has both kinds of operands!
 1076     break;
 1077 
 1078   case 0x62: // EVEX_4bytes
 1079     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1080     assert(ip == inst+1, "no prefixes allowed");
 1081     // no EVEX collisions, all instructions that have 0x62 opcodes
 1082     // have EVEX versions and are subopcodes of 0x66
 1083     ip++; // skip P0 and examine W in P1
 1084     is_64bit = ((VEX_W & *ip) == VEX_W);
 1085     ip++; // move to P2
 1086     ip++; // skip P2, move to opcode
 1087     // To find the end of instruction (which == end_pc_operand).
 1088     switch (0xFF & *ip) {
 1089     case 0x22: // pinsrd r, r/a, #8
 1090     case 0x61: // pcmpestri r, r/a, #8
 1091     case 0x70: // pshufd r, r/a, #8
 1092     case 0x73: // psrldq r, #8
 1093     case 0x1f: // evpcmpd/evpcmpq
 1094     case 0x3f: // evpcmpb/evpcmpw
 1095       tail_size = 1;  // the imm8
 1096       break;
 1097     default:
 1098       break;
 1099     }
 1100     ip++; // skip opcode
 1101     debug_only(has_disp32 = true); // has both kinds of operands!
 1102     break;
 1103 
 1104   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1105   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1106   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1107   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1108   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1109   case 0xDF: // fild_d a; fistp_d a
 1110   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1111   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1112   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1113     debug_only(has_disp32 = true);
 1114     break;
 1115 
 1116   case 0xE8: // call rdisp32
 1117   case 0xE9: // jmp  rdisp32
 1118     if (which == end_pc_operand)  return ip + 4;
 1119     assert(which == call32_operand, "call has no disp32 or imm");
 1120     return ip;
 1121 
 1122   case 0xF0:                    // Lock
 1123     goto again_after_prefix;
 1124 
 1125   case 0xF3:                    // For SSE
 1126   case 0xF2:                    // For SSE2
 1127     switch (0xFF & *ip++) {
 1128     case REX:
 1129     case REX_B:
 1130     case REX_X:
 1131     case REX_XB:
 1132     case REX_R:
 1133     case REX_RB:
 1134     case REX_RX:
 1135     case REX_RXB:
 1136     case REX_W:
 1137     case REX_WB:
 1138     case REX_WX:
 1139     case REX_WXB:
 1140     case REX_WR:
 1141     case REX_WRB:
 1142     case REX_WRX:
 1143     case REX_WRXB:
 1144       NOT_LP64(assert(false, "found 64bit prefix"));
 1145       ip++;
 1146     default:
 1147       ip++;
 1148     }
 1149     debug_only(has_disp32 = true); // has both kinds of operands!
 1150     break;
 1151 
 1152   default:
 1153     ShouldNotReachHere();
 1154 
 1155 #undef REP8
 1156 #undef REP16
 1157   }
 1158 
 1159   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1160 #ifdef _LP64
 1161   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1162 #else
 1163   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1164   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1165 #endif // LP64
 1166   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1167 
 1168   // parse the output of emit_operand
 1169   int op2 = 0xFF & *ip++;
 1170   int base = op2 & 0x07;
 1171   int op3 = -1;
 1172   const int b100 = 4;
 1173   const int b101 = 5;
 1174   if (base == b100 && (op2 >> 6) != 3) {
 1175     op3 = 0xFF & *ip++;
 1176     base = op3 & 0x07;   // refetch the base
 1177   }
 1178   // now ip points at the disp (if any)
 1179 
 1180   switch (op2 >> 6) {
 1181   case 0:
 1182     // [00 reg  100][ss index base]
 1183     // [00 reg  100][00   100  esp]
 1184     // [00 reg base]
 1185     // [00 reg  100][ss index  101][disp32]
 1186     // [00 reg  101]               [disp32]
 1187 
 1188     if (base == b101) {
 1189       if (which == disp32_operand)
 1190         return ip;              // caller wants the disp32
 1191       ip += 4;                  // skip the disp32
 1192     }
 1193     break;
 1194 
 1195   case 1:
 1196     // [01 reg  100][ss index base][disp8]
 1197     // [01 reg  100][00   100  esp][disp8]
 1198     // [01 reg base]               [disp8]
 1199     ip += 1;                    // skip the disp8
 1200     break;
 1201 
 1202   case 2:
 1203     // [10 reg  100][ss index base][disp32]
 1204     // [10 reg  100][00   100  esp][disp32]
 1205     // [10 reg base]               [disp32]
 1206     if (which == disp32_operand)
 1207       return ip;                // caller wants the disp32
 1208     ip += 4;                    // skip the disp32
 1209     break;
 1210 
 1211   case 3:
 1212     // [11 reg base]  (not a memory addressing mode)
 1213     break;
 1214   }
 1215 
 1216   if (which == end_pc_operand) {
 1217     return ip + tail_size;
 1218   }
 1219 
 1220 #ifdef _LP64
 1221   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1222 #else
 1223   assert(which == imm_operand, "instruction has only an imm field");
 1224 #endif // LP64
 1225   return ip;
 1226 }
 1227 
 1228 address Assembler::locate_next_instruction(address inst) {
 1229   // Secretly share code with locate_operand:
 1230   return locate_operand(inst, end_pc_operand);
 1231 }
 1232 
 1233 
 1234 #ifdef ASSERT
 1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1236   address inst = inst_mark();
 1237   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1238   address opnd;
 1239 
 1240   Relocation* r = rspec.reloc();
 1241   if (r->type() == relocInfo::none) {
 1242     return;
 1243   } else if (r->is_call() || format == call32_operand) {
 1244     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1245     opnd = locate_operand(inst, call32_operand);
 1246   } else if (r->is_data()) {
 1247     assert(format == imm_operand || format == disp32_operand
 1248            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1249     opnd = locate_operand(inst, (WhichOperand)format);
 1250   } else {
 1251     assert(format == imm_operand, "cannot specify a format");
 1252     return;
 1253   }
 1254   assert(opnd == pc(), "must put operand where relocs can find it");
 1255 }
 1256 #endif // ASSERT
 1257 
 1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
 1259   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1260 }
 1261 
 1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
 1263   if (adr.isxmmindex()) {
 1264      emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1265   } else {
 1266      emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1267   }
 1268 }
 1269 
 1270 // Now the Assembler instructions (identical for 32/64 bits)
 1271 
 1272 void Assembler::adcl(Address dst, int32_t imm32) {
 1273   InstructionMark im(this);
 1274   prefix(dst);
 1275   emit_arith_operand(0x81, rdx, dst, imm32);
 1276 }
 1277 
 1278 void Assembler::adcl(Address dst, Register src) {
 1279   InstructionMark im(this);
 1280   prefix(dst, src);
 1281   emit_int8(0x11);
 1282   emit_operand(src, dst, 0);
 1283 }
 1284 
 1285 void Assembler::adcl(Register dst, int32_t imm32) {
 1286   prefix(dst);
 1287   emit_arith(0x81, 0xD0, dst, imm32);
 1288 }
 1289 
 1290 void Assembler::adcl(Register dst, Address src) {
 1291   InstructionMark im(this);
 1292   prefix(src, dst);
 1293   emit_int8(0x13);
 1294   emit_operand(dst, src, 0);
 1295 }
 1296 
 1297 void Assembler::adcl(Register dst, Register src) {
 1298   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1299   emit_arith(0x13, 0xC0, dst, src);
 1300 }
 1301 
 1302 void Assembler::addl(Address dst, int32_t imm32) {
 1303   InstructionMark im(this);
 1304   prefix(dst);
 1305   emit_arith_operand(0x81, rax, dst, imm32);
 1306 }
 1307 
 1308 void Assembler::addb(Address dst, int imm8) {
 1309   InstructionMark im(this);
 1310   prefix(dst);
 1311   emit_int8((unsigned char)0x80);
 1312   emit_operand(rax, dst, 1);
 1313   emit_int8(imm8);
 1314 }
 1315 
 1316 void Assembler::addw(Register dst, Register src) {
 1317   emit_int8(0x66);
 1318   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1319   emit_arith(0x03, 0xC0, dst, src);
 1320 }
 1321 
 1322 void Assembler::addw(Address dst, int imm16) {
 1323   InstructionMark im(this);
 1324   emit_int8(0x66);
 1325   prefix(dst);
 1326   emit_int8((unsigned char)0x81);
 1327   emit_operand(rax, dst, 2);
 1328   emit_int16(imm16);
 1329 }
 1330 
 1331 void Assembler::addl(Address dst, Register src) {
 1332   InstructionMark im(this);
 1333   prefix(dst, src);
 1334   emit_int8(0x01);
 1335   emit_operand(src, dst, 0);
 1336 }
 1337 
 1338 void Assembler::addl(Register dst, int32_t imm32) {
 1339   prefix(dst);
 1340   emit_arith(0x81, 0xC0, dst, imm32);
 1341 }
 1342 
 1343 void Assembler::addl(Register dst, Address src) {
 1344   InstructionMark im(this);
 1345   prefix(src, dst);
 1346   emit_int8(0x03);
 1347   emit_operand(dst, src, 0);
 1348 }
 1349 
 1350 void Assembler::addl(Register dst, Register src) {
 1351   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1352   emit_arith(0x03, 0xC0, dst, src);
 1353 }
 1354 
 1355 void Assembler::addr_nop_4() {
 1356   assert(UseAddressNop, "no CPU support");
 1357   // 4 bytes: NOP DWORD PTR [EAX+0]
 1358   emit_int32(0x0F,
 1359              0x1F,
 1360              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1361              0);   // 8-bits offset (1 byte)
 1362 }
 1363 
 1364 void Assembler::addr_nop_5() {
 1365   assert(UseAddressNop, "no CPU support");
 1366   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1367   emit_int32(0x0F,
 1368              0x1F,
 1369              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1370              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1371   emit_int8(0);     // 8-bits offset (1 byte)
 1372 }
 1373 
 1374 void Assembler::addr_nop_7() {
 1375   assert(UseAddressNop, "no CPU support");
 1376   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1377   emit_int24(0x0F,
 1378              0x1F,
 1379              (unsigned char)0x80);
 1380                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1381   emit_int32(0);   // 32-bits offset (4 bytes)
 1382 }
 1383 
 1384 void Assembler::addr_nop_8() {
 1385   assert(UseAddressNop, "no CPU support");
 1386   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1387   emit_int32(0x0F,
 1388              0x1F,
 1389              (unsigned char)0x84,
 1390                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1391              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1392   emit_int32(0);    // 32-bits offset (4 bytes)
 1393 }
 1394 
 1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1397   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1398   attributes.set_rex_vex_w_reverted();
 1399   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1400   emit_int16(0x58, (0xC0 | encode));
 1401 }
 1402 
 1403 void Assembler::addsd(XMMRegister dst, Address src) {
 1404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1405   InstructionMark im(this);
 1406   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1407   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1408   attributes.set_rex_vex_w_reverted();
 1409   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1410   emit_int8(0x58);
 1411   emit_operand(dst, src, 0);
 1412 }
 1413 
 1414 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1415   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1416   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1417   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1418   emit_int16(0x58, (0xC0 | encode));
 1419 }
 1420 
 1421 void Assembler::addss(XMMRegister dst, Address src) {
 1422   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1423   InstructionMark im(this);
 1424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1425   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1426   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1427   emit_int8(0x58);
 1428   emit_operand(dst, src, 0);
 1429 }
 1430 
 1431 void Assembler::aesdec(XMMRegister dst, Address src) {
 1432   assert(VM_Version::supports_aes(), "");
 1433   InstructionMark im(this);
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int8((unsigned char)0xDE);
 1437   emit_operand(dst, src, 0);
 1438 }
 1439 
 1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1441   assert(VM_Version::supports_aes(), "");
 1442   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1443   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1448   assert(VM_Version::supports_avx512_vaes(), "");
 1449   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1450   attributes.set_is_evex_instruction();
 1451   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1453 }
 1454 
 1455 
 1456 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionMark im(this);
 1459   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1460   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1461   emit_int8((unsigned char)0xDF);
 1462   emit_operand(dst, src, 0);
 1463 }
 1464 
 1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1466   assert(VM_Version::supports_aes(), "");
 1467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1468   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1469   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1470 }
 1471 
 1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1473   assert(VM_Version::supports_avx512_vaes(), "");
 1474   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1475   attributes.set_is_evex_instruction();
 1476   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1477   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1478 }
 1479 
 1480 void Assembler::aesenc(XMMRegister dst, Address src) {
 1481   assert(VM_Version::supports_aes(), "");
 1482   InstructionMark im(this);
 1483   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1484   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1485   emit_int8((unsigned char)0xDC);
 1486   emit_operand(dst, src, 0);
 1487 }
 1488 
 1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1490   assert(VM_Version::supports_aes(), "");
 1491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1492   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1493   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1494 }
 1495 
 1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1497   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1499   attributes.set_is_evex_instruction();
 1500   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1501   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1502 }
 1503 
 1504 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1505   assert(VM_Version::supports_aes(), "");
 1506   InstructionMark im(this);
 1507   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1508   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1509   emit_int8((unsigned char)0xDD);
 1510   emit_operand(dst, src, 0);
 1511 }
 1512 
 1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1514   assert(VM_Version::supports_aes(), "");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1521   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1522   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1523   attributes.set_is_evex_instruction();
 1524   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1526 }
 1527 
 1528 void Assembler::andb(Address dst, Register src) {
 1529   InstructionMark im(this);
 1530   prefix(dst, src, true);
 1531   emit_int8(0x20);
 1532   emit_operand(src, dst, 0);
 1533 }
 1534 
 1535 void Assembler::andw(Register dst, Register src) {
 1536   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1537   emit_arith(0x23, 0xC0, dst, src);
 1538 }
 1539 
 1540 void Assembler::andl(Address dst, int32_t imm32) {
 1541   InstructionMark im(this);
 1542   prefix(dst);
 1543   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1544 }
 1545 
 1546 void Assembler::andl(Register dst, int32_t imm32) {
 1547   prefix(dst);
 1548   emit_arith(0x81, 0xE0, dst, imm32);
 1549 }
 1550 
 1551 void Assembler::andl(Address dst, Register src) {
 1552   InstructionMark im(this);
 1553   prefix(dst, src);
 1554   emit_int8(0x21);
 1555   emit_operand(src, dst, 0);
 1556 }
 1557 
 1558 void Assembler::andl(Register dst, Address src) {
 1559   InstructionMark im(this);
 1560   prefix(src, dst);
 1561   emit_int8(0x23);
 1562   emit_operand(dst, src, 0);
 1563 }
 1564 
 1565 void Assembler::andl(Register dst, Register src) {
 1566   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1567   emit_arith(0x23, 0xC0, dst, src);
 1568 }
 1569 
 1570 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1571   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1573   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1574   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1575 }
 1576 
 1577 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1578   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1579   InstructionMark im(this);
 1580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1581   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1582   emit_int8((unsigned char)0xF2);
 1583   emit_operand(dst, src2, 0);
 1584 }
 1585 
 1586 void Assembler::bsfl(Register dst, Register src) {
 1587   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1588   emit_int24(0x0F,
 1589              (unsigned char)0xBC,
 1590              0xC0 | encode);
 1591 }
 1592 
 1593 void Assembler::bsrl(Register dst, Register src) {
 1594   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1595   emit_int24(0x0F,
 1596              (unsigned char)0xBD,
 1597              0xC0 | encode);
 1598 }
 1599 
 1600 void Assembler::bswapl(Register reg) { // bswap
 1601   int encode = prefix_and_encode(reg->encoding());
 1602   emit_int16(0x0F, (0xC8 | encode));
 1603 }
 1604 
 1605 void Assembler::blsil(Register dst, Register src) {
 1606   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1607   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1608   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1609   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1610 }
 1611 
 1612 void Assembler::blsil(Register dst, Address src) {
 1613   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1614   InstructionMark im(this);
 1615   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1616   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1617   emit_int8((unsigned char)0xF3);
 1618   emit_operand(rbx, src, 0);
 1619 }
 1620 
 1621 void Assembler::blsmskl(Register dst, Register src) {
 1622   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1623   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1624   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1625   emit_int16((unsigned char)0xF3,
 1626              0xC0 | encode);
 1627 }
 1628 
 1629 void Assembler::blsmskl(Register dst, Address src) {
 1630   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1631   InstructionMark im(this);
 1632   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1633   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1634   emit_int8((unsigned char)0xF3);
 1635   emit_operand(rdx, src, 0);
 1636 }
 1637 
 1638 void Assembler::blsrl(Register dst, Register src) {
 1639   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1640   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1641   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1642   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1643 }
 1644 
 1645 void Assembler::blsrl(Register dst, Address src) {
 1646   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1647   InstructionMark im(this);
 1648   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1649   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1650   emit_int8((unsigned char)0xF3);
 1651   emit_operand(rcx, src, 0);
 1652 }
 1653 
 1654 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1655   // suspect disp32 is always good
 1656   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1657 
 1658   if (L.is_bound()) {
 1659     const int long_size = 5;
 1660     int offs = (int)( target(L) - pc() );
 1661     assert(offs <= 0, "assembler error");
 1662     InstructionMark im(this);
 1663     // 1110 1000 #32-bit disp
 1664     emit_int8((unsigned char)0xE8);
 1665     emit_data(offs - long_size, rtype, operand);
 1666   } else {
 1667     InstructionMark im(this);
 1668     // 1110 1000 #32-bit disp
 1669     L.add_patch_at(code(), locator());
 1670 
 1671     emit_int8((unsigned char)0xE8);
 1672     emit_data(int(0), rtype, operand);
 1673   }
 1674 }
 1675 
 1676 void Assembler::call(Register dst) {
 1677   int encode = prefix_and_encode(dst->encoding());
 1678   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1679 }
 1680 
 1681 
 1682 void Assembler::call(Address adr) {
 1683   InstructionMark im(this);
 1684   prefix(adr);
 1685   emit_int8((unsigned char)0xFF);
 1686   emit_operand(rdx, adr, 0);
 1687 }
 1688 
 1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1690   InstructionMark im(this);
 1691   emit_int8((unsigned char)0xE8);
 1692   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1693   // Entry is NULL in case of a scratch emit.
 1694   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1695   // Technically, should use call32_operand, but this format is
 1696   // implied by the fact that we're emitting a call instruction.
 1697 
 1698   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1699   emit_data((int) disp, rspec, operand);
 1700 }
 1701 
 1702 void Assembler::cdql() {
 1703   emit_int8((unsigned char)0x99);
 1704 }
 1705 
 1706 void Assembler::cld() {
 1707   emit_int8((unsigned char)0xFC);
 1708 }
 1709 
 1710 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1711   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1712   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1713   emit_int24(0x0F,
 1714              0x40 | cc,
 1715              0xC0 | encode);
 1716 }
 1717 
 1718 
 1719 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1720   InstructionMark im(this);
 1721   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1722   prefix(src, dst);
 1723   emit_int16(0x0F, (0x40 | cc));
 1724   emit_operand(dst, src, 0);
 1725 }
 1726 
 1727 void Assembler::cmpb(Address dst, int imm8) {
 1728   InstructionMark im(this);
 1729   prefix(dst);
 1730   emit_int8((unsigned char)0x80);
 1731   emit_operand(rdi, dst, 1);
 1732   emit_int8(imm8);
 1733 }
 1734 
 1735 void Assembler::cmpl(Address dst, int32_t imm32) {
 1736   InstructionMark im(this);
 1737   prefix(dst);
 1738   emit_arith_operand(0x81, as_Register(7), dst, imm32);
 1739 }
 1740 
 1741 void Assembler::cmpl(Register dst, int32_t imm32) {
 1742   prefix(dst);
 1743   emit_arith(0x81, 0xF8, dst, imm32);
 1744 }
 1745 
 1746 void Assembler::cmpl(Register dst, Register src) {
 1747   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1748   emit_arith(0x3B, 0xC0, dst, src);
 1749 }
 1750 
 1751 void Assembler::cmpl(Register dst, Address src) {
 1752   InstructionMark im(this);
 1753   prefix(src, dst);
 1754   emit_int8(0x3B);
 1755   emit_operand(dst, src, 0);
 1756 }
 1757 
 1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
 1759   InstructionMark im(this);
 1760   prefix(dst);
 1761   emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
 1762 }
 1763 
 1764 void Assembler::cmpw(Address dst, int imm16) {
 1765   InstructionMark im(this);
 1766   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1767   emit_int16(0x66, (unsigned char)0x81);
 1768   emit_operand(rdi, dst, 2);
 1769   emit_int16(imm16);
 1770 }
 1771 
 1772 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1773 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1774 // The ZF is set if the compared values were equal, and cleared otherwise.
 1775 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1776   InstructionMark im(this);
 1777   prefix(adr, reg);
 1778   emit_int16(0x0F, (unsigned char)0xB1);
 1779   emit_operand(reg, adr, 0);
 1780 }
 1781 
 1782 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1783   InstructionMark im(this);
 1784   size_prefix();
 1785   prefix(adr, reg);
 1786   emit_int16(0x0F, (unsigned char)0xB1);
 1787   emit_operand(reg, adr, 0);
 1788 }
 1789 
 1790 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1791 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1792 // The ZF is set if the compared values were equal, and cleared otherwise.
 1793 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1794   InstructionMark im(this);
 1795   prefix(adr, reg, true);
 1796   emit_int16(0x0F, (unsigned char)0xB0);
 1797   emit_operand(reg, adr, 0);
 1798 }
 1799 
 1800 void Assembler::comisd(XMMRegister dst, Address src) {
 1801   // NOTE: dbx seems to decode this as comiss even though the
 1802   // 0x66 is there. Strangely ucomisd comes out correct
 1803   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1804   InstructionMark im(this);
 1805   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1806   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1807   attributes.set_rex_vex_w_reverted();
 1808   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1809   emit_int8(0x2F);
 1810   emit_operand(dst, src, 0);
 1811 }
 1812 
 1813 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1814   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1815   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1816   attributes.set_rex_vex_w_reverted();
 1817   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1818   emit_int16(0x2F, (0xC0 | encode));
 1819 }
 1820 
 1821 void Assembler::comiss(XMMRegister dst, Address src) {
 1822   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1823   InstructionMark im(this);
 1824   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1825   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1826   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1827   emit_int8(0x2F);
 1828   emit_operand(dst, src, 0);
 1829 }
 1830 
 1831 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1832   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1833   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1835   emit_int16(0x2F, (0xC0 | encode));
 1836 }
 1837 
 1838 void Assembler::cpuid() {
 1839   emit_int16(0x0F, (unsigned char)0xA2);
 1840 }
 1841 
 1842 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1843 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1844 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1845 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1846 //
 1847 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1848 //
 1849 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1850 //
 1851 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1852 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1853   assert(VM_Version::supports_sse4_2(), "");
 1854   int8_t w = 0x01;
 1855   Prefix p = Prefix_EMPTY;
 1856 
 1857   emit_int8((unsigned char)0xF2);
 1858   switch (sizeInBytes) {
 1859   case 1:
 1860     w = 0;
 1861     break;
 1862   case 2:
 1863   case 4:
 1864     break;
 1865   LP64_ONLY(case 8:)
 1866     // This instruction is not valid in 32 bits
 1867     // Note:
 1868     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1869     //
 1870     // Page B - 72   Vol. 2C says
 1871     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1872     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1873     //                                                                            F0!!!
 1874     // while 3 - 208 Vol. 2A
 1875     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1876     //
 1877     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1878     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1879     p = REX_W;
 1880     break;
 1881   default:
 1882     assert(0, "Unsupported value for a sizeInBytes argument");
 1883     break;
 1884   }
 1885   LP64_ONLY(prefix(crc, v, p);)
 1886   emit_int32(0x0F,
 1887              0x38,
 1888              0xF0 | w,
 1889              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1890 }
 1891 
 1892 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1893   assert(VM_Version::supports_sse4_2(), "");
 1894   InstructionMark im(this);
 1895   int8_t w = 0x01;
 1896   Prefix p = Prefix_EMPTY;
 1897 
 1898   emit_int8((int8_t)0xF2);
 1899   switch (sizeInBytes) {
 1900   case 1:
 1901     w = 0;
 1902     break;
 1903   case 2:
 1904   case 4:
 1905     break;
 1906   LP64_ONLY(case 8:)
 1907     // This instruction is not valid in 32 bits
 1908     p = REX_W;
 1909     break;
 1910   default:
 1911     assert(0, "Unsupported value for a sizeInBytes argument");
 1912     break;
 1913   }
 1914   LP64_ONLY(prefix(crc, adr, p);)
 1915   emit_int24(0x0F, 0x38, (0xF0 | w));
 1916   emit_operand(crc, adr, 0);
 1917 }
 1918 
 1919 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1920   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1922   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1923   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1924 }
 1925 
 1926 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1927   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1928   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1929   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1930   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1931 }
 1932 
 1933 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 1934   assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
 1935   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1936   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1937   emit_int24(0x1D, (0xC0 | encode), imm8);
 1938 }
 1939 
 1940 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
 1941   assert(VM_Version::supports_avx512vl(), "");
 1942   InstructionMark im(this);
 1943   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 1944   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
 1945   attributes.reset_is_clear_context();
 1946   attributes.set_embedded_opmask_register_specifier(mask);
 1947   attributes.set_is_evex_instruction();
 1948   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1949   emit_int8(0x1D);
 1950   emit_operand(src, dst, 1);
 1951   emit_int8(imm8);
 1952 }
 1953 
 1954 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1955   assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
 1956   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
 1957   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1958   emit_int16(0x13, (0xC0 | encode));
 1959 }
 1960 
 1961 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1962   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1964   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1965   emit_int16(0x5B, (0xC0 | encode));
 1966 }
 1967 
 1968 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1969   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1970   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1971   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1972   emit_int16(0x5B, (0xC0 | encode));
 1973 }
 1974 
 1975 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1977   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1978   attributes.set_rex_vex_w_reverted();
 1979   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1980   emit_int16(0x5A, (0xC0 | encode));
 1981 }
 1982 
 1983 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1984   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1985   InstructionMark im(this);
 1986   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1987   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1988   attributes.set_rex_vex_w_reverted();
 1989   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1990   emit_int8(0x5A);
 1991   emit_operand(dst, src, 0);
 1992 }
 1993 
 1994 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1996   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1997   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1998   emit_int16(0x2A, (0xC0 | encode));
 1999 }
 2000 
 2001 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 2002   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2003   InstructionMark im(this);
 2004   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2005   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2006   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2007   emit_int8(0x2A);
 2008   emit_operand(dst, src, 0);
 2009 }
 2010 
 2011 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 2012   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2013   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2014   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2015   emit_int16(0x2A, (0xC0 | encode));
 2016 }
 2017 
 2018 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 2019   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2020   InstructionMark im(this);
 2021   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2022   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2023   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2024   emit_int8(0x2A);
 2025   emit_operand(dst, src, 0);
 2026 }
 2027 
 2028 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 2029   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2030   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2031   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2032   emit_int16(0x2A, (0xC0 | encode));
 2033 }
 2034 
 2035 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 2036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2038   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2039   emit_int16(0x5A, (0xC0 | encode));
 2040 }
 2041 
 2042 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 2043   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2044   InstructionMark im(this);
 2045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2046   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2047   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2048   emit_int8(0x5A);
 2049   emit_operand(dst, src, 0);
 2050 }
 2051 
 2052 
 2053 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 2054   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2055   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2056   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2057   emit_int16(0x2C, (0xC0 | encode));
 2058 }
 2059 
 2060 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 2061   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2063   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2064   emit_int16(0x2D, (0xC0 | encode));
 2065 }
 2066 
 2067 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2068   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2070   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2071   emit_int16(0x2C, (0xC0 | encode));
 2072 }
 2073 
 2074 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2075   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2076   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2077   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2078   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2079   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2080 }
 2081 
 2082 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2083   assert(VM_Version::supports_ssse3(), "");
 2084   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2085   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2086   emit_int16(0x1C, (0xC0 | encode));
 2087 }
 2088 
 2089 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2090   assert(VM_Version::supports_ssse3(), "");
 2091   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2092   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2093   emit_int16(0x1D, (0xC0 | encode));
 2094 }
 2095 
 2096 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2097   assert(VM_Version::supports_ssse3(), "");
 2098   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2099   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x1E, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2105          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2106          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2107   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2108   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2109   emit_int16(0x1C, (0xC0 | encode));
 2110 }
 2111 
 2112 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2113   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2114          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2115          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2116   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2117   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2118   emit_int16(0x1D, (0xC0 | encode));
 2119 }
 2120 
 2121 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2122   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2123   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2124   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2125   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2126   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2127   emit_int16(0x1E, (0xC0 | encode));
 2128 }
 2129 
 2130 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2131   assert(UseAVX > 2, "");
 2132   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2133   attributes.set_is_evex_instruction();
 2134   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2135   emit_int16(0x1F, (0xC0 | encode));
 2136 }
 2137 
 2138 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2139   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2140   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2141   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2142   emit_int16(0x5A, (0xC0 | encode));
 2143 }
 2144 
 2145 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2146   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2147   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2149   attributes.set_rex_vex_w_reverted();
 2150   emit_int16(0x5A, (0xC0 | encode));
 2151 }
 2152 
 2153 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2154   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2155   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2156   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2157   emit_int16(0x5B, (0xC0 | encode));
 2158 }
 2159 
 2160 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2161   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2162   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2163   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2164   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2165 }
 2166 
 2167 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2168   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2169   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2170   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2171   emit_int16(0x5B, (0xC0 | encode));
 2172 }
 2173 
 2174 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2175   assert(VM_Version::supports_avx512dq(), "");
 2176   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2177   attributes.set_is_evex_instruction();
 2178   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2179   emit_int16(0x7A, (0xC0 | encode));
 2180 }
 2181 
 2182 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2183   assert(VM_Version::supports_avx512dq(), "");
 2184   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2185   attributes.set_is_evex_instruction();
 2186   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2187   emit_int16(0x7B, (0xC0 | encode));
 2188 }
 2189 
 2190 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2191   assert(VM_Version::supports_avx512dq(), "");
 2192   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2193   attributes.set_is_evex_instruction();
 2194   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2195   emit_int16(0x5B, (0xC0 | encode));
 2196 }
 2197 
 2198 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2199   assert(VM_Version::supports_avx512dq(), "");
 2200   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2201   attributes.set_is_evex_instruction();
 2202   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2203   emit_int16(0x7A, (0xC0 | encode));
 2204 }
 2205 
 2206 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2207   assert(VM_Version::supports_avx512dq(), "");
 2208   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2209   attributes.set_is_evex_instruction();
 2210   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2211   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2212 }
 2213 
 2214 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2215   assert(VM_Version::supports_avx512bw(), "");
 2216   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2217   attributes.set_is_evex_instruction();
 2218   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2219   emit_int16(0x30, (0xC0 | encode));
 2220 }
 2221 
 2222 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2223   assert(UseAVX > 2, "");
 2224   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2225   attributes.set_is_evex_instruction();
 2226   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2227   emit_int16(0x33, (0xC0 | encode));
 2228 }
 2229 
 2230 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2231   assert(UseAVX > 2, "");
 2232   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2233   attributes.set_is_evex_instruction();
 2234   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2235   emit_int16(0x31, (0xC0 | encode));
 2236 }
 2237 
 2238 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2239   assert(UseAVX > 2, "");
 2240   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2241   attributes.set_is_evex_instruction();
 2242   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2243   emit_int16(0x35, (0xC0 | encode));
 2244 }
 2245 
 2246 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2247   assert(UseAVX > 2, "");
 2248   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2249   attributes.set_is_evex_instruction();
 2250   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2251   emit_int16(0x32, (0xC0 | encode));
 2252 }
 2253 
 2254 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2255   assert(UseAVX > 2, "");
 2256   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2257   attributes.set_is_evex_instruction();
 2258   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2259   emit_int16(0x34, (0xC0 | encode));
 2260 }
 2261 
 2262 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2263   assert(UseAVX > 2, "");
 2264   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2265   attributes.set_is_evex_instruction();
 2266   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2267   emit_int16(0x25, (0xC0 | encode));
 2268 }
 2269 
 2270 void Assembler::decl(Address dst) {
 2271   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2272   InstructionMark im(this);
 2273   prefix(dst);
 2274   emit_int8((unsigned char)0xFF);
 2275   emit_operand(rcx, dst, 0);
 2276 }
 2277 
 2278 void Assembler::divsd(XMMRegister dst, Address src) {
 2279   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2280   InstructionMark im(this);
 2281   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2282   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2283   attributes.set_rex_vex_w_reverted();
 2284   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2285   emit_int8(0x5E);
 2286   emit_operand(dst, src, 0);
 2287 }
 2288 
 2289 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2290   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2291   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2292   attributes.set_rex_vex_w_reverted();
 2293   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2294   emit_int16(0x5E, (0xC0 | encode));
 2295 }
 2296 
 2297 void Assembler::divss(XMMRegister dst, Address src) {
 2298   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2299   InstructionMark im(this);
 2300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2301   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2302   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2303   emit_int8(0x5E);
 2304   emit_operand(dst, src, 0);
 2305 }
 2306 
 2307 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2308   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2310   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2311   emit_int16(0x5E, (0xC0 | encode));
 2312 }
 2313 
 2314 void Assembler::hlt() {
 2315   emit_int8((unsigned char)0xF4);
 2316 }
 2317 
 2318 void Assembler::idivl(Register src) {
 2319   int encode = prefix_and_encode(src->encoding());
 2320   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2321 }
 2322 
 2323 void Assembler::divl(Register src) { // Unsigned
 2324   int encode = prefix_and_encode(src->encoding());
 2325   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2326 }
 2327 
 2328 void Assembler::imull(Register src) {
 2329   int encode = prefix_and_encode(src->encoding());
 2330   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2331 }
 2332 
 2333 void Assembler::imull(Register dst, Register src) {
 2334   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2335   emit_int24(0x0F,
 2336              (unsigned char)0xAF,
 2337              (0xC0 | encode));
 2338 }
 2339 
 2340 void Assembler::imull(Register dst, Address src, int32_t value) {
 2341   InstructionMark im(this);
 2342   prefix(src, dst);
 2343   if (is8bit(value)) {
 2344     emit_int8((unsigned char)0x6B);
 2345     emit_operand(dst, src, 1);
 2346     emit_int8(value);
 2347   } else {
 2348     emit_int8((unsigned char)0x69);
 2349     emit_operand(dst, src, 4);
 2350     emit_int32(value);
 2351   }
 2352 }
 2353 
 2354 void Assembler::imull(Register dst, Register src, int value) {
 2355   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2356   if (is8bit(value)) {
 2357     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2358   } else {
 2359     emit_int16(0x69, (0xC0 | encode));
 2360     emit_int32(value);
 2361   }
 2362 }
 2363 
 2364 void Assembler::imull(Register dst, Address src) {
 2365   InstructionMark im(this);
 2366   prefix(src, dst);
 2367   emit_int16(0x0F, (unsigned char)0xAF);
 2368   emit_operand(dst, src, 0);
 2369 }
 2370 
 2371 
 2372 void Assembler::incl(Address dst) {
 2373   // Don't use it directly. Use MacroAssembler::increment() instead.
 2374   InstructionMark im(this);
 2375   prefix(dst);
 2376   emit_int8((unsigned char)0xFF);
 2377   emit_operand(rax, dst, 0);
 2378 }
 2379 
 2380 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2381   InstructionMark im(this);
 2382   assert((0 <= cc) && (cc < 16), "illegal cc");
 2383   if (L.is_bound()) {
 2384     address dst = target(L);
 2385     assert(dst != NULL, "jcc most probably wrong");
 2386 
 2387     const int short_size = 2;
 2388     const int long_size = 6;
 2389     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2390     if (maybe_short && is8bit(offs - short_size)) {
 2391       // 0111 tttn #8-bit disp
 2392       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2393     } else {
 2394       // 0000 1111 1000 tttn #32-bit disp
 2395       assert(is_simm32(offs - long_size),
 2396              "must be 32bit offset (call4)");
 2397       emit_int16(0x0F, (0x80 | cc));
 2398       emit_int32(offs - long_size);
 2399     }
 2400   } else {
 2401     // Note: could eliminate cond. jumps to this jump if condition
 2402     //       is the same however, seems to be rather unlikely case.
 2403     // Note: use jccb() if label to be bound is very close to get
 2404     //       an 8-bit displacement
 2405     L.add_patch_at(code(), locator());
 2406     emit_int16(0x0F, (0x80 | cc));
 2407     emit_int32(0);
 2408   }
 2409 }
 2410 
 2411 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2412   if (L.is_bound()) {
 2413     const int short_size = 2;
 2414     address entry = target(L);
 2415 #ifdef ASSERT
 2416     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2417     intptr_t delta = short_branch_delta();
 2418     if (delta != 0) {
 2419       dist += (dist < 0 ? (-delta) :delta);
 2420     }
 2421     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2422 #endif
 2423     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2424     // 0111 tttn #8-bit disp
 2425     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2426   } else {
 2427     InstructionMark im(this);
 2428     L.add_patch_at(code(), locator(), file, line);
 2429     emit_int16(0x70 | cc, 0);
 2430   }
 2431 }
 2432 
 2433 void Assembler::jmp(Address adr) {
 2434   InstructionMark im(this);
 2435   prefix(adr);
 2436   emit_int8((unsigned char)0xFF);
 2437   emit_operand(rsp, adr, 0);
 2438 }
 2439 
 2440 void Assembler::jmp(Label& L, bool maybe_short) {
 2441   if (L.is_bound()) {
 2442     address entry = target(L);
 2443     assert(entry != NULL, "jmp most probably wrong");
 2444     InstructionMark im(this);
 2445     const int short_size = 2;
 2446     const int long_size = 5;
 2447     intptr_t offs = entry - pc();
 2448     if (maybe_short && is8bit(offs - short_size)) {
 2449       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2450     } else {
 2451       emit_int8((unsigned char)0xE9);
 2452       emit_int32(offs - long_size);
 2453     }
 2454   } else {
 2455     // By default, forward jumps are always 32-bit displacements, since
 2456     // we can't yet know where the label will be bound.  If you're sure that
 2457     // the forward jump will not run beyond 256 bytes, use jmpb to
 2458     // force an 8-bit displacement.
 2459     InstructionMark im(this);
 2460     L.add_patch_at(code(), locator());
 2461     emit_int8((unsigned char)0xE9);
 2462     emit_int32(0);
 2463   }
 2464 }
 2465 
 2466 void Assembler::jmp(Register entry) {
 2467   int encode = prefix_and_encode(entry->encoding());
 2468   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2469 }
 2470 
 2471 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2472   InstructionMark im(this);
 2473   emit_int8((unsigned char)0xE9);
 2474   assert(dest != NULL, "must have a target");
 2475   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2476   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2477   emit_data(disp, rspec.reloc(), call32_operand);
 2478 }
 2479 
 2480 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2481   if (L.is_bound()) {
 2482     const int short_size = 2;
 2483     address entry = target(L);
 2484     assert(entry != NULL, "jmp most probably wrong");
 2485 #ifdef ASSERT
 2486     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2487     intptr_t delta = short_branch_delta();
 2488     if (delta != 0) {
 2489       dist += (dist < 0 ? (-delta) :delta);
 2490     }
 2491     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2492 #endif
 2493     intptr_t offs = entry - pc();
 2494     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2495   } else {
 2496     InstructionMark im(this);
 2497     L.add_patch_at(code(), locator(), file, line);
 2498     emit_int16((unsigned char)0xEB, 0);
 2499   }
 2500 }
 2501 
 2502 void Assembler::ldmxcsr( Address src) {
 2503   if (UseAVX > 0 ) {
 2504     InstructionMark im(this);
 2505     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2506     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2507     emit_int8((unsigned char)0xAE);
 2508     emit_operand(as_Register(2), src, 0);
 2509   } else {
 2510     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2511     InstructionMark im(this);
 2512     prefix(src);
 2513     emit_int16(0x0F, (unsigned char)0xAE);
 2514     emit_operand(as_Register(2), src, 0);
 2515   }
 2516 }
 2517 
 2518 void Assembler::leal(Register dst, Address src) {
 2519   InstructionMark im(this);
 2520   prefix(src, dst);
 2521   emit_int8((unsigned char)0x8D);
 2522   emit_operand(dst, src, 0);
 2523 }
 2524 
 2525 void Assembler::lfence() {
 2526   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2527 }
 2528 
 2529 void Assembler::lock() {
 2530   emit_int8((unsigned char)0xF0);
 2531 }
 2532 
 2533 void Assembler::size_prefix() {
 2534   emit_int8(0x66);
 2535 }
 2536 
 2537 void Assembler::lzcntl(Register dst, Register src) {
 2538   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2539   emit_int8((unsigned char)0xF3);
 2540   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2541   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2542 }
 2543 
 2544 void Assembler::lzcntl(Register dst, Address src) {
 2545   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2546   InstructionMark im(this);
 2547   emit_int8((unsigned char)0xF3);
 2548   prefix(src, dst);
 2549   emit_int16(0x0F, (unsigned char)0xBD);
 2550   emit_operand(dst, src, 0);
 2551 }
 2552 
 2553 // Emit mfence instruction
 2554 void Assembler::mfence() {
 2555   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2556   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2557 }
 2558 
 2559 // Emit sfence instruction
 2560 void Assembler::sfence() {
 2561   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2562   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2563 }
 2564 
 2565 void Assembler::mov(Register dst, Register src) {
 2566   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2567 }
 2568 
 2569 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2570   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2571   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2572   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2573   attributes.set_rex_vex_w_reverted();
 2574   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2575   emit_int16(0x28, (0xC0 | encode));
 2576 }
 2577 
 2578 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2579   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2580   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2581   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2582   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2583   emit_int16(0x28, (0xC0 | encode));
 2584 }
 2585 
 2586 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2587   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2589   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2590   emit_int16(0x16, (0xC0 | encode));
 2591 }
 2592 
 2593 void Assembler::movb(Register dst, Address src) {
 2594   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2595   InstructionMark im(this);
 2596   prefix(src, dst, true);
 2597   emit_int8((unsigned char)0x8A);
 2598   emit_operand(dst, src, 0);
 2599 }
 2600 
 2601 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2602   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2603   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2604   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2605   attributes.set_rex_vex_w_reverted();
 2606   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2607   emit_int16(0x12, 0xC0 | encode);
 2608 }
 2609 
 2610 void Assembler::movddup(XMMRegister dst, Address src) {
 2611   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2612   InstructionMark im(this);
 2613   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2614   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2615   attributes.set_rex_vex_w_reverted();
 2616   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2617   emit_int8(0x12);
 2618   emit_operand(dst, src, 0);
 2619 }
 2620 
 2621 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2622   assert(VM_Version::supports_avx(), "");
 2623   InstructionMark im(this);
 2624   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2625   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2626   attributes.set_rex_vex_w_reverted();
 2627   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2628   emit_int8(0x12);
 2629   emit_operand(dst, src, 0);
 2630 }
 2631 
 2632 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2633   assert(VM_Version::supports_avx512dq(), "");
 2634   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2635   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2636   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2637 }
 2638 
 2639 void Assembler::kmovbl(KRegister dst, Register src) {
 2640   assert(VM_Version::supports_avx512dq(), "");
 2641   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2642   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2643   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2644 }
 2645 
 2646 void Assembler::kmovbl(Register dst, KRegister src) {
 2647   assert(VM_Version::supports_avx512dq(), "");
 2648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2649   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2650   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2651 }
 2652 
 2653 void Assembler::kmovwl(KRegister dst, Register src) {
 2654   assert(VM_Version::supports_evex(), "");
 2655   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2656   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2657   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2658 }
 2659 
 2660 void Assembler::kmovwl(Register dst, KRegister src) {
 2661   assert(VM_Version::supports_evex(), "");
 2662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2663   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2664   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2665 }
 2666 
 2667 void Assembler::kmovwl(KRegister dst, Address src) {
 2668   assert(VM_Version::supports_evex(), "");
 2669   InstructionMark im(this);
 2670   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2671   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2672   emit_int8((unsigned char)0x90);
 2673   emit_operand(dst, src, 0);
 2674 }
 2675 
 2676 void Assembler::kmovwl(Address dst, KRegister src) {
 2677   assert(VM_Version::supports_evex(), "");
 2678   InstructionMark im(this);
 2679   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2680   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2681   emit_int8((unsigned char)0x91);
 2682   emit_operand(src, dst, 0);
 2683 }
 2684 
 2685 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2686   assert(VM_Version::supports_evex(), "");
 2687   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2688   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2689   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2690 }
 2691 
 2692 void Assembler::kmovdl(KRegister dst, Register src) {
 2693   assert(VM_Version::supports_avx512bw(), "");
 2694   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2695   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2696   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2697 }
 2698 
 2699 void Assembler::kmovdl(Register dst, KRegister src) {
 2700   assert(VM_Version::supports_avx512bw(), "");
 2701   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2702   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2703   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2704 }
 2705 
 2706 void Assembler::kmovql(KRegister dst, KRegister src) {
 2707   assert(VM_Version::supports_avx512bw(), "");
 2708   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2709   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2710   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2711 }
 2712 
 2713 void Assembler::kmovql(KRegister dst, Address src) {
 2714   assert(VM_Version::supports_avx512bw(), "");
 2715   InstructionMark im(this);
 2716   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2717   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2718   emit_int8((unsigned char)0x90);
 2719   emit_operand(dst, src, 0);
 2720 }
 2721 
 2722 void Assembler::kmovql(Address dst, KRegister src) {
 2723   assert(VM_Version::supports_avx512bw(), "");
 2724   InstructionMark im(this);
 2725   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2726   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2727   emit_int8((unsigned char)0x91);
 2728   emit_operand(src, dst, 0);
 2729 }
 2730 
 2731 void Assembler::kmovql(KRegister dst, Register src) {
 2732   assert(VM_Version::supports_avx512bw(), "");
 2733   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2734   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2735   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2736 }
 2737 
 2738 void Assembler::kmovql(Register dst, KRegister src) {
 2739   assert(VM_Version::supports_avx512bw(), "");
 2740   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2741   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2742   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2743 }
 2744 
 2745 void Assembler::knotwl(KRegister dst, KRegister src) {
 2746   assert(VM_Version::supports_evex(), "");
 2747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2748   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2749   emit_int16(0x44, (0xC0 | encode));
 2750 }
 2751 
 2752 void Assembler::knotbl(KRegister dst, KRegister src) {
 2753   assert(VM_Version::supports_evex(), "");
 2754   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2755   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2756   emit_int16(0x44, (0xC0 | encode));
 2757 }
 2758 
 2759 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2760   assert(VM_Version::supports_avx512dq(), "");
 2761   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2762   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2763   emit_int16(0x45, (0xC0 | encode));
 2764 }
 2765 
 2766 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2767   assert(VM_Version::supports_evex(), "");
 2768   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2769   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2770   emit_int16(0x45, (0xC0 | encode));
 2771 }
 2772 
 2773 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2774   assert(VM_Version::supports_avx512bw(), "");
 2775   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2776   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2777   emit_int16(0x45, (0xC0 | encode));
 2778 }
 2779 
 2780 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2781   assert(VM_Version::supports_avx512bw(), "");
 2782   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2783   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2784   emit_int16(0x45, (0xC0 | encode));
 2785 }
 2786 
 2787 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2788   assert(VM_Version::supports_avx512dq(), "");
 2789   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2790   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2791   emit_int16(0x47, (0xC0 | encode));
 2792 }
 2793 
 2794 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2795   assert(VM_Version::supports_evex(), "");
 2796   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2797   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2798   emit_int16(0x47, (0xC0 | encode));
 2799 }
 2800 
 2801 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2802   assert(VM_Version::supports_avx512bw(), "");
 2803   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2804   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2805   emit_int16(0x47, (0xC0 | encode));
 2806 }
 2807 
 2808 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2809   assert(VM_Version::supports_avx512bw(), "");
 2810   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2811   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2812   emit_int16(0x47, (0xC0 | encode));
 2813 }
 2814 
 2815 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2816   assert(VM_Version::supports_avx512dq(), "");
 2817   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2818   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2819   emit_int16(0x41, (0xC0 | encode));
 2820 }
 2821 
 2822 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2823   assert(VM_Version::supports_evex(), "");
 2824   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2825   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2826   emit_int16(0x41, (0xC0 | encode));
 2827 }
 2828 
 2829 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2830   assert(VM_Version::supports_avx512bw(), "");
 2831   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2832   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2833   emit_int16(0x41, (0xC0 | encode));
 2834 }
 2835 
 2836 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2837   assert(VM_Version::supports_avx512bw(), "");
 2838   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2839   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2840   emit_int16(0x41, (0xC0 | encode));
 2841 }
 2842 
 2843 void Assembler::knotdl(KRegister dst, KRegister src) {
 2844   assert(VM_Version::supports_avx512bw(), "");
 2845   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2846   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2847   emit_int16(0x44, (0xC0 | encode));
 2848 }
 2849 
 2850 void Assembler::knotql(KRegister dst, KRegister src) {
 2851   assert(VM_Version::supports_avx512bw(), "");
 2852   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2853   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2854   emit_int16(0x44, (0xC0 | encode));
 2855 }
 2856 
 2857 // This instruction produces ZF or CF flags
 2858 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2859   assert(VM_Version::supports_avx512dq(), "");
 2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2861   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2862   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2863 }
 2864 
 2865 // This instruction produces ZF or CF flags
 2866 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2867   assert(VM_Version::supports_evex(), "");
 2868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2869   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2870   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2871 }
 2872 
 2873 // This instruction produces ZF or CF flags
 2874 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2875   assert(VM_Version::supports_avx512bw(), "");
 2876   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2877   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2878   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2879 }
 2880 
 2881 // This instruction produces ZF or CF flags
 2882 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2883   assert(VM_Version::supports_avx512bw(), "");
 2884   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2885   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2886   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2887 }
 2888 
 2889 // This instruction produces ZF or CF flags
 2890 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2891   assert(VM_Version::supports_avx512bw(), "");
 2892   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2893   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2894   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2895 }
 2896 
 2897 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2898   assert(VM_Version::supports_avx512bw(), "");
 2899   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2900   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2901   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2902 }
 2903 
 2904 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2905   assert(VM_Version::supports_avx512dq(), "");
 2906   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2907   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2908   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2909 }
 2910 
 2911 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2912   assert(VM_Version::supports_avx512dq(), "");
 2913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2914   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2915   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2916 }
 2917 
 2918 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2919   assert(VM_Version::supports_avx512bw(), "");
 2920   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2921   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2922   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2923 }
 2924 
 2925 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2926   assert(VM_Version::supports_avx512bw(), "");
 2927   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2928   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2929   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2930 }
 2931 
 2932 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2933   assert(VM_Version::supports_avx512dq(), "");
 2934   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2935   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2936   emit_int16(0x46, (0xC0 | encode));
 2937 }
 2938 
 2939 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2940   assert(VM_Version::supports_avx512dq(), "");
 2941   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2942   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2943   emit_int16(0x32, (0xC0 | encode));
 2944   emit_int8(imm8);
 2945 }
 2946 
 2947 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2948   assert(VM_Version::supports_avx512bw(), "");
 2949   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2950   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2951   emit_int16(0x33, (0xC0 | encode));
 2952   emit_int8(imm8);
 2953 }
 2954 
 2955 
 2956 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2957   assert(VM_Version::supports_avx512dq(), "");
 2958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2959   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2960   emit_int16(0x30, (0xC0 | encode));
 2961 }
 2962 
 2963 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2964   assert(VM_Version::supports_evex(), "");
 2965   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2966   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2967   emit_int16(0x30, (0xC0 | encode));
 2968   emit_int8(imm8);
 2969 }
 2970 
 2971 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2972   assert(VM_Version::supports_avx512bw(), "");
 2973   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2974   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2975   emit_int16(0x31, (0xC0 | encode));
 2976   emit_int8(imm8);
 2977 }
 2978 
 2979 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2980   assert(VM_Version::supports_avx512bw(), "");
 2981   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2982   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2983   emit_int16(0x31, (0xC0 | encode));
 2984   emit_int8(imm8);
 2985 }
 2986 
 2987 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 2988   assert(VM_Version::supports_avx512bw(), "");
 2989   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2990   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2991   emit_int16(0x4B, (0xC0 | encode));
 2992 }
 2993 
 2994 void Assembler::movb(Address dst, int imm8) {
 2995   InstructionMark im(this);
 2996    prefix(dst);
 2997   emit_int8((unsigned char)0xC6);
 2998   emit_operand(rax, dst, 1);
 2999   emit_int8(imm8);
 3000 }
 3001 
 3002 
 3003 void Assembler::movb(Address dst, Register src) {
 3004   assert(src->has_byte_register(), "must have byte register");
 3005   InstructionMark im(this);
 3006   prefix(dst, src, true);
 3007   emit_int8((unsigned char)0x88);
 3008   emit_operand(src, dst, 0);
 3009 }
 3010 
 3011 void Assembler::movdl(XMMRegister dst, Register src) {
 3012   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3013   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3014   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3015   emit_int16(0x6E, (0xC0 | encode));
 3016 }
 3017 
 3018 void Assembler::movdl(Register dst, XMMRegister src) {
 3019   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3021   // swap src/dst to get correct prefix
 3022   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3023   emit_int16(0x7E, (0xC0 | encode));
 3024 }
 3025 
 3026 void Assembler::movdl(XMMRegister dst, Address src) {
 3027   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3028   InstructionMark im(this);
 3029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3030   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3031   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3032   emit_int8(0x6E);
 3033   emit_operand(dst, src, 0);
 3034 }
 3035 
 3036 void Assembler::movdl(Address dst, XMMRegister src) {
 3037   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3038   InstructionMark im(this);
 3039   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3040   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3041   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3042   emit_int8(0x7E);
 3043   emit_operand(src, dst, 0);
 3044 }
 3045 
 3046 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 3047   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3048   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3049   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3050   emit_int16(0x6F, (0xC0 | encode));
 3051 }
 3052 
 3053 void Assembler::movdqa(XMMRegister dst, Address src) {
 3054   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3055   InstructionMark im(this);
 3056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3057   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3058   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3059   emit_int8(0x6F);
 3060   emit_operand(dst, src, 0);
 3061 }
 3062 
 3063 void Assembler::movdqu(XMMRegister dst, Address src) {
 3064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3065   InstructionMark im(this);
 3066   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3067   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3068   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3069   emit_int8(0x6F);
 3070   emit_operand(dst, src, 0);
 3071 }
 3072 
 3073 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 3074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3075   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3076   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3077   emit_int16(0x6F, (0xC0 | encode));
 3078 }
 3079 
 3080 void Assembler::movdqu(Address dst, XMMRegister src) {
 3081   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3082   InstructionMark im(this);
 3083   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3084   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3085   attributes.reset_is_clear_context();
 3086   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3087   emit_int8(0x7F);
 3088   emit_operand(src, dst, 0);
 3089 }
 3090 
 3091 // Move Unaligned 256bit Vector
 3092 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 3093   assert(UseAVX > 0, "");
 3094   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3095   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3096   emit_int16(0x6F, (0xC0 | encode));
 3097 }
 3098 
 3099 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3100   assert(UseAVX > 0, "");
 3101   InstructionMark im(this);
 3102   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3103   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3104   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3105   emit_int8(0x6F);
 3106   emit_operand(dst, src, 0);
 3107 }
 3108 
 3109 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3110   assert(UseAVX > 0, "");
 3111   InstructionMark im(this);
 3112   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3113   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3114   attributes.reset_is_clear_context();
 3115   // swap src<->dst for encoding
 3116   assert(src != xnoreg, "sanity");
 3117   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3118   emit_int8(0x7F);
 3119   emit_operand(src, dst, 0);
 3120 }
 3121 
 3122 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3123   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3124   InstructionMark im(this);
 3125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3126   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3127   emit_int8((unsigned char)0x8C);
 3128   emit_operand(dst, src, 0);
 3129 }
 3130 
 3131 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3132   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3133   InstructionMark im(this);
 3134   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3135   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3136   emit_int8((unsigned char)0x8C);
 3137   emit_operand(dst, src, 0);
 3138 }
 3139 
 3140 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3141   assert(UseAVX > 0, "requires some form of AVX");
 3142   InstructionMark im(this);
 3143   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3144   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3145   emit_int8(0x2C);
 3146   emit_operand(dst, src, 0);
 3147 }
 3148 
 3149 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3150   assert(UseAVX > 0, "requires some form of AVX");
 3151   InstructionMark im(this);
 3152   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3153   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3154   emit_int8(0x2D);
 3155   emit_operand(dst, src, 0);
 3156 }
 3157 
 3158 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3159   assert(UseAVX > 0, "");
 3160   InstructionMark im(this);
 3161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3162   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3163   emit_int8(0x2E);
 3164   emit_operand(src, dst, 0);
 3165 }
 3166 
 3167 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3168   assert(UseAVX > 0, "");
 3169   InstructionMark im(this);
 3170   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3171   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3172   emit_int8(0x2F);
 3173   emit_operand(src, dst, 0);
 3174 }
 3175 
 3176 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3177 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3178   assert(VM_Version::supports_avx512vlbw(), "");
 3179   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3180   attributes.set_embedded_opmask_register_specifier(mask);
 3181   attributes.set_is_evex_instruction();
 3182   if (merge) {
 3183     attributes.reset_is_clear_context();
 3184   }
 3185   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3186   emit_int16(0x6F, (0xC0 | encode));
 3187 }
 3188 
 3189 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
 3190   // Unmasked instruction
 3191   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3192 }
 3193 
 3194 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3195   assert(VM_Version::supports_avx512vlbw(), "");
 3196   InstructionMark im(this);
 3197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3198   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3199   attributes.set_embedded_opmask_register_specifier(mask);
 3200   attributes.set_is_evex_instruction();
 3201   if (merge) {
 3202     attributes.reset_is_clear_context();
 3203   }
 3204   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3205   emit_int8(0x6F);
 3206   emit_operand(dst, src, 0);
 3207 }
 3208 
 3209 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
 3210   // Unmasked instruction
 3211   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3212 }
 3213 
 3214 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3215   assert(VM_Version::supports_avx512vlbw(), "");
 3216   assert(src != xnoreg, "sanity");
 3217   InstructionMark im(this);
 3218   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3219   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3220   attributes.set_embedded_opmask_register_specifier(mask);
 3221   attributes.set_is_evex_instruction();
 3222   if (merge) {
 3223     attributes.reset_is_clear_context();
 3224   }
 3225   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3226   emit_int8(0x7F);
 3227   emit_operand(src, dst, 0);
 3228 }
 3229 
 3230 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
 3231   // Unmasked instruction
 3232   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3233 }
 3234 
 3235 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3236   assert(VM_Version::supports_avx512vlbw(), "");
 3237   InstructionMark im(this);
 3238   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3239   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3240   attributes.set_embedded_opmask_register_specifier(mask);
 3241   attributes.set_is_evex_instruction();
 3242   if (merge) {
 3243     attributes.reset_is_clear_context();
 3244   }
 3245   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3246   emit_int8(0x6F);
 3247   emit_operand(dst, src, 0);
 3248 }
 3249 
 3250 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
 3251   // Unmasked instruction
 3252   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3253 }
 3254 
 3255 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3256   assert(VM_Version::supports_avx512vlbw(), "");
 3257   assert(src != xnoreg, "sanity");
 3258   InstructionMark im(this);
 3259   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3260   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3261   attributes.set_embedded_opmask_register_specifier(mask);
 3262   attributes.set_is_evex_instruction();
 3263   if (merge) {
 3264     attributes.reset_is_clear_context();
 3265   }
 3266   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3267   emit_int8(0x7F);
 3268   emit_operand(src, dst, 0);
 3269 }
 3270 
 3271 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3272   // Unmasked instruction
 3273   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3274 }
 3275 
 3276 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3277   assert(VM_Version::supports_evex(), "");
 3278   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3279   attributes.set_embedded_opmask_register_specifier(mask);
 3280   attributes.set_is_evex_instruction();
 3281   if (merge) {
 3282     attributes.reset_is_clear_context();
 3283   }
 3284   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3285   emit_int16(0x6F, (0xC0 | encode));
 3286 }
 3287 
 3288 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3289   // Unmasked instruction
 3290   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3291 }
 3292 
 3293 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3294   assert(VM_Version::supports_evex(), "");
 3295   InstructionMark im(this);
 3296   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3297   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3298   attributes.set_embedded_opmask_register_specifier(mask);
 3299   attributes.set_is_evex_instruction();
 3300   if (merge) {
 3301     attributes.reset_is_clear_context();
 3302   }
 3303   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3304   emit_int8(0x6F);
 3305   emit_operand(dst, src, 0);
 3306 }
 3307 
 3308 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3309   // Unmasked isntruction
 3310   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3311 }
 3312 
 3313 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3314   assert(VM_Version::supports_evex(), "");
 3315   assert(src != xnoreg, "sanity");
 3316   InstructionMark im(this);
 3317   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3318   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3319   attributes.set_embedded_opmask_register_specifier(mask);
 3320   attributes.set_is_evex_instruction();
 3321   if (merge) {
 3322     attributes.reset_is_clear_context();
 3323   }
 3324   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3325   emit_int8(0x7F);
 3326   emit_operand(src, dst, 0);
 3327 }
 3328 
 3329 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3330   // Unmasked instruction
 3331   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3332 }
 3333 
 3334 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3335   assert(VM_Version::supports_evex(), "");
 3336   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3337   attributes.set_embedded_opmask_register_specifier(mask);
 3338   attributes.set_is_evex_instruction();
 3339   if (merge) {
 3340     attributes.reset_is_clear_context();
 3341   }
 3342   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3343   emit_int16(0x6F, (0xC0 | encode));
 3344 }
 3345 
 3346 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3347   // Unmasked instruction
 3348   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3349 }
 3350 
 3351 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3352   assert(VM_Version::supports_evex(), "");
 3353   InstructionMark im(this);
 3354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3355   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3356   attributes.set_embedded_opmask_register_specifier(mask);
 3357   attributes.set_is_evex_instruction();
 3358   if (merge) {
 3359     attributes.reset_is_clear_context();
 3360   }
 3361   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3362   emit_int8(0x6F);
 3363   emit_operand(dst, src, 0);
 3364 }
 3365 
 3366 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3367   // Unmasked instruction
 3368   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3369 }
 3370 
 3371 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3372   assert(VM_Version::supports_evex(), "");
 3373   assert(src != xnoreg, "sanity");
 3374   InstructionMark im(this);
 3375   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3376   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3377   attributes.set_embedded_opmask_register_specifier(mask);
 3378   if (merge) {
 3379     attributes.reset_is_clear_context();
 3380   }
 3381   attributes.set_is_evex_instruction();
 3382   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3383   emit_int8(0x7F);
 3384   emit_operand(src, dst, 0);
 3385 }
 3386 
 3387 // Uses zero extension on 64bit
 3388 
 3389 void Assembler::movl(Register dst, int32_t imm32) {
 3390   int encode = prefix_and_encode(dst->encoding());
 3391   emit_int8(0xB8 | encode);
 3392   emit_int32(imm32);
 3393 }
 3394 
 3395 void Assembler::movl(Register dst, Register src) {
 3396   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3397   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3398 }
 3399 
 3400 void Assembler::movl(Register dst, Address src) {
 3401   InstructionMark im(this);
 3402   prefix(src, dst);
 3403   emit_int8((unsigned char)0x8B);
 3404   emit_operand(dst, src, 0);
 3405 }
 3406 
 3407 void Assembler::movl(Address dst, int32_t imm32) {
 3408   InstructionMark im(this);
 3409   prefix(dst);
 3410   emit_int8((unsigned char)0xC7);
 3411   emit_operand(rax, dst, 4);
 3412   emit_int32(imm32);
 3413 }
 3414 
 3415 void Assembler::movl(Address dst, Register src) {
 3416   InstructionMark im(this);
 3417   prefix(dst, src);
 3418   emit_int8((unsigned char)0x89);
 3419   emit_operand(src, dst, 0);
 3420 }
 3421 
 3422 // New cpus require to use movsd and movss to avoid partial register stall
 3423 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3424 // The selection is done in MacroAssembler::movdbl() and movflt().
 3425 void Assembler::movlpd(XMMRegister dst, Address src) {
 3426   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3427   InstructionMark im(this);
 3428   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3429   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3430   attributes.set_rex_vex_w_reverted();
 3431   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3432   emit_int8(0x12);
 3433   emit_operand(dst, src, 0);
 3434 }
 3435 
 3436 void Assembler::movq(XMMRegister dst, Address src) {
 3437   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3438   InstructionMark im(this);
 3439   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3440   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3441   attributes.set_rex_vex_w_reverted();
 3442   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3443   emit_int8(0x7E);
 3444   emit_operand(dst, src, 0);
 3445 }
 3446 
 3447 void Assembler::movq(Address dst, XMMRegister src) {
 3448   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3449   InstructionMark im(this);
 3450   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3451   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3452   attributes.set_rex_vex_w_reverted();
 3453   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3454   emit_int8((unsigned char)0xD6);
 3455   emit_operand(src, dst, 0);
 3456 }
 3457 
 3458 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3460   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3461   attributes.set_rex_vex_w_reverted();
 3462   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3463   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3464 }
 3465 
 3466 void Assembler::movq(Register dst, XMMRegister src) {
 3467   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3468   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3469   // swap src/dst to get correct prefix
 3470   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3471   emit_int16(0x7E, (0xC0 | encode));
 3472 }
 3473 
 3474 void Assembler::movq(XMMRegister dst, Register src) {
 3475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3477   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3478   emit_int16(0x6E, (0xC0 | encode));
 3479 }
 3480 
 3481 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3482   InstructionMark im(this);
 3483   prefix(src, dst);
 3484   emit_int16(0x0F, (unsigned char)0xBE);
 3485   emit_operand(dst, src, 0);
 3486 }
 3487 
 3488 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3489   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3490   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3491   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3492 }
 3493 
 3494 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3495   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3496   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3497   attributes.set_rex_vex_w_reverted();
 3498   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3499   emit_int16(0x10, (0xC0 | encode));
 3500 }
 3501 
 3502 void Assembler::movsd(XMMRegister dst, Address src) {
 3503   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3504   InstructionMark im(this);
 3505   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3506   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3507   attributes.set_rex_vex_w_reverted();
 3508   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3509   emit_int8(0x10);
 3510   emit_operand(dst, src, 0);
 3511 }
 3512 
 3513 void Assembler::movsd(Address dst, XMMRegister src) {
 3514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3515   InstructionMark im(this);
 3516   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3517   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3518   attributes.reset_is_clear_context();
 3519   attributes.set_rex_vex_w_reverted();
 3520   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3521   emit_int8(0x11);
 3522   emit_operand(src, dst, 0);
 3523 }
 3524 
 3525 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3526   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3527   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3528   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3529   emit_int16(0x10, (0xC0 | encode));
 3530 }
 3531 
 3532 void Assembler::movss(XMMRegister dst, Address src) {
 3533   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3534   InstructionMark im(this);
 3535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3536   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3537   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3538   emit_int8(0x10);
 3539   emit_operand(dst, src, 0);
 3540 }
 3541 
 3542 void Assembler::movss(Address dst, XMMRegister src) {
 3543   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3544   InstructionMark im(this);
 3545   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3547   attributes.reset_is_clear_context();
 3548   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3549   emit_int8(0x11);
 3550   emit_operand(src, dst, 0);
 3551 }
 3552 
 3553 void Assembler::movswl(Register dst, Address src) { // movsxw
 3554   InstructionMark im(this);
 3555   prefix(src, dst);
 3556   emit_int16(0x0F, (unsigned char)0xBF);
 3557   emit_operand(dst, src, 0);
 3558 }
 3559 
 3560 void Assembler::movswl(Register dst, Register src) { // movsxw
 3561   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3562   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3563 }
 3564 
 3565 void Assembler::movups(XMMRegister dst, Address src) {
 3566   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3567   InstructionMark im(this);
 3568   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3569   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3570   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3571   emit_int8(0x10);
 3572   emit_operand(dst, src, 0);
 3573 }
 3574 
 3575 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
 3576   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3577   InstructionMark im(this);
 3578   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3579   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3580   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3581   emit_int8(0x10);
 3582   emit_operand(dst, src, 0);
 3583 }
 3584 
 3585 void Assembler::movups(Address dst, XMMRegister src) {
 3586   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3587   InstructionMark im(this);
 3588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3589   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3590   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3591   emit_int8(0x11);
 3592   emit_operand(src, dst, 0);
 3593 }
 3594 
 3595 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
 3596   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3597   InstructionMark im(this);
 3598   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3599   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3600   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3601   emit_int8(0x11);
 3602   emit_operand(src, dst, 0);
 3603 }
 3604 
 3605 void Assembler::movw(Address dst, int imm16) {
 3606   InstructionMark im(this);
 3607 
 3608   emit_int8(0x66); // switch to 16-bit mode
 3609   prefix(dst);
 3610   emit_int8((unsigned char)0xC7);
 3611   emit_operand(rax, dst, 2);
 3612   emit_int16(imm16);
 3613 }
 3614 
 3615 void Assembler::movw(Register dst, Address src) {
 3616   InstructionMark im(this);
 3617   emit_int8(0x66);
 3618   prefix(src, dst);
 3619   emit_int8((unsigned char)0x8B);
 3620   emit_operand(dst, src, 0);
 3621 }
 3622 
 3623 void Assembler::movw(Address dst, Register src) {
 3624   InstructionMark im(this);
 3625   emit_int8(0x66);
 3626   prefix(dst, src);
 3627   emit_int8((unsigned char)0x89);
 3628   emit_operand(src, dst, 0);
 3629 }
 3630 
 3631 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3632   InstructionMark im(this);
 3633   prefix(src, dst);
 3634   emit_int16(0x0F, (unsigned char)0xB6);
 3635   emit_operand(dst, src, 0);
 3636 }
 3637 
 3638 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3639   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3640   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3641   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3642 }
 3643 
 3644 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3645   InstructionMark im(this);
 3646   prefix(src, dst);
 3647   emit_int16(0x0F, (unsigned char)0xB7);
 3648   emit_operand(dst, src, 0);
 3649 }
 3650 
 3651 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3652   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3653   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3654 }
 3655 
 3656 void Assembler::mull(Address src) {
 3657   InstructionMark im(this);
 3658   prefix(src);
 3659   emit_int8((unsigned char)0xF7);
 3660   emit_operand(rsp, src, 0);
 3661 }
 3662 
 3663 void Assembler::mull(Register src) {
 3664   int encode = prefix_and_encode(src->encoding());
 3665   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3666 }
 3667 
 3668 void Assembler::mulsd(XMMRegister dst, Address src) {
 3669   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3670   InstructionMark im(this);
 3671   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3672   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3673   attributes.set_rex_vex_w_reverted();
 3674   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3675   emit_int8(0x59);
 3676   emit_operand(dst, src, 0);
 3677 }
 3678 
 3679 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3680   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3681   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3682   attributes.set_rex_vex_w_reverted();
 3683   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3684   emit_int16(0x59, (0xC0 | encode));
 3685 }
 3686 
 3687 void Assembler::mulss(XMMRegister dst, Address src) {
 3688   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3689   InstructionMark im(this);
 3690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3691   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3692   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3693   emit_int8(0x59);
 3694   emit_operand(dst, src, 0);
 3695 }
 3696 
 3697 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3698   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3699   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3700   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3701   emit_int16(0x59, (0xC0 | encode));
 3702 }
 3703 
 3704 void Assembler::negl(Register dst) {
 3705   int encode = prefix_and_encode(dst->encoding());
 3706   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3707 }
 3708 
 3709 void Assembler::negl(Address dst) {
 3710   InstructionMark im(this);
 3711   prefix(dst);
 3712   emit_int8((unsigned char)0xF7);
 3713   emit_operand(as_Register(3), dst, 0);
 3714 }
 3715 
 3716 void Assembler::nop(int i) {
 3717 #ifdef ASSERT
 3718   assert(i > 0, " ");
 3719   // The fancy nops aren't currently recognized by debuggers making it a
 3720   // pain to disassemble code while debugging. If asserts are on clearly
 3721   // speed is not an issue so simply use the single byte traditional nop
 3722   // to do alignment.
 3723 
 3724   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3725   return;
 3726 
 3727 #endif // ASSERT
 3728 
 3729   if (UseAddressNop && VM_Version::is_intel()) {
 3730     //
 3731     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3732     //  1: 0x90
 3733     //  2: 0x66 0x90
 3734     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3735     //  4: 0x0F 0x1F 0x40 0x00
 3736     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3737     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3738     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3739     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3740     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3741     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3742     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3743 
 3744     // The rest coding is Intel specific - don't use consecutive address nops
 3745 
 3746     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3747     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3748     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3749     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3750 
 3751     while(i >= 15) {
 3752       // For Intel don't generate consecutive address nops (mix with regular nops)
 3753       i -= 15;
 3754       emit_int24(0x66, 0x66, 0x66);
 3755       addr_nop_8();
 3756       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3757     }
 3758     switch (i) {
 3759       case 14:
 3760         emit_int8(0x66); // size prefix
 3761       case 13:
 3762         emit_int8(0x66); // size prefix
 3763       case 12:
 3764         addr_nop_8();
 3765         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3766         break;
 3767       case 11:
 3768         emit_int8(0x66); // size prefix
 3769       case 10:
 3770         emit_int8(0x66); // size prefix
 3771       case 9:
 3772         emit_int8(0x66); // size prefix
 3773       case 8:
 3774         addr_nop_8();
 3775         break;
 3776       case 7:
 3777         addr_nop_7();
 3778         break;
 3779       case 6:
 3780         emit_int8(0x66); // size prefix
 3781       case 5:
 3782         addr_nop_5();
 3783         break;
 3784       case 4:
 3785         addr_nop_4();
 3786         break;
 3787       case 3:
 3788         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3789         emit_int8(0x66); // size prefix
 3790       case 2:
 3791         emit_int8(0x66); // size prefix
 3792       case 1:
 3793         emit_int8((unsigned char)0x90);
 3794                          // nop
 3795         break;
 3796       default:
 3797         assert(i == 0, " ");
 3798     }
 3799     return;
 3800   }
 3801   if (UseAddressNop && VM_Version::is_amd_family()) {
 3802     //
 3803     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3804     //  1: 0x90
 3805     //  2: 0x66 0x90
 3806     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3807     //  4: 0x0F 0x1F 0x40 0x00
 3808     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3809     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3810     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3811     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3812     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3813     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3814     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3815 
 3816     // The rest coding is AMD specific - use consecutive address nops
 3817 
 3818     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3819     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3820     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3821     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3822     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3823     //     Size prefixes (0x66) are added for larger sizes
 3824 
 3825     while(i >= 22) {
 3826       i -= 11;
 3827       emit_int24(0x66, 0x66, 0x66);
 3828       addr_nop_8();
 3829     }
 3830     // Generate first nop for size between 21-12
 3831     switch (i) {
 3832       case 21:
 3833         i -= 1;
 3834         emit_int8(0x66); // size prefix
 3835       case 20:
 3836       case 19:
 3837         i -= 1;
 3838         emit_int8(0x66); // size prefix
 3839       case 18:
 3840       case 17:
 3841         i -= 1;
 3842         emit_int8(0x66); // size prefix
 3843       case 16:
 3844       case 15:
 3845         i -= 8;
 3846         addr_nop_8();
 3847         break;
 3848       case 14:
 3849       case 13:
 3850         i -= 7;
 3851         addr_nop_7();
 3852         break;
 3853       case 12:
 3854         i -= 6;
 3855         emit_int8(0x66); // size prefix
 3856         addr_nop_5();
 3857         break;
 3858       default:
 3859         assert(i < 12, " ");
 3860     }
 3861 
 3862     // Generate second nop for size between 11-1
 3863     switch (i) {
 3864       case 11:
 3865         emit_int8(0x66); // size prefix
 3866       case 10:
 3867         emit_int8(0x66); // size prefix
 3868       case 9:
 3869         emit_int8(0x66); // size prefix
 3870       case 8:
 3871         addr_nop_8();
 3872         break;
 3873       case 7:
 3874         addr_nop_7();
 3875         break;
 3876       case 6:
 3877         emit_int8(0x66); // size prefix
 3878       case 5:
 3879         addr_nop_5();
 3880         break;
 3881       case 4:
 3882         addr_nop_4();
 3883         break;
 3884       case 3:
 3885         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3886         emit_int8(0x66); // size prefix
 3887       case 2:
 3888         emit_int8(0x66); // size prefix
 3889       case 1:
 3890         emit_int8((unsigned char)0x90);
 3891                          // nop
 3892         break;
 3893       default:
 3894         assert(i == 0, " ");
 3895     }
 3896     return;
 3897   }
 3898 
 3899   if (UseAddressNop && VM_Version::is_zx()) {
 3900     //
 3901     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3902     //  1: 0x90
 3903     //  2: 0x66 0x90
 3904     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3905     //  4: 0x0F 0x1F 0x40 0x00
 3906     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3907     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3908     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3909     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3910     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3911     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3912     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3913 
 3914     // The rest coding is ZX specific - don't use consecutive address nops
 3915 
 3916     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3917     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3918     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3919     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3920 
 3921     while (i >= 15) {
 3922       // For ZX don't generate consecutive address nops (mix with regular nops)
 3923       i -= 15;
 3924       emit_int24(0x66, 0x66, 0x66);
 3925       addr_nop_8();
 3926       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3927     }
 3928     switch (i) {
 3929       case 14:
 3930         emit_int8(0x66); // size prefix
 3931       case 13:
 3932         emit_int8(0x66); // size prefix
 3933       case 12:
 3934         addr_nop_8();
 3935         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3936         break;
 3937       case 11:
 3938         emit_int8(0x66); // size prefix
 3939       case 10:
 3940         emit_int8(0x66); // size prefix
 3941       case 9:
 3942         emit_int8(0x66); // size prefix
 3943       case 8:
 3944         addr_nop_8();
 3945         break;
 3946       case 7:
 3947         addr_nop_7();
 3948         break;
 3949       case 6:
 3950         emit_int8(0x66); // size prefix
 3951       case 5:
 3952         addr_nop_5();
 3953         break;
 3954       case 4:
 3955         addr_nop_4();
 3956         break;
 3957       case 3:
 3958         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3959         emit_int8(0x66); // size prefix
 3960       case 2:
 3961         emit_int8(0x66); // size prefix
 3962       case 1:
 3963         emit_int8((unsigned char)0x90);
 3964                          // nop
 3965         break;
 3966       default:
 3967         assert(i == 0, " ");
 3968     }
 3969     return;
 3970   }
 3971 
 3972   // Using nops with size prefixes "0x66 0x90".
 3973   // From AMD Optimization Guide:
 3974   //  1: 0x90
 3975   //  2: 0x66 0x90
 3976   //  3: 0x66 0x66 0x90
 3977   //  4: 0x66 0x66 0x66 0x90
 3978   //  5: 0x66 0x66 0x90 0x66 0x90
 3979   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3980   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3981   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3982   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3983   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3984   //
 3985   while (i > 12) {
 3986     i -= 4;
 3987     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3988   }
 3989   // 1 - 12 nops
 3990   if (i > 8) {
 3991     if (i > 9) {
 3992       i -= 1;
 3993       emit_int8(0x66);
 3994     }
 3995     i -= 3;
 3996     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3997   }
 3998   // 1 - 8 nops
 3999   if (i > 4) {
 4000     if (i > 6) {
 4001       i -= 1;
 4002       emit_int8(0x66);
 4003     }
 4004     i -= 3;
 4005     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4006   }
 4007   switch (i) {
 4008     case 4:
 4009       emit_int8(0x66);
 4010     case 3:
 4011       emit_int8(0x66);
 4012     case 2:
 4013       emit_int8(0x66);
 4014     case 1:
 4015       emit_int8((unsigned char)0x90);
 4016       break;
 4017     default:
 4018       assert(i == 0, " ");
 4019   }
 4020 }
 4021 
 4022 void Assembler::notl(Register dst) {
 4023   int encode = prefix_and_encode(dst->encoding());
 4024   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 4025 }
 4026 
 4027 void Assembler::orw(Register dst, Register src) {
 4028   (void)prefix_and_encode(dst->encoding(), src->encoding());
 4029   emit_arith(0x0B, 0xC0, dst, src);
 4030 }
 4031 
 4032 void Assembler::orl(Address dst, int32_t imm32) {
 4033   InstructionMark im(this);
 4034   prefix(dst);
 4035   emit_arith_operand(0x81, rcx, dst, imm32);
 4036 }
 4037 
 4038 void Assembler::orl(Register dst, int32_t imm32) {
 4039   prefix(dst);
 4040   emit_arith(0x81, 0xC8, dst, imm32);
 4041 }
 4042 
 4043 void Assembler::orl(Register dst, Address src) {
 4044   InstructionMark im(this);
 4045   prefix(src, dst);
 4046   emit_int8(0x0B);
 4047   emit_operand(dst, src, 0);
 4048 }
 4049 
 4050 void Assembler::orl(Register dst, Register src) {
 4051   (void) prefix_and_encode(dst->encoding(), src->encoding());
 4052   emit_arith(0x0B, 0xC0, dst, src);
 4053 }
 4054 
 4055 void Assembler::orl(Address dst, Register src) {
 4056   InstructionMark im(this);
 4057   prefix(dst, src);
 4058   emit_int8(0x09);
 4059   emit_operand(src, dst, 0);
 4060 }
 4061 
 4062 void Assembler::orb(Address dst, int imm8) {
 4063   InstructionMark im(this);
 4064   prefix(dst);
 4065   emit_int8((unsigned char)0x80);
 4066   emit_operand(rcx, dst, 1);
 4067   emit_int8(imm8);
 4068 }
 4069 
 4070 void Assembler::orb(Address dst, Register src) {
 4071   InstructionMark im(this);
 4072   prefix(dst, src, true);
 4073   emit_int8(0x08);
 4074   emit_operand(src, dst, 0);
 4075 }
 4076 
 4077 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 4078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4080   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4081   emit_int16(0x63, (0xC0 | encode));
 4082 }
 4083 
 4084 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4085   assert(UseAVX > 0, "some form of AVX must be enabled");
 4086   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4087   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4088   emit_int16(0x63, (0xC0 | encode));
 4089 }
 4090 
 4091 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 4092   assert(VM_Version::supports_sse2(), "");
 4093   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4094   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4095   emit_int16(0x6B, (0xC0 | encode));
 4096 }
 4097 
 4098 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4099   assert(UseAVX > 0, "some form of AVX must be enabled");
 4100   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4101   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4102   emit_int16(0x6B, (0xC0 | encode));
 4103 }
 4104 
 4105 void Assembler::packuswb(XMMRegister dst, Address src) {
 4106   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4107   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4108   InstructionMark im(this);
 4109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4110   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4111   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4112   emit_int8(0x67);
 4113   emit_operand(dst, src, 0);
 4114 }
 4115 
 4116 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 4117   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4118   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4119   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4120   emit_int16(0x67, (0xC0 | encode));
 4121 }
 4122 
 4123 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4124   assert(UseAVX > 0, "some form of AVX must be enabled");
 4125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4126   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4127   emit_int16(0x67, (0xC0 | encode));
 4128 }
 4129 
 4130 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4131   assert(VM_Version::supports_sse4_1(), "");
 4132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4133   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4134   emit_int16(0x2B, (0xC0 | encode));
 4135 }
 4136 
 4137 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4138   assert(UseAVX > 0, "some form of AVX must be enabled");
 4139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4140   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4141   emit_int16(0x2B, (0xC0 | encode));
 4142 }
 4143 
 4144 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4145   assert(VM_Version::supports_avx2(), "");
 4146   assert(vector_len != AVX_128bit, "");
 4147   // VEX.256.66.0F3A.W1 00 /r ib
 4148   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4149   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4150   emit_int24(0x00, (0xC0 | encode), imm8);
 4151 }
 4152 
 4153 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4154   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4155          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4156   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4157   attributes.set_is_evex_instruction();
 4158   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4159   emit_int16(0x36, (0xC0 | encode));
 4160 }
 4161 
 4162 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4163   assert(VM_Version::supports_avx512_vbmi(), "");
 4164   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4165   attributes.set_is_evex_instruction();
 4166   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4167   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4168 }
 4169 
 4170 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4171   assert(VM_Version::supports_avx512_vbmi(), "");
 4172   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4173   attributes.set_is_evex_instruction();
 4174   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4175   emit_int8((unsigned char)0x8D);
 4176   emit_operand(dst, src, 0);
 4177 }
 4178 
 4179 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4180   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4181          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4182          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4183   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4184   attributes.set_is_evex_instruction();
 4185   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4186   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4187 }
 4188 
 4189 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4190   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4191   // VEX.NDS.256.66.0F38.W0 36 /r
 4192   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4193   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4194   emit_int16(0x36, (0xC0 | encode));
 4195 }
 4196 
 4197 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4198   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4199   // VEX.NDS.256.66.0F38.W0 36 /r
 4200   InstructionMark im(this);
 4201   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4202   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4203   emit_int8(0x36);
 4204   emit_operand(dst, src, 0);
 4205 }
 4206 
 4207 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4208   assert(VM_Version::supports_avx2(), "");
 4209   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4211   emit_int24(0x46, (0xC0 | encode), imm8);
 4212 }
 4213 
 4214 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4215   assert(VM_Version::supports_avx(), "");
 4216   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4217   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4218   emit_int24(0x06, (0xC0 | encode), imm8);
 4219 }
 4220 
 4221 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4222   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4223   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4225   emit_int24(0x04, (0xC0 | encode), imm8);
 4226 }
 4227 
 4228 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4229   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4230   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4231   attributes.set_rex_vex_w_reverted();
 4232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4233   emit_int24(0x05, (0xC0 | encode), imm8);
 4234 }
 4235 
 4236 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4237   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4238   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4239   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4240   emit_int24(0x01, (0xC0 | encode), imm8);
 4241 }
 4242 
 4243 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4244   assert(VM_Version::supports_evex(), "");
 4245   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4246   attributes.set_is_evex_instruction();
 4247   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4248   emit_int16(0x76, (0xC0 | encode));
 4249 }
 4250 
 4251 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4252   assert(VM_Version::supports_avx512_vbmi(), "");
 4253   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4254   attributes.set_is_evex_instruction();
 4255   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4256   emit_int16(0x7D, (0xC0 | encode));
 4257 }
 4258 
 4259 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4260   assert(VM_Version::supports_avx512_vbmi(), "");
 4261   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4262   attributes.set_is_evex_instruction();
 4263   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4264   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4265 }
 4266 
 4267 void Assembler::pause() {
 4268   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4269 }
 4270 
 4271 void Assembler::ud2() {
 4272   emit_int16(0x0F, 0x0B);
 4273 }
 4274 
 4275 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4276   assert(VM_Version::supports_sse4_2(), "");
 4277   InstructionMark im(this);
 4278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4279   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4280   emit_int8(0x61);
 4281   emit_operand(dst, src, 1);
 4282   emit_int8(imm8);
 4283 }
 4284 
 4285 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4286   assert(VM_Version::supports_sse4_2(), "");
 4287   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4288   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4289   emit_int24(0x61, (0xC0 | encode), imm8);
 4290 }
 4291 
 4292 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4293 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4294   assert(VM_Version::supports_sse2(), "");
 4295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4296   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4297   emit_int16(0x74, (0xC0 | encode));
 4298 }
 4299 
 4300 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4301   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4302   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4303   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4304   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4305   emit_int16(cond_encoding, (0xC0 | encode));
 4306 }
 4307 
 4308 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4309 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4310   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4311   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4312   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4313   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4314   emit_int16(0x74, (0xC0 | encode));
 4315 }
 4316 
 4317 // In this context, kdst is written the mask used to process the equal components
 4318 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4319   assert(VM_Version::supports_avx512bw(), "");
 4320   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4321   attributes.set_is_evex_instruction();
 4322   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4323   emit_int16(0x74, (0xC0 | encode));
 4324 }
 4325 
 4326 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4327   assert(VM_Version::supports_avx512vlbw(), "");
 4328   InstructionMark im(this);
 4329   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4330   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4331   attributes.set_is_evex_instruction();
 4332   int dst_enc = kdst->encoding();
 4333   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4334   emit_int8(0x64);
 4335   emit_operand(as_Register(dst_enc), src, 0);
 4336 }
 4337 
 4338 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4339   assert(VM_Version::supports_avx512vlbw(), "");
 4340   InstructionMark im(this);
 4341   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4342   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4343   attributes.reset_is_clear_context();
 4344   attributes.set_embedded_opmask_register_specifier(mask);
 4345   attributes.set_is_evex_instruction();
 4346   int dst_enc = kdst->encoding();
 4347   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4348   emit_int8(0x64);
 4349   emit_operand(as_Register(dst_enc), src, 0);
 4350 }
 4351 
 4352 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4353   assert(VM_Version::supports_avx512vlbw(), "");
 4354   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4355   attributes.set_is_evex_instruction();
 4356   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4357   emit_int24(0x3E, (0xC0 | encode), vcc);
 4358 }
 4359 
 4360 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4361   assert(VM_Version::supports_avx512vlbw(), "");
 4362   InstructionMark im(this);
 4363   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4364   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4365   attributes.set_is_evex_instruction();
 4366   int dst_enc = kdst->encoding();
 4367   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4368   emit_int8(0x3E);
 4369   emit_operand(as_Register(dst_enc), src, 1);
 4370   emit_int8(vcc);
 4371 }
 4372 
 4373 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4374   assert(VM_Version::supports_avx512bw(), "");
 4375   InstructionMark im(this);
 4376   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4377   attributes.set_is_evex_instruction();
 4378   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4379   int dst_enc = kdst->encoding();
 4380   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4381   emit_int8(0x74);
 4382   emit_operand(as_Register(dst_enc), src, 0);
 4383 }
 4384 
 4385 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4386   assert(VM_Version::supports_avx512vlbw(), "");
 4387   InstructionMark im(this);
 4388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4389   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4390   attributes.reset_is_clear_context();
 4391   attributes.set_embedded_opmask_register_specifier(mask);
 4392   attributes.set_is_evex_instruction();
 4393   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4394   emit_int8(0x74);
 4395   emit_operand(as_Register(kdst->encoding()), src, 0);
 4396 }
 4397 
 4398 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4399 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4400   assert(VM_Version::supports_sse2(), "");
 4401   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4402   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4403   emit_int16(0x75, (0xC0 | encode));
 4404 }
 4405 
 4406 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4407 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4408   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4409   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4410   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4411   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4412   emit_int16(0x75, (0xC0 | encode));
 4413 }
 4414 
 4415 // In this context, kdst is written the mask used to process the equal components
 4416 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4417   assert(VM_Version::supports_avx512bw(), "");
 4418   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4419   attributes.set_is_evex_instruction();
 4420   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4421   emit_int16(0x75, (0xC0 | encode));
 4422 }
 4423 
 4424 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4425   assert(VM_Version::supports_avx512bw(), "");
 4426   InstructionMark im(this);
 4427   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4428   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4429   attributes.set_is_evex_instruction();
 4430   int dst_enc = kdst->encoding();
 4431   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4432   emit_int8(0x75);
 4433   emit_operand(as_Register(dst_enc), src, 0);
 4434 }
 4435 
 4436 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4437 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4438   assert(VM_Version::supports_sse2(), "");
 4439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4440   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4441   emit_int16(0x76, (0xC0 | encode));
 4442 }
 4443 
 4444 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4445 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4446   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4447   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4448   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4450   emit_int16(0x76, (0xC0 | encode));
 4451 }
 4452 
 4453 // In this context, kdst is written the mask used to process the equal components
 4454 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4455   assert(VM_Version::supports_evex(), "");
 4456   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4457   attributes.set_is_evex_instruction();
 4458   attributes.reset_is_clear_context();
 4459   attributes.set_embedded_opmask_register_specifier(mask);
 4460   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4461   emit_int16(0x76, (0xC0 | encode));
 4462 }
 4463 
 4464 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4465   assert(VM_Version::supports_evex(), "");
 4466   InstructionMark im(this);
 4467   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4468   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4469   attributes.set_is_evex_instruction();
 4470   attributes.reset_is_clear_context();
 4471   attributes.set_embedded_opmask_register_specifier(mask);
 4472   int dst_enc = kdst->encoding();
 4473   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4474   emit_int8(0x76);
 4475   emit_operand(as_Register(dst_enc), src, 0);
 4476 }
 4477 
 4478 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4479 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4480   assert(VM_Version::supports_sse4_1(), "");
 4481   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4482   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4483   emit_int16(0x29, (0xC0 | encode));
 4484 }
 4485 
 4486 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4487   assert(VM_Version::supports_evex(), "");
 4488   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4489   attributes.set_is_evex_instruction();
 4490   attributes.reset_is_clear_context();
 4491   attributes.set_embedded_opmask_register_specifier(mask);
 4492   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4493   emit_int16(0x29, (0xC0 | encode));
 4494 }
 4495 
 4496 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4497   assert(VM_Version::supports_avx(), "");
 4498   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4499   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4500   emit_int16(cond_encoding, (0xC0 | encode));
 4501 }
 4502 
 4503 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4504 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4505   assert(VM_Version::supports_avx(), "");
 4506   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4507   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4508   emit_int16(0x29, (0xC0 | encode));
 4509 }
 4510 
 4511 // In this context, kdst is written the mask used to process the equal components
 4512 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4513   assert(VM_Version::supports_evex(), "");
 4514   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4515   attributes.reset_is_clear_context();
 4516   attributes.set_is_evex_instruction();
 4517   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4518   emit_int16(0x29, (0xC0 | encode));
 4519 }
 4520 
 4521 // In this context, kdst is written the mask used to process the equal components
 4522 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4523   assert(VM_Version::supports_evex(), "");
 4524   InstructionMark im(this);
 4525   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4526   attributes.reset_is_clear_context();
 4527   attributes.set_is_evex_instruction();
 4528   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4529   int dst_enc = kdst->encoding();
 4530   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4531   emit_int8(0x29);
 4532   emit_operand(as_Register(dst_enc), src, 0);
 4533 }
 4534 
 4535 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4536   assert(VM_Version::supports_sse4_1(), "");
 4537   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4538   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4539   emit_int16(0x37, (0xC0 | encode));
 4540 }
 4541 
 4542 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4543   assert(VM_Version::supports_sse2(), "");
 4544   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4545   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4546   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4547 }
 4548 
 4549 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4550   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4551          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4552   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4553   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4554   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4555 }
 4556 
 4557 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4558   assert(VM_Version::supports_avx(), "");
 4559   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4560   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4561   emit_int16(0x50, (0xC0 | encode));
 4562 }
 4563 
 4564 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4565   assert(VM_Version::supports_avx(), "");
 4566   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4567   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4568   emit_int16(0x50, (0xC0 | encode));
 4569 }
 4570 
 4571 
 4572 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4573   assert(VM_Version::supports_sse4_1(), "");
 4574   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4575   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4576   emit_int24(0x16, (0xC0 | encode), imm8);
 4577 }
 4578 
 4579 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4580   assert(VM_Version::supports_sse4_1(), "");
 4581   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4582   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4583   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4584   emit_int8(0x16);
 4585   emit_operand(src, dst, 1);
 4586   emit_int8(imm8);
 4587 }
 4588 
 4589 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4590   assert(VM_Version::supports_sse4_1(), "");
 4591   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4592   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4593   emit_int24(0x16, (0xC0 | encode), imm8);
 4594 }
 4595 
 4596 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4597   assert(VM_Version::supports_sse4_1(), "");
 4598   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4599   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4600   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4601   emit_int8(0x16);
 4602   emit_operand(src, dst, 1);
 4603   emit_int8(imm8);
 4604 }
 4605 
 4606 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4607   assert(VM_Version::supports_sse2(), "");
 4608   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4609   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4610   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4611 }
 4612 
 4613 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4614   assert(VM_Version::supports_sse4_1(), "");
 4615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4616   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4617   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4618   emit_int8(0x15);
 4619   emit_operand(src, dst, 1);
 4620   emit_int8(imm8);
 4621 }
 4622 
 4623 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4624   assert(VM_Version::supports_sse4_1(), "");
 4625   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4626   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4627   emit_int24(0x14, (0xC0 | encode), imm8);
 4628 }
 4629 
 4630 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4631   assert(VM_Version::supports_sse4_1(), "");
 4632   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4633   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4634   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4635   emit_int8(0x14);
 4636   emit_operand(src, dst, 1);
 4637   emit_int8(imm8);
 4638 }
 4639 
 4640 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4641   assert(VM_Version::supports_sse4_1(), "");
 4642   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4643   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4644   emit_int24(0x22, (0xC0 | encode), imm8);
 4645 }
 4646 
 4647 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4648   assert(VM_Version::supports_sse4_1(), "");
 4649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4650   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4651   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4652   emit_int8(0x22);
 4653   emit_operand(dst, src, 1);
 4654   emit_int8(imm8);
 4655 }
 4656 
 4657 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4658   assert(VM_Version::supports_avx(), "");
 4659   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4660   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4661   emit_int24(0x22, (0xC0 | encode), imm8);
 4662 }
 4663 
 4664 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4665   assert(VM_Version::supports_sse4_1(), "");
 4666   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4667   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4668   emit_int24(0x22, (0xC0 | encode), imm8);
 4669 }
 4670 
 4671 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4672   assert(VM_Version::supports_sse4_1(), "");
 4673   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4674   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4675   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4676   emit_int8(0x22);
 4677   emit_operand(dst, src, 1);
 4678   emit_int8(imm8);
 4679 }
 4680 
 4681 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4682   assert(VM_Version::supports_avx(), "");
 4683   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4684   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4685   emit_int24(0x22, (0xC0 | encode), imm8);
 4686 }
 4687 
 4688 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4689   assert(VM_Version::supports_sse2(), "");
 4690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4691   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4692   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4693 }
 4694 
 4695 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4696   assert(VM_Version::supports_sse2(), "");
 4697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4698   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4699   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4700   emit_int8((unsigned char)0xC4);
 4701   emit_operand(dst, src, 1);
 4702   emit_int8(imm8);
 4703 }
 4704 
 4705 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4706   assert(VM_Version::supports_avx(), "");
 4707   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4708   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4709   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4710 }
 4711 
 4712 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4713   assert(VM_Version::supports_sse4_1(), "");
 4714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4715   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4716   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4717   emit_int8(0x20);
 4718   emit_operand(dst, src, 1);
 4719   emit_int8(imm8);
 4720 }
 4721 
 4722 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4723   assert(VM_Version::supports_sse4_1(), "");
 4724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4725   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4726   emit_int24(0x20, (0xC0 | encode), imm8);
 4727 }
 4728 
 4729 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4730   assert(VM_Version::supports_avx(), "");
 4731   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4732   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4733   emit_int24(0x20, (0xC0 | encode), imm8);
 4734 }
 4735 
 4736 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4737   assert(VM_Version::supports_sse4_1(), "");
 4738   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4739   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4740   emit_int24(0x21, (0xC0 | encode), imm8);
 4741 }
 4742 
 4743 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4744   assert(VM_Version::supports_avx(), "");
 4745   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4746   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4747   emit_int24(0x21, (0xC0 | encode), imm8);
 4748 }
 4749 
 4750 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4751   assert(VM_Version::supports_sse4_1(), "");
 4752   InstructionMark im(this);
 4753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4754   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4755   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4756   emit_int8(0x30);
 4757   emit_operand(dst, src, 0);
 4758 }
 4759 
 4760 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4761   assert(VM_Version::supports_sse4_1(), "");
 4762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4763   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4764   emit_int16(0x30, (0xC0 | encode));
 4765 }
 4766 
 4767 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4768   assert(VM_Version::supports_sse4_1(), "");
 4769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4770   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4771   emit_int16(0x20, (0xC0 | encode));
 4772 }
 4773 
 4774 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4775   assert(VM_Version::supports_sse4_1(), "");
 4776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4777   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4778   emit_int16(0x35, (0xC0 | encode));
 4779 }
 4780 
 4781 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4782   assert(VM_Version::supports_sse4_1(), "");
 4783   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4784   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4785   emit_int16(0x21, (0xC0 | encode));
 4786 }
 4787 
 4788 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4789   assert(VM_Version::supports_sse4_1(), "");
 4790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4791   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4792   emit_int16(0x31, (0xC0 | encode));
 4793 }
 4794 
 4795 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4796   assert(VM_Version::supports_sse4_1(), "");
 4797   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4798   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4799   emit_int16(0x22, (0xC0 | encode));
 4800 }
 4801 
 4802 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4803   assert(VM_Version::supports_sse4_1(), "");
 4804   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4805   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4806   emit_int16(0x23, (0xC0 | encode));
 4807 }
 4808 
 4809 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4810   assert(VM_Version::supports_avx(), "");
 4811   InstructionMark im(this);
 4812   assert(dst != xnoreg, "sanity");
 4813   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4814   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4815   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4816   emit_int8(0x30);
 4817   emit_operand(dst, src, 0);
 4818 }
 4819 
 4820 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4821   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4822   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4823   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4824   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4825   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4826   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4827 }
 4828 
 4829 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4830   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4831   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4832   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4833   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4835   emit_int16(0x20, (0xC0 | encode));
 4836 }
 4837 
 4838 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4839   assert(VM_Version::supports_avx512vlbw(), "");
 4840   assert(dst != xnoreg, "sanity");
 4841   InstructionMark im(this);
 4842   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4843   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4844   attributes.set_embedded_opmask_register_specifier(mask);
 4845   attributes.set_is_evex_instruction();
 4846   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4847   emit_int8(0x30);
 4848   emit_operand(dst, src, 0);
 4849 }
 4850 
 4851 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4852   assert(VM_Version::supports_evex(), "");
 4853   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4854   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4855   attributes.set_is_evex_instruction();
 4856   attributes.set_embedded_opmask_register_specifier(mask);
 4857   if (merge) {
 4858     attributes.reset_is_clear_context();
 4859   }
 4860   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4861   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4862 }
 4863 
 4864 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4865   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4866   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4868   emit_int16(0x35, (0xC0 | encode));
 4869 }
 4870 
 4871 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4872   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4875   emit_int16(0x31, (0xC0 | encode));
 4876 }
 4877 
 4878 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4879   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4880   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4882   emit_int16(0x32, (0xC0 | encode));
 4883 }
 4884 
 4885 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4886   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4887          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4888              VM_Version::supports_evex(), "");
 4889   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4890   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4891   emit_int16(0x21, (0xC0 | encode));
 4892 }
 4893 
 4894 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4895   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4896          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4897              VM_Version::supports_evex(), "");
 4898   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4899   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4900   emit_int16(0x22, (0xC0 | encode));
 4901 }
 4902 
 4903 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4904   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4905          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4906              VM_Version::supports_evex(), "");
 4907   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4908   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4909   emit_int16(0x23, (0xC0 | encode));
 4910 }
 4911 
 4912 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4913   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4914          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4915              VM_Version::supports_evex(), "");
 4916   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4917   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4918   emit_int16(0x24, (0xC0 | encode));
 4919 }
 4920 
 4921 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4922   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4923          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4924              VM_Version::supports_evex(), "");
 4925   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4926   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4927   emit_int16(0x25, (0xC0 | encode));
 4928 }
 4929 
 4930 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4931   assert(VM_Version::supports_avx512vlbw(), "");
 4932   assert(src != xnoreg, "sanity");
 4933   InstructionMark im(this);
 4934   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4935   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4936   attributes.set_is_evex_instruction();
 4937   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4938   emit_int8(0x30);
 4939   emit_operand(src, dst, 0);
 4940 }
 4941 
 4942 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4943   assert(VM_Version::supports_avx512vlbw(), "");
 4944   assert(src != xnoreg, "sanity");
 4945   InstructionMark im(this);
 4946   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4947   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4948   attributes.reset_is_clear_context();
 4949   attributes.set_embedded_opmask_register_specifier(mask);
 4950   attributes.set_is_evex_instruction();
 4951   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4952   emit_int8(0x30);
 4953   emit_operand(src, dst, 0);
 4954 }
 4955 
 4956 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4957   assert(VM_Version::supports_evex(), "");
 4958   assert(src != xnoreg, "sanity");
 4959   InstructionMark im(this);
 4960   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4961   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4962   attributes.set_is_evex_instruction();
 4963   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4964   emit_int8(0x31);
 4965   emit_operand(src, dst, 0);
 4966 }
 4967 
 4968 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4969   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4970   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4971   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4972   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4973   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4974   emit_int16(0x33, (0xC0 | encode));
 4975 }
 4976 
 4977 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4978   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4979   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4980   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4981   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4982   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4983   emit_int16(0x34, (0xC0 | encode));
 4984 }
 4985 
 4986 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4989   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4990   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4991 }
 4992 
 4993 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4994   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4995     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4996     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4997   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4998   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4999   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5000 }
 5001 
 5002 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5003 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5004        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5005        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5006   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5007   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5008   emit_int16(0x04, (0xC0 | encode));
 5009 }
 5010 
 5011 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5012   assert(VM_Version::supports_evex(), "");
 5013   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 5014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5015   attributes.set_is_evex_instruction();
 5016   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5017   emit_int16(0x52, (0xC0 | encode));
 5018 }
 5019 
 5020 // generic
 5021 void Assembler::pop(Register dst) {
 5022   int encode = prefix_and_encode(dst->encoding());
 5023   emit_int8(0x58 | encode);
 5024 }
 5025 
 5026 void Assembler::popcntl(Register dst, Address src) {
 5027   assert(VM_Version::supports_popcnt(), "must support");
 5028   InstructionMark im(this);
 5029   emit_int8((unsigned char)0xF3);
 5030   prefix(src, dst);
 5031   emit_int16(0x0F, (unsigned char)0xB8);
 5032   emit_operand(dst, src, 0);
 5033 }
 5034 
 5035 void Assembler::popcntl(Register dst, Register src) {
 5036   assert(VM_Version::supports_popcnt(), "must support");
 5037   emit_int8((unsigned char)0xF3);
 5038   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5039   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 5040 }
 5041 
 5042 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5043   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5044   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5045   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5046   attributes.set_embedded_opmask_register_specifier(mask);
 5047   attributes.set_is_evex_instruction();
 5048   if (merge) {
 5049     attributes.reset_is_clear_context();
 5050   }
 5051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5052   emit_int16(0x54, (0xC0 | encode));
 5053 }
 5054 
 5055 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5056   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5057   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5058   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5059   attributes.set_is_evex_instruction();
 5060   attributes.set_embedded_opmask_register_specifier(mask);
 5061   if (merge) {
 5062     attributes.reset_is_clear_context();
 5063   }
 5064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5065   emit_int16(0x54, (0xC0 | encode));
 5066 }
 5067 
 5068 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5069   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5070   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5072   attributes.set_is_evex_instruction();
 5073   attributes.set_embedded_opmask_register_specifier(mask);
 5074   if (merge) {
 5075     attributes.reset_is_clear_context();
 5076   }
 5077   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5078   emit_int16(0x55, (0xC0 | encode));
 5079 }
 5080 
 5081 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5082   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5083   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5084   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5085   attributes.set_is_evex_instruction();
 5086   attributes.set_embedded_opmask_register_specifier(mask);
 5087   if (merge) {
 5088     attributes.reset_is_clear_context();
 5089   }
 5090   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5091   emit_int16(0x55, (0xC0 | encode));
 5092 }
 5093 
 5094 void Assembler::popf() {
 5095   emit_int8((unsigned char)0x9D);
 5096 }
 5097 
 5098 #ifndef _LP64 // no 32bit push/pop on amd64
 5099 void Assembler::popl(Address dst) {
 5100   // NOTE: this will adjust stack by 8byte on 64bits
 5101   InstructionMark im(this);
 5102   prefix(dst);
 5103   emit_int8((unsigned char)0x8F);
 5104   emit_operand(rax, dst, 0);
 5105 }
 5106 #endif
 5107 
 5108 void Assembler::prefetchnta(Address src) {
 5109   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5110   InstructionMark im(this);
 5111   prefix(src);
 5112   emit_int16(0x0F, 0x18);
 5113   emit_operand(rax, src, 0); // 0, src
 5114 }
 5115 
 5116 void Assembler::prefetchr(Address src) {
 5117   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5118   InstructionMark im(this);
 5119   prefix(src);
 5120   emit_int16(0x0F, 0x0D);
 5121   emit_operand(rax, src, 0); // 0, src
 5122 }
 5123 
 5124 void Assembler::prefetcht0(Address src) {
 5125   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5126   InstructionMark im(this);
 5127   prefix(src);
 5128   emit_int16(0x0F, 0x18);
 5129   emit_operand(rcx, src, 0); // 1, src
 5130 }
 5131 
 5132 void Assembler::prefetcht1(Address src) {
 5133   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5134   InstructionMark im(this);
 5135   prefix(src);
 5136   emit_int16(0x0F, 0x18);
 5137   emit_operand(rdx, src, 0); // 2, src
 5138 }
 5139 
 5140 void Assembler::prefetcht2(Address src) {
 5141   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5142   InstructionMark im(this);
 5143   prefix(src);
 5144   emit_int16(0x0F, 0x18);
 5145   emit_operand(rbx, src, 0); // 3, src
 5146 }
 5147 
 5148 void Assembler::prefetchw(Address src) {
 5149   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5150   InstructionMark im(this);
 5151   prefix(src);
 5152   emit_int16(0x0F, 0x0D);
 5153   emit_operand(rcx, src, 0); // 1, src
 5154 }
 5155 
 5156 void Assembler::prefix(Prefix p) {
 5157   emit_int8(p);
 5158 }
 5159 
 5160 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5161   assert(VM_Version::supports_ssse3(), "");
 5162   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5163   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5164   emit_int16(0x00, (0xC0 | encode));
 5165 }
 5166 
 5167 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 5168   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 5169   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5170   attributes.set_is_evex_instruction();
 5171   attributes.set_embedded_opmask_register_specifier(mask);
 5172   if (merge) {
 5173     attributes.reset_is_clear_context();
 5174   }
 5175   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5176   emit_int16(0x00, (0xC0 | encode));
 5177 }
 5178 
 5179 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5180   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5181          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5182          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5183   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5184   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5185   emit_int16(0x00, (0xC0 | encode));
 5186 }
 5187 
 5188 void Assembler::pshufb(XMMRegister dst, Address src) {
 5189   assert(VM_Version::supports_ssse3(), "");
 5190   InstructionMark im(this);
 5191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5192   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5193   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5194   emit_int8(0x00);
 5195   emit_operand(dst, src, 0);
 5196 }
 5197 
 5198 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5199   assert(isByte(mode), "invalid value");
 5200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5201   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5202   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5203   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5204   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5205 }
 5206 
 5207 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5208   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5209          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5210          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5211   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5212   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5213   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5214   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5215 }
 5216 
 5217 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5218   assert(isByte(mode), "invalid value");
 5219   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5220   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5221   InstructionMark im(this);
 5222   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5223   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5224   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5225   emit_int8(0x70);
 5226   emit_operand(dst, src, 1);
 5227   emit_int8(mode & 0xFF);
 5228 }
 5229 
 5230 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5231   assert(isByte(mode), "invalid value");
 5232   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5233   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5234   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5235   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5236 }
 5237 
 5238 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5239   assert(isByte(mode), "invalid value");
 5240   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5242   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5243   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5244 }
 5245 
 5246 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5247   assert(isByte(mode), "invalid value");
 5248   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5249   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5250   InstructionMark im(this);
 5251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5252   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5253   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5254   emit_int8(0x70);
 5255   emit_operand(dst, src, 1);
 5256   emit_int8(mode & 0xFF);
 5257 }
 5258 
 5259 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5260   assert(VM_Version::supports_evex(), "requires EVEX support");
 5261   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5262   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5263   attributes.set_is_evex_instruction();
 5264   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5265   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5266 }
 5267 
 5268 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5269   assert(isByte(imm8), "invalid value");
 5270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5272   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5273   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5274 }
 5275 
 5276 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5277   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5278   attributes.set_rex_vex_w_reverted();
 5279   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5280   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5281 }
 5282 
 5283 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
 5284   assert(isByte(imm8), "invalid value");
 5285   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5287   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5288   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5289 }
 5290 
 5291 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5292   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5294   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5295 }
 5296 
 5297 void Assembler::psrldq(XMMRegister dst, int shift) {
 5298   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5301   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5302   emit_int24(0x73, (0xC0 | encode), shift);
 5303 }
 5304 
 5305 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5306   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5307          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5308          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5309   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5310   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5311   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5312 }
 5313 
 5314 void Assembler::pslldq(XMMRegister dst, int shift) {
 5315   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5318   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5319   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5320   emit_int24(0x73, (0xC0 | encode), shift);
 5321 }
 5322 
 5323 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5324   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5325          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5326          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5327   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5328   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5329   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5330 }
 5331 
 5332 void Assembler::ptest(XMMRegister dst, Address src) {
 5333   assert(VM_Version::supports_sse4_1(), "");
 5334   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5335   InstructionMark im(this);
 5336   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5337   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5338   emit_int8(0x17);
 5339   emit_operand(dst, src, 0);
 5340 }
 5341 
 5342 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5343   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5344   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5345   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5346   emit_int8(0x17);
 5347   emit_int8((0xC0 | encode));
 5348 }
 5349 
 5350 void Assembler::vptest(XMMRegister dst, Address src) {
 5351   assert(VM_Version::supports_avx(), "");
 5352   InstructionMark im(this);
 5353   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5354   assert(dst != xnoreg, "sanity");
 5355   // swap src<->dst for encoding
 5356   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5357   emit_int8(0x17);
 5358   emit_operand(dst, src, 0);
 5359 }
 5360 
 5361 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5362   assert(VM_Version::supports_avx(), "");
 5363   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5364   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5365   emit_int16(0x17, (0xC0 | encode));
 5366 }
 5367 
 5368 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5369   assert(VM_Version::supports_avx(), "");
 5370   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5371   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5372   emit_int16(0x17, (0xC0 | encode));
 5373 }
 5374 
 5375 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5376   assert(VM_Version::supports_avx512vlbw(), "");
 5377   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5379   attributes.set_is_evex_instruction();
 5380   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5381   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5382 }
 5383 
 5384 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5386   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5387   InstructionMark im(this);
 5388   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5389   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5390   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5391   emit_int8(0x60);
 5392   emit_operand(dst, src, 0);
 5393 }
 5394 
 5395 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5397   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5398   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5399   emit_int16(0x60, (0xC0 | encode));
 5400 }
 5401 
 5402 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5403   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5404   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5405   InstructionMark im(this);
 5406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5407   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5408   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5409   emit_int8(0x62);
 5410   emit_operand(dst, src, 0);
 5411 }
 5412 
 5413 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5414   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5415   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5416   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5417   emit_int16(0x62, (0xC0 | encode));
 5418 }
 5419 
 5420 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5422   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5423   attributes.set_rex_vex_w_reverted();
 5424   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5425   emit_int16(0x6C, (0xC0 | encode));
 5426 }
 5427 
 5428 void Assembler::push(int32_t imm32) {
 5429   // in 64bits we push 64bits onto the stack but only
 5430   // take a 32bit immediate
 5431   emit_int8(0x68);
 5432   emit_int32(imm32);
 5433 }
 5434 
 5435 void Assembler::push(Register src) {
 5436   int encode = prefix_and_encode(src->encoding());
 5437   emit_int8(0x50 | encode);
 5438 }
 5439 
 5440 void Assembler::pushf() {
 5441   emit_int8((unsigned char)0x9C);
 5442 }
 5443 
 5444 #ifndef _LP64 // no 32bit push/pop on amd64
 5445 void Assembler::pushl(Address src) {
 5446   // Note this will push 64bit on 64bit
 5447   InstructionMark im(this);
 5448   prefix(src);
 5449   emit_int8((unsigned char)0xFF);
 5450   emit_operand(rsi, src, 0);
 5451 }
 5452 #endif
 5453 
 5454 void Assembler::rcll(Register dst, int imm8) {
 5455   assert(isShiftCount(imm8), "illegal shift count");
 5456   int encode = prefix_and_encode(dst->encoding());
 5457   if (imm8 == 1) {
 5458     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5459   } else {
 5460     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5461   }
 5462 }
 5463 
 5464 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5465   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5467   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5468   emit_int16(0x53, (0xC0 | encode));
 5469 }
 5470 
 5471 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5472   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5473   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5474   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5475   emit_int16(0x53, (0xC0 | encode));
 5476 }
 5477 
 5478 void Assembler::rdtsc() {
 5479   emit_int16(0x0F, 0x31);
 5480 }
 5481 
 5482 void Assembler::rdtscp() {
 5483   emit_int24(0x0F, 0x01, (unsigned char)0xF9);
 5484 }
 5485 
 5486 // copies data from [esi] to [edi] using rcx pointer sized words
 5487 // generic
 5488 void Assembler::rep_mov() {
 5489   // REP
 5490   // MOVSQ
 5491   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5492   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5493 }
 5494 
 5495 // sets rcx bytes with rax, value at [edi]
 5496 void Assembler::rep_stosb() {
 5497   // REP
 5498   // STOSB
 5499   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5500   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5501 }
 5502 
 5503 // sets rcx pointer sized words with rax, value at [edi]
 5504 // generic
 5505 void Assembler::rep_stos() {
 5506   // REP
 5507   // LP64:STOSQ, LP32:STOSD
 5508   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5509   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5510 }
 5511 
 5512 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5513 // generic
 5514 void Assembler::repne_scan() { // repne_scan
 5515   // SCASQ
 5516   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5517   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5518 }
 5519 
 5520 #ifdef _LP64
 5521 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5522 // generic
 5523 void Assembler::repne_scanl() { // repne_scan
 5524   // SCASL
 5525   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5526 }
 5527 #endif
 5528 
 5529 void Assembler::ret(int imm16) {
 5530   if (imm16 == 0) {
 5531     emit_int8((unsigned char)0xC3);
 5532   } else {
 5533     emit_int8((unsigned char)0xC2);
 5534     emit_int16(imm16);
 5535   }
 5536 }
 5537 
 5538 void Assembler::roll(Register dst, int imm8) {
 5539   assert(isShiftCount(imm8), "illegal shift count");
 5540   int encode = prefix_and_encode(dst->encoding());
 5541   if (imm8 == 1) {
 5542     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5543   } else {
 5544     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5545   }
 5546 }
 5547 
 5548 void Assembler::roll(Register dst) {
 5549   int encode = prefix_and_encode(dst->encoding());
 5550   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5551 }
 5552 
 5553 void Assembler::rorl(Register dst, int imm8) {
 5554   assert(isShiftCount(imm8), "illegal shift count");
 5555   int encode = prefix_and_encode(dst->encoding());
 5556   if (imm8 == 1) {
 5557     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5558   } else {
 5559     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5560   }
 5561 }
 5562 
 5563 void Assembler::rorl(Register dst) {
 5564   int encode = prefix_and_encode(dst->encoding());
 5565   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5566 }
 5567 
 5568 #ifdef _LP64
 5569 void Assembler::rorq(Register dst) {
 5570   int encode = prefixq_and_encode(dst->encoding());
 5571   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5572 }
 5573 
 5574 void Assembler::rorq(Register dst, int imm8) {
 5575   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5576   int encode = prefixq_and_encode(dst->encoding());
 5577   if (imm8 == 1) {
 5578     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5579   } else {
 5580     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5581   }
 5582 }
 5583 
 5584 void Assembler::rolq(Register dst) {
 5585   int encode = prefixq_and_encode(dst->encoding());
 5586   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5587 }
 5588 
 5589 void Assembler::rolq(Register dst, int imm8) {
 5590   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5591   int encode = prefixq_and_encode(dst->encoding());
 5592   if (imm8 == 1) {
 5593     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5594   } else {
 5595     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5596   }
 5597 }
 5598 #endif
 5599 
 5600 void Assembler::sahf() {
 5601 #ifdef _LP64
 5602   // Not supported in 64bit mode
 5603   ShouldNotReachHere();
 5604 #endif
 5605   emit_int8((unsigned char)0x9E);
 5606 }
 5607 
 5608 void Assembler::sall(Address dst, int imm8) {
 5609   InstructionMark im(this);
 5610   assert(isShiftCount(imm8), "illegal shift count");
 5611   prefix(dst);
 5612   if (imm8 == 1) {
 5613     emit_int8((unsigned char)0xD1);
 5614     emit_operand(as_Register(4), dst, 0);
 5615   }
 5616   else {
 5617     emit_int8((unsigned char)0xC1);
 5618     emit_operand(as_Register(4), dst, 1);
 5619     emit_int8(imm8);
 5620   }
 5621 }
 5622 
 5623 void Assembler::sall(Address dst) {
 5624   InstructionMark im(this);
 5625   prefix(dst);
 5626   emit_int8((unsigned char)0xD3);
 5627   emit_operand(as_Register(4), dst, 0);
 5628 }
 5629 
 5630 void Assembler::sall(Register dst, int imm8) {
 5631   assert(isShiftCount(imm8), "illegal shift count");
 5632   int encode = prefix_and_encode(dst->encoding());
 5633   if (imm8 == 1) {
 5634     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5635   } else {
 5636     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5637   }
 5638 }
 5639 
 5640 void Assembler::sall(Register dst) {
 5641   int encode = prefix_and_encode(dst->encoding());
 5642   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5643 }
 5644 
 5645 void Assembler::sarl(Address dst, int imm8) {
 5646   assert(isShiftCount(imm8), "illegal shift count");
 5647   InstructionMark im(this);
 5648   prefix(dst);
 5649   if (imm8 == 1) {
 5650     emit_int8((unsigned char)0xD1);
 5651     emit_operand(as_Register(7), dst, 0);
 5652   }
 5653   else {
 5654     emit_int8((unsigned char)0xC1);
 5655     emit_operand(as_Register(7), dst, 1);
 5656     emit_int8(imm8);
 5657   }
 5658 }
 5659 
 5660 void Assembler::sarl(Address dst) {
 5661   InstructionMark im(this);
 5662   prefix(dst);
 5663   emit_int8((unsigned char)0xD3);
 5664   emit_operand(as_Register(7), dst, 0);
 5665 }
 5666 
 5667 void Assembler::sarl(Register dst, int imm8) {
 5668   int encode = prefix_and_encode(dst->encoding());
 5669   assert(isShiftCount(imm8), "illegal shift count");
 5670   if (imm8 == 1) {
 5671     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5672   } else {
 5673     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5674   }
 5675 }
 5676 
 5677 void Assembler::sarl(Register dst) {
 5678   int encode = prefix_and_encode(dst->encoding());
 5679   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5680 }
 5681 
 5682 void Assembler::sbbl(Address dst, int32_t imm32) {
 5683   InstructionMark im(this);
 5684   prefix(dst);
 5685   emit_arith_operand(0x81, rbx, dst, imm32);
 5686 }
 5687 
 5688 void Assembler::sbbl(Register dst, int32_t imm32) {
 5689   prefix(dst);
 5690   emit_arith(0x81, 0xD8, dst, imm32);
 5691 }
 5692 
 5693 
 5694 void Assembler::sbbl(Register dst, Address src) {
 5695   InstructionMark im(this);
 5696   prefix(src, dst);
 5697   emit_int8(0x1B);
 5698   emit_operand(dst, src, 0);
 5699 }
 5700 
 5701 void Assembler::sbbl(Register dst, Register src) {
 5702   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5703   emit_arith(0x1B, 0xC0, dst, src);
 5704 }
 5705 
 5706 void Assembler::setb(Condition cc, Register dst) {
 5707   assert(0 <= cc && cc < 16, "illegal cc");
 5708   int encode = prefix_and_encode(dst->encoding(), true);
 5709   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5710 }
 5711 
 5712 void Assembler::sete(Register dst) {
 5713   int encode = prefix_and_encode(dst->encoding(), true);
 5714   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5715 }
 5716 
 5717 void Assembler::setl(Register dst) {
 5718   int encode = prefix_and_encode(dst->encoding(), true);
 5719   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5720 }
 5721 
 5722 void Assembler::setne(Register dst) {
 5723   int encode = prefix_and_encode(dst->encoding(), true);
 5724   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5725 }
 5726 
 5727 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5728   assert(VM_Version::supports_ssse3(), "");
 5729   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5730   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5731   emit_int24(0x0F, (0xC0 | encode), imm8);
 5732 }
 5733 
 5734 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5735   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5736          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5737          0, "");
 5738   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5739   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5740   emit_int24(0x0F, (0xC0 | encode), imm8);
 5741 }
 5742 
 5743 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5744   assert(VM_Version::supports_evex(), "");
 5745   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5746   attributes.set_is_evex_instruction();
 5747   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5748   emit_int24(0x3, (0xC0 | encode), imm8);
 5749 }
 5750 
 5751 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5752   assert(VM_Version::supports_sse4_1(), "");
 5753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5754   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5755   emit_int24(0x0E, (0xC0 | encode), imm8);
 5756 }
 5757 
 5758 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5759   assert(VM_Version::supports_sha(), "");
 5760   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5761   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5762 }
 5763 
 5764 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5765   assert(VM_Version::supports_sha(), "");
 5766   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5767   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5768 }
 5769 
 5770 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5771   assert(VM_Version::supports_sha(), "");
 5772   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5773   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5774 }
 5775 
 5776 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5777   assert(VM_Version::supports_sha(), "");
 5778   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5779   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5780 }
 5781 
 5782 // xmm0 is implicit additional source to this instruction.
 5783 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5784   assert(VM_Version::supports_sha(), "");
 5785   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5786   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5787 }
 5788 
 5789 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5790   assert(VM_Version::supports_sha(), "");
 5791   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5792   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5793 }
 5794 
 5795 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5796   assert(VM_Version::supports_sha(), "");
 5797   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5798   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5799 }
 5800 
 5801 
 5802 void Assembler::shll(Register dst, int imm8) {
 5803   assert(isShiftCount(imm8), "illegal shift count");
 5804   int encode = prefix_and_encode(dst->encoding());
 5805   if (imm8 == 1 ) {
 5806     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5807   } else {
 5808     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5809   }
 5810 }
 5811 
 5812 void Assembler::shll(Register dst) {
 5813   int encode = prefix_and_encode(dst->encoding());
 5814   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5815 }
 5816 
 5817 void Assembler::shrl(Register dst, int imm8) {
 5818   assert(isShiftCount(imm8), "illegal shift count");
 5819   int encode = prefix_and_encode(dst->encoding());
 5820   if (imm8 == 1) {
 5821     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5822   }
 5823   else {
 5824     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5825   }
 5826 }
 5827 
 5828 void Assembler::shrl(Register dst) {
 5829   int encode = prefix_and_encode(dst->encoding());
 5830   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5831 }
 5832 
 5833 void Assembler::shrl(Address dst) {
 5834   InstructionMark im(this);
 5835   prefix(dst);
 5836   emit_int8((unsigned char)0xD3);
 5837   emit_operand(as_Register(5), dst, 0);
 5838 }
 5839 
 5840 void Assembler::shrl(Address dst, int imm8) {
 5841   InstructionMark im(this);
 5842   assert(isShiftCount(imm8), "illegal shift count");
 5843   prefix(dst);
 5844   if (imm8 == 1) {
 5845     emit_int8((unsigned char)0xD1);
 5846     emit_operand(as_Register(5), dst, 0);
 5847   }
 5848   else {
 5849     emit_int8((unsigned char)0xC1);
 5850     emit_operand(as_Register(5), dst, 1);
 5851     emit_int8(imm8);
 5852   }
 5853 }
 5854 
 5855 
 5856 void Assembler::shldl(Register dst, Register src) {
 5857   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5858   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5859 }
 5860 
 5861 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5862   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5863   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5864 }
 5865 
 5866 void Assembler::shrdl(Register dst, Register src) {
 5867   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5868   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5869 }
 5870 
 5871 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5872   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5873   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5874 }
 5875 
 5876 // copies a single word from [esi] to [edi]
 5877 void Assembler::smovl() {
 5878   emit_int8((unsigned char)0xA5);
 5879 }
 5880 
 5881 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5882   assert(VM_Version::supports_sse4_1(), "");
 5883   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5884   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5885   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5886 }
 5887 
 5888 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5889   assert(VM_Version::supports_sse4_1(), "");
 5890   InstructionMark im(this);
 5891   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5892   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5893   emit_int8(0x0B);
 5894   emit_operand(dst, src, 1);
 5895   emit_int8((unsigned char)rmode);
 5896 }
 5897 
 5898 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5899   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5900   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5901   attributes.set_rex_vex_w_reverted();
 5902   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5903   emit_int16(0x51, (0xC0 | encode));
 5904 }
 5905 
 5906 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5907   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5908   InstructionMark im(this);
 5909   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5910   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5911   attributes.set_rex_vex_w_reverted();
 5912   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5913   emit_int8(0x51);
 5914   emit_operand(dst, src, 0);
 5915 }
 5916 
 5917 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5918   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5919   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5920   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5921   emit_int16(0x51, (0xC0 | encode));
 5922 }
 5923 
 5924 void Assembler::std() {
 5925   emit_int8((unsigned char)0xFD);
 5926 }
 5927 
 5928 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5929   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5930   InstructionMark im(this);
 5931   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5932   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5933   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5934   emit_int8(0x51);
 5935   emit_operand(dst, src, 0);
 5936 }
 5937 
 5938 void Assembler::stmxcsr( Address dst) {
 5939   if (UseAVX > 0 ) {
 5940     assert(VM_Version::supports_avx(), "");
 5941     InstructionMark im(this);
 5942     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5943     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5944     emit_int8((unsigned char)0xAE);
 5945     emit_operand(as_Register(3), dst, 0);
 5946   } else {
 5947     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5948     InstructionMark im(this);
 5949     prefix(dst);
 5950     emit_int16(0x0F, (unsigned char)0xAE);
 5951     emit_operand(as_Register(3), dst, 0);
 5952   }
 5953 }
 5954 
 5955 void Assembler::subl(Address dst, int32_t imm32) {
 5956   InstructionMark im(this);
 5957   prefix(dst);
 5958   emit_arith_operand(0x81, rbp, dst, imm32);
 5959 }
 5960 
 5961 void Assembler::subl(Address dst, Register src) {
 5962   InstructionMark im(this);
 5963   prefix(dst, src);
 5964   emit_int8(0x29);
 5965   emit_operand(src, dst, 0);
 5966 }
 5967 
 5968 void Assembler::subl(Register dst, int32_t imm32) {
 5969   prefix(dst);
 5970   emit_arith(0x81, 0xE8, dst, imm32);
 5971 }
 5972 
 5973 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5974 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5975   prefix(dst);
 5976   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5977 }
 5978 
 5979 void Assembler::subl(Register dst, Address src) {
 5980   InstructionMark im(this);
 5981   prefix(src, dst);
 5982   emit_int8(0x2B);
 5983   emit_operand(dst, src, 0);
 5984 }
 5985 
 5986 void Assembler::subl(Register dst, Register src) {
 5987   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5988   emit_arith(0x2B, 0xC0, dst, src);
 5989 }
 5990 
 5991 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5993   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5994   attributes.set_rex_vex_w_reverted();
 5995   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5996   emit_int16(0x5C, (0xC0 | encode));
 5997 }
 5998 
 5999 void Assembler::subsd(XMMRegister dst, Address src) {
 6000   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6001   InstructionMark im(this);
 6002   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6003   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6004   attributes.set_rex_vex_w_reverted();
 6005   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6006   emit_int8(0x5C);
 6007   emit_operand(dst, src, 0);
 6008 }
 6009 
 6010 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 6011   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 6013   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6014   emit_int16(0x5C, (0xC0 | encode));
 6015 }
 6016 
 6017 void Assembler::subss(XMMRegister dst, Address src) {
 6018   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6019   InstructionMark im(this);
 6020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6021   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6022   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6023   emit_int8(0x5C);
 6024   emit_operand(dst, src, 0);
 6025 }
 6026 
 6027 void Assembler::testb(Register dst, int imm8) {
 6028   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 6029   if (dst == rax) {
 6030     emit_int8((unsigned char)0xA8);
 6031     emit_int8(imm8);
 6032   } else {
 6033     (void) prefix_and_encode(dst->encoding(), true);
 6034     emit_arith_b(0xF6, 0xC0, dst, imm8);
 6035   }
 6036 }
 6037 
 6038 void Assembler::testb(Address dst, int imm8) {
 6039   InstructionMark im(this);
 6040   prefix(dst);
 6041   emit_int8((unsigned char)0xF6);
 6042   emit_operand(rax, dst, 1);
 6043   emit_int8(imm8);
 6044 }
 6045 
 6046 void Assembler::testl(Address dst, int32_t imm32) {
 6047   InstructionMark im(this);
 6048   prefix(dst);
 6049   emit_int8((unsigned char)0xF7);
 6050   emit_operand(as_Register(0), dst, 4);
 6051   emit_int32(imm32);
 6052 }
 6053 
 6054 void Assembler::testl(Register dst, int32_t imm32) {
 6055   // not using emit_arith because test
 6056   // doesn't support sign-extension of
 6057   // 8bit operands
 6058   if (dst == rax) {
 6059     emit_int8((unsigned char)0xA9);
 6060     emit_int32(imm32);
 6061   } else {
 6062     int encode = dst->encoding();
 6063     encode = prefix_and_encode(encode);
 6064     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 6065     emit_int32(imm32);
 6066   }
 6067 }
 6068 
 6069 void Assembler::testl(Register dst, Register src) {
 6070   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6071   emit_arith(0x85, 0xC0, dst, src);
 6072 }
 6073 
 6074 void Assembler::testl(Register dst, Address src) {
 6075   InstructionMark im(this);
 6076   prefix(src, dst);
 6077   emit_int8((unsigned char)0x85);
 6078   emit_operand(dst, src, 0);
 6079 }
 6080 
 6081 void Assembler::tzcntl(Register dst, Register src) {
 6082   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6083   emit_int8((unsigned char)0xF3);
 6084   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6085   emit_int24(0x0F,
 6086              (unsigned char)0xBC,
 6087              0xC0 | encode);
 6088 }
 6089 
 6090 void Assembler::tzcntl(Register dst, Address src) {
 6091   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6092   InstructionMark im(this);
 6093   emit_int8((unsigned char)0xF3);
 6094   prefix(src, dst);
 6095   emit_int16(0x0F, (unsigned char)0xBC);
 6096   emit_operand(dst, src, 0);
 6097 }
 6098 
 6099 void Assembler::tzcntq(Register dst, Register src) {
 6100   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6101   emit_int8((unsigned char)0xF3);
 6102   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 6103   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 6104 }
 6105 
 6106 void Assembler::tzcntq(Register dst, Address src) {
 6107   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6108   InstructionMark im(this);
 6109   emit_int8((unsigned char)0xF3);
 6110   prefixq(src, dst);
 6111   emit_int16(0x0F, (unsigned char)0xBC);
 6112   emit_operand(dst, src, 0);
 6113 }
 6114 
 6115 void Assembler::ucomisd(XMMRegister dst, Address src) {
 6116   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6117   InstructionMark im(this);
 6118   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6119   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6120   attributes.set_rex_vex_w_reverted();
 6121   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6122   emit_int8(0x2E);
 6123   emit_operand(dst, src, 0);
 6124 }
 6125 
 6126 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6127   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6128   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6129   attributes.set_rex_vex_w_reverted();
 6130   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6131   emit_int16(0x2E, (0xC0 | encode));
 6132 }
 6133 
 6134 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6135   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6136   InstructionMark im(this);
 6137   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6138   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6139   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6140   emit_int8(0x2E);
 6141   emit_operand(dst, src, 0);
 6142 }
 6143 
 6144 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6145   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6146   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6147   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6148   emit_int16(0x2E, (0xC0 | encode));
 6149 }
 6150 
 6151 void Assembler::xabort(int8_t imm8) {
 6152   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6153 }
 6154 
 6155 void Assembler::xaddb(Address dst, Register src) {
 6156   InstructionMark im(this);
 6157   prefix(dst, src, true);
 6158   emit_int16(0x0F, (unsigned char)0xC0);
 6159   emit_operand(src, dst, 0);
 6160 }
 6161 
 6162 void Assembler::xaddw(Address dst, Register src) {
 6163   InstructionMark im(this);
 6164   emit_int8(0x66);
 6165   prefix(dst, src);
 6166   emit_int16(0x0F, (unsigned char)0xC1);
 6167   emit_operand(src, dst, 0);
 6168 }
 6169 
 6170 void Assembler::xaddl(Address dst, Register src) {
 6171   InstructionMark im(this);
 6172   prefix(dst, src);
 6173   emit_int16(0x0F, (unsigned char)0xC1);
 6174   emit_operand(src, dst, 0);
 6175 }
 6176 
 6177 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6178   InstructionMark im(this);
 6179   relocate(rtype);
 6180   if (abort.is_bound()) {
 6181     address entry = target(abort);
 6182     assert(entry != NULL, "abort entry NULL");
 6183     intptr_t offset = entry - pc();
 6184     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6185     emit_int32(offset - 6); // 2 opcode + 4 address
 6186   } else {
 6187     abort.add_patch_at(code(), locator());
 6188     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6189     emit_int32(0);
 6190   }
 6191 }
 6192 
 6193 void Assembler::xchgb(Register dst, Address src) { // xchg
 6194   InstructionMark im(this);
 6195   prefix(src, dst, true);
 6196   emit_int8((unsigned char)0x86);
 6197   emit_operand(dst, src, 0);
 6198 }
 6199 
 6200 void Assembler::xchgw(Register dst, Address src) { // xchg
 6201   InstructionMark im(this);
 6202   emit_int8(0x66);
 6203   prefix(src, dst);
 6204   emit_int8((unsigned char)0x87);
 6205   emit_operand(dst, src, 0);
 6206 }
 6207 
 6208 void Assembler::xchgl(Register dst, Address src) { // xchg
 6209   InstructionMark im(this);
 6210   prefix(src, dst);
 6211   emit_int8((unsigned char)0x87);
 6212   emit_operand(dst, src, 0);
 6213 }
 6214 
 6215 void Assembler::xchgl(Register dst, Register src) {
 6216   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6217   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6218 }
 6219 
 6220 void Assembler::xend() {
 6221   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6222 }
 6223 
 6224 void Assembler::xgetbv() {
 6225   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6226 }
 6227 
 6228 void Assembler::xorl(Address dst, int32_t imm32) {
 6229   InstructionMark im(this);
 6230   prefix(dst);
 6231   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6232 }
 6233 
 6234 void Assembler::xorl(Register dst, int32_t imm32) {
 6235   prefix(dst);
 6236   emit_arith(0x81, 0xF0, dst, imm32);
 6237 }
 6238 
 6239 void Assembler::xorl(Register dst, Address src) {
 6240   InstructionMark im(this);
 6241   prefix(src, dst);
 6242   emit_int8(0x33);
 6243   emit_operand(dst, src, 0);
 6244 }
 6245 
 6246 void Assembler::xorl(Register dst, Register src) {
 6247   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6248   emit_arith(0x33, 0xC0, dst, src);
 6249 }
 6250 
 6251 void Assembler::xorl(Address dst, Register src) {
 6252   InstructionMark im(this);
 6253   prefix(dst, src);
 6254   emit_int8(0x31);
 6255   emit_operand(src, dst, 0);
 6256 }
 6257 
 6258 void Assembler::xorb(Register dst, Address src) {
 6259   InstructionMark im(this);
 6260   prefix(src, dst);
 6261   emit_int8(0x32);
 6262   emit_operand(dst, src, 0);
 6263 }
 6264 
 6265 void Assembler::xorb(Address dst, Register src) {
 6266   InstructionMark im(this);
 6267   prefix(dst, src, true);
 6268   emit_int8(0x30);
 6269   emit_operand(src, dst, 0);
 6270 }
 6271 
 6272 void Assembler::xorw(Register dst, Register src) {
 6273   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6274   emit_arith(0x33, 0xC0, dst, src);
 6275 }
 6276 
 6277 // AVX 3-operands scalar float-point arithmetic instructions
 6278 
 6279 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6280   assert(VM_Version::supports_avx(), "");
 6281   InstructionMark im(this);
 6282   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6283   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6284   attributes.set_rex_vex_w_reverted();
 6285   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6286   emit_int8(0x58);
 6287   emit_operand(dst, src, 0);
 6288 }
 6289 
 6290 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6291   assert(VM_Version::supports_avx(), "");
 6292   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6293   attributes.set_rex_vex_w_reverted();
 6294   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6295   emit_int16(0x58, (0xC0 | encode));
 6296 }
 6297 
 6298 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6299   assert(VM_Version::supports_avx(), "");
 6300   InstructionMark im(this);
 6301   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6302   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6303   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6304   emit_int8(0x58);
 6305   emit_operand(dst, src, 0);
 6306 }
 6307 
 6308 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6309   assert(VM_Version::supports_avx(), "");
 6310   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6311   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6312   emit_int16(0x58, (0xC0 | encode));
 6313 }
 6314 
 6315 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6316   assert(VM_Version::supports_avx(), "");
 6317   InstructionMark im(this);
 6318   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6319   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6320   attributes.set_rex_vex_w_reverted();
 6321   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6322   emit_int8(0x5E);
 6323   emit_operand(dst, src, 0);
 6324 }
 6325 
 6326 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6327   assert(VM_Version::supports_avx(), "");
 6328   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6329   attributes.set_rex_vex_w_reverted();
 6330   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6331   emit_int16(0x5E, (0xC0 | encode));
 6332 }
 6333 
 6334 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6335   assert(VM_Version::supports_avx(), "");
 6336   InstructionMark im(this);
 6337   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6338   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6339   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6340   emit_int8(0x5E);
 6341   emit_operand(dst, src, 0);
 6342 }
 6343 
 6344 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6345   assert(VM_Version::supports_avx(), "");
 6346   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6347   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6348   emit_int16(0x5E, (0xC0 | encode));
 6349 }
 6350 
 6351 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6352   assert(VM_Version::supports_fma(), "");
 6353   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6354   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6355   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6356 }
 6357 
 6358 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6359   assert(VM_Version::supports_fma(), "");
 6360   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6361   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6362   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6363 }
 6364 
 6365 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6366   assert(VM_Version::supports_avx(), "");
 6367   InstructionMark im(this);
 6368   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6369   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6370   attributes.set_rex_vex_w_reverted();
 6371   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6372   emit_int8(0x59);
 6373   emit_operand(dst, src, 0);
 6374 }
 6375 
 6376 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6377   assert(VM_Version::supports_avx(), "");
 6378   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6379   attributes.set_rex_vex_w_reverted();
 6380   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6381   emit_int16(0x59, (0xC0 | encode));
 6382 }
 6383 
 6384 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6385   assert(VM_Version::supports_avx(), "");
 6386   InstructionMark im(this);
 6387   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6388   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6389   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6390   emit_int8(0x59);
 6391   emit_operand(dst, src, 0);
 6392 }
 6393 
 6394 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6395   assert(VM_Version::supports_avx(), "");
 6396   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6397   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6398   emit_int16(0x59, (0xC0 | encode));
 6399 }
 6400 
 6401 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6402   assert(VM_Version::supports_avx(), "");
 6403   InstructionMark im(this);
 6404   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6405   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6406   attributes.set_rex_vex_w_reverted();
 6407   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6408   emit_int8(0x5C);
 6409   emit_operand(dst, src, 0);
 6410 }
 6411 
 6412 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6413   assert(VM_Version::supports_avx(), "");
 6414   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6415   attributes.set_rex_vex_w_reverted();
 6416   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6417   emit_int16(0x5C, (0xC0 | encode));
 6418 }
 6419 
 6420 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6421   assert(VM_Version::supports_avx(), "");
 6422   InstructionMark im(this);
 6423   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6424   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6425   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6426   emit_int8(0x5C);
 6427   emit_operand(dst, src, 0);
 6428 }
 6429 
 6430 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6431   assert(VM_Version::supports_avx(), "");
 6432   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6433   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6434   emit_int16(0x5C, (0xC0 | encode));
 6435 }
 6436 
 6437 //====================VECTOR ARITHMETIC=====================================
 6438 
 6439 // Float-point vector arithmetic
 6440 
 6441 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6442   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6443   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6444   attributes.set_rex_vex_w_reverted();
 6445   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6446   emit_int16(0x58, (0xC0 | encode));
 6447 }
 6448 
 6449 void Assembler::addpd(XMMRegister dst, Address src) {
 6450   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6451   InstructionMark im(this);
 6452   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6453   attributes.set_rex_vex_w_reverted();
 6454   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6455   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6456   emit_int8(0x58);
 6457   emit_operand(dst, src, 0);
 6458 }
 6459 
 6460 
 6461 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6462   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6464   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6465   emit_int16(0x58, (0xC0 | encode));
 6466 }
 6467 
 6468 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6469   assert(VM_Version::supports_avx(), "");
 6470   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6471   attributes.set_rex_vex_w_reverted();
 6472   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6473   emit_int16(0x58, (0xC0 | encode));
 6474 }
 6475 
 6476 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6477   assert(VM_Version::supports_avx(), "");
 6478   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6479   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6480   emit_int16(0x58, (0xC0 | encode));
 6481 }
 6482 
 6483 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6484   assert(VM_Version::supports_avx(), "");
 6485   InstructionMark im(this);
 6486   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6487   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6488   attributes.set_rex_vex_w_reverted();
 6489   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6490   emit_int8(0x58);
 6491   emit_operand(dst, src, 0);
 6492 }
 6493 
 6494 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6495   assert(VM_Version::supports_avx(), "");
 6496   InstructionMark im(this);
 6497   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6498   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6499   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6500   emit_int8(0x58);
 6501   emit_operand(dst, src, 0);
 6502 }
 6503 
 6504 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6505   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6506   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6507   attributes.set_rex_vex_w_reverted();
 6508   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6509   emit_int16(0x5C, (0xC0 | encode));
 6510 }
 6511 
 6512 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6513   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6514   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6515   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6516   emit_int16(0x5C, (0xC0 | encode));
 6517 }
 6518 
 6519 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6520   assert(VM_Version::supports_avx(), "");
 6521   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6522   attributes.set_rex_vex_w_reverted();
 6523   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6524   emit_int16(0x5C, (0xC0 | encode));
 6525 }
 6526 
 6527 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6528   assert(VM_Version::supports_avx(), "");
 6529   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6530   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6531   emit_int16(0x5C, (0xC0 | encode));
 6532 }
 6533 
 6534 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6535   assert(VM_Version::supports_avx(), "");
 6536   InstructionMark im(this);
 6537   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6538   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6539   attributes.set_rex_vex_w_reverted();
 6540   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6541   emit_int8(0x5C);
 6542   emit_operand(dst, src, 0);
 6543 }
 6544 
 6545 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6546   assert(VM_Version::supports_avx(), "");
 6547   InstructionMark im(this);
 6548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6549   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6550   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6551   emit_int8(0x5C);
 6552   emit_operand(dst, src, 0);
 6553 }
 6554 
 6555 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6556   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6557   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6558   attributes.set_rex_vex_w_reverted();
 6559   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6560   emit_int16(0x59, (0xC0 | encode));
 6561 }
 6562 
 6563 void Assembler::mulpd(XMMRegister dst, Address src) {
 6564   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6565   InstructionMark im(this);
 6566   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6567   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6568   attributes.set_rex_vex_w_reverted();
 6569   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6570   emit_int8(0x59);
 6571   emit_operand(dst, src, 0);
 6572 }
 6573 
 6574 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6575   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6576   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6577   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6578   emit_int16(0x59, (0xC0 | encode));
 6579 }
 6580 
 6581 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6582   assert(VM_Version::supports_avx(), "");
 6583   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6584   attributes.set_rex_vex_w_reverted();
 6585   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6586   emit_int16(0x59, (0xC0 | encode));
 6587 }
 6588 
 6589 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6590   assert(VM_Version::supports_avx(), "");
 6591   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6592   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6593   emit_int16(0x59, (0xC0 | encode));
 6594 }
 6595 
 6596 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6597   assert(VM_Version::supports_avx(), "");
 6598   InstructionMark im(this);
 6599   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6600   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6601   attributes.set_rex_vex_w_reverted();
 6602   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6603   emit_int8(0x59);
 6604   emit_operand(dst, src, 0);
 6605 }
 6606 
 6607 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6608   assert(VM_Version::supports_avx(), "");
 6609   InstructionMark im(this);
 6610   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6611   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6612   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6613   emit_int8(0x59);
 6614   emit_operand(dst, src, 0);
 6615 }
 6616 
 6617 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6618   assert(VM_Version::supports_fma(), "");
 6619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6620   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6621   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6622 }
 6623 
 6624 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6625   assert(VM_Version::supports_fma(), "");
 6626   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6627   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6628   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6629 }
 6630 
 6631 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6632   assert(VM_Version::supports_fma(), "");
 6633   InstructionMark im(this);
 6634   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6635   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6636   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6637   emit_int8((unsigned char)0xB8);
 6638   emit_operand(dst, src2, 0);
 6639 }
 6640 
 6641 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6642   assert(VM_Version::supports_fma(), "");
 6643   InstructionMark im(this);
 6644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6645   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6646   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6647   emit_int8((unsigned char)0xB8);
 6648   emit_operand(dst, src2, 0);
 6649 }
 6650 
 6651 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6652   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6653   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6654   attributes.set_rex_vex_w_reverted();
 6655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6656   emit_int16(0x5E, (0xC0 | encode));
 6657 }
 6658 
 6659 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6660   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6661   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6662   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6663   emit_int16(0x5E, (0xC0 | encode));
 6664 }
 6665 
 6666 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6667   assert(VM_Version::supports_avx(), "");
 6668   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6669   attributes.set_rex_vex_w_reverted();
 6670   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6671   emit_int16(0x5E, (0xC0 | encode));
 6672 }
 6673 
 6674 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6675   assert(VM_Version::supports_avx(), "");
 6676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6678   emit_int16(0x5E, (0xC0 | encode));
 6679 }
 6680 
 6681 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6682   assert(VM_Version::supports_avx(), "");
 6683   InstructionMark im(this);
 6684   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6685   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6686   attributes.set_rex_vex_w_reverted();
 6687   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6688   emit_int8(0x5E);
 6689   emit_operand(dst, src, 0);
 6690 }
 6691 
 6692 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6693   assert(VM_Version::supports_avx(), "");
 6694   InstructionMark im(this);
 6695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6696   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6697   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6698   emit_int8(0x5E);
 6699   emit_operand(dst, src, 0);
 6700 }
 6701 
 6702 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6703   assert(VM_Version::supports_avx(), "");
 6704   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6705   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6706   emit_int24(0x09, (0xC0 | encode), (rmode));
 6707 }
 6708 
 6709 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6710   assert(VM_Version::supports_avx(), "");
 6711   InstructionMark im(this);
 6712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6713   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6714   emit_int8(0x09);
 6715   emit_operand(dst, src, 1);
 6716   emit_int8((rmode));
 6717 }
 6718 
 6719 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6720   assert(VM_Version::supports_evex(), "requires EVEX support");
 6721   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6722   attributes.set_is_evex_instruction();
 6723   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6724   emit_int24(0x09, (0xC0 | encode), (rmode));
 6725 }
 6726 
 6727 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6728   assert(VM_Version::supports_evex(), "requires EVEX support");
 6729   assert(dst != xnoreg, "sanity");
 6730   InstructionMark im(this);
 6731   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6732   attributes.set_is_evex_instruction();
 6733   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6734   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6735   emit_int8(0x09);
 6736   emit_operand(dst, src, 1);
 6737   emit_int8((rmode));
 6738 }
 6739 
 6740 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6741   assert(VM_Version::supports_avx(), "");
 6742   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6743   attributes.set_rex_vex_w_reverted();
 6744   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6745   emit_int16(0x51, (0xC0 | encode));
 6746 }
 6747 
 6748 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6749   assert(VM_Version::supports_avx(), "");
 6750   InstructionMark im(this);
 6751   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6752   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6753   attributes.set_rex_vex_w_reverted();
 6754   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6755   emit_int8(0x51);
 6756   emit_operand(dst, src, 0);
 6757 }
 6758 
 6759 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6760   assert(VM_Version::supports_avx(), "");
 6761   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6762   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6763   emit_int16(0x51, (0xC0 | encode));
 6764 }
 6765 
 6766 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6767   assert(VM_Version::supports_avx(), "");
 6768   InstructionMark im(this);
 6769   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6770   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6771   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6772   emit_int8(0x51);
 6773   emit_operand(dst, src, 0);
 6774 }
 6775 
 6776 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6778   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6779   attributes.set_rex_vex_w_reverted();
 6780   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6781   emit_int16(0x54, (0xC0 | encode));
 6782 }
 6783 
 6784 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6785   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6786   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6787   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6788   emit_int16(0x54, (0xC0 | encode));
 6789 }
 6790 
 6791 void Assembler::andps(XMMRegister dst, Address src) {
 6792   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6793   InstructionMark im(this);
 6794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6795   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6796   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6797   emit_int8(0x54);
 6798   emit_operand(dst, src, 0);
 6799 }
 6800 
 6801 void Assembler::andpd(XMMRegister dst, Address src) {
 6802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6803   InstructionMark im(this);
 6804   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6805   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6806   attributes.set_rex_vex_w_reverted();
 6807   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6808   emit_int8(0x54);
 6809   emit_operand(dst, src, 0);
 6810 }
 6811 
 6812 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6813   assert(VM_Version::supports_avx(), "");
 6814   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6815   attributes.set_rex_vex_w_reverted();
 6816   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6817   emit_int16(0x54, (0xC0 | encode));
 6818 }
 6819 
 6820 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6821   assert(VM_Version::supports_avx(), "");
 6822   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6824   emit_int16(0x54, (0xC0 | encode));
 6825 }
 6826 
 6827 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6828   assert(VM_Version::supports_avx(), "");
 6829   InstructionMark im(this);
 6830   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6832   attributes.set_rex_vex_w_reverted();
 6833   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6834   emit_int8(0x54);
 6835   emit_operand(dst, src, 0);
 6836 }
 6837 
 6838 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6839   assert(VM_Version::supports_avx(), "");
 6840   InstructionMark im(this);
 6841   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6842   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6843   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6844   emit_int8(0x54);
 6845   emit_operand(dst, src, 0);
 6846 }
 6847 
 6848 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6849   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6850   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6851   attributes.set_rex_vex_w_reverted();
 6852   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6853   emit_int8(0x15);
 6854   emit_int8((0xC0 | encode));
 6855 }
 6856 
 6857 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6858   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6859   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6860   attributes.set_rex_vex_w_reverted();
 6861   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6862   emit_int16(0x14, (0xC0 | encode));
 6863 }
 6864 
 6865 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6866   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6867   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6868   attributes.set_rex_vex_w_reverted();
 6869   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6870   emit_int16(0x57, (0xC0 | encode));
 6871 }
 6872 
 6873 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6874   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6875   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6876   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6877   emit_int16(0x57, (0xC0 | encode));
 6878 }
 6879 
 6880 void Assembler::xorpd(XMMRegister dst, Address src) {
 6881   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6882   InstructionMark im(this);
 6883   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6884   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6885   attributes.set_rex_vex_w_reverted();
 6886   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6887   emit_int8(0x57);
 6888   emit_operand(dst, src, 0);
 6889 }
 6890 
 6891 void Assembler::xorps(XMMRegister dst, Address src) {
 6892   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6893   InstructionMark im(this);
 6894   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6895   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6896   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6897   emit_int8(0x57);
 6898   emit_operand(dst, src, 0);
 6899 }
 6900 
 6901 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6902   assert(VM_Version::supports_avx(), "");
 6903   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6904   attributes.set_rex_vex_w_reverted();
 6905   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6906   emit_int16(0x57, (0xC0 | encode));
 6907 }
 6908 
 6909 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6910   assert(VM_Version::supports_avx(), "");
 6911   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6912   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6913   emit_int16(0x57, (0xC0 | encode));
 6914 }
 6915 
 6916 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6917   assert(VM_Version::supports_avx(), "");
 6918   InstructionMark im(this);
 6919   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6920   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6921   attributes.set_rex_vex_w_reverted();
 6922   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6923   emit_int8(0x57);
 6924   emit_operand(dst, src, 0);
 6925 }
 6926 
 6927 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6928   assert(VM_Version::supports_avx(), "");
 6929   InstructionMark im(this);
 6930   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6931   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6932   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6933   emit_int8(0x57);
 6934   emit_operand(dst, src, 0);
 6935 }
 6936 
 6937 // Integer vector arithmetic
 6938 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6939   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6940          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6941   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6942   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6943   emit_int16(0x01, (0xC0 | encode));
 6944 }
 6945 
 6946 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6947   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6948          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6949   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6950   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6951   emit_int16(0x02, (0xC0 | encode));
 6952 }
 6953 
 6954 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6955   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6956   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6957   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6958   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6959 }
 6960 
 6961 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6962   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6964   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6965   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6966 }
 6967 
 6968 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6970   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6971   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6972   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6973 }
 6974 
 6975 void Assembler::paddd(XMMRegister dst, Address src) {
 6976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6977   InstructionMark im(this);
 6978   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6979   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6980   emit_int8((unsigned char)0xFE);
 6981   emit_operand(dst, src, 0);
 6982 }
 6983 
 6984 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6986   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6987   attributes.set_rex_vex_w_reverted();
 6988   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6989   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6990 }
 6991 
 6992 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6993   assert(VM_Version::supports_sse3(), "");
 6994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6995   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6996   emit_int16(0x01, (0xC0 | encode));
 6997 }
 6998 
 6999 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 7000   assert(VM_Version::supports_sse3(), "");
 7001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7002   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7003   emit_int16(0x02, (0xC0 | encode));
 7004 }
 7005 
 7006 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7007   assert(UseAVX > 0, "requires some form of AVX");
 7008   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7009   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7010   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7011 }
 7012 
 7013 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7014   assert(UseAVX > 0, "requires some form of AVX");
 7015   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7016   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7017   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7018 }
 7019 
 7020 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7021   assert(UseAVX > 0, "requires some form of AVX");
 7022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7023   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7024   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7025 }
 7026 
 7027 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7028   assert(UseAVX > 0, "requires some form of AVX");
 7029   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7030   attributes.set_rex_vex_w_reverted();
 7031   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7032   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7033 }
 7034 
 7035 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7036   assert(UseAVX > 0, "requires some form of AVX");
 7037   InstructionMark im(this);
 7038   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7039   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7040   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7041   emit_int8((unsigned char)0xFC);
 7042   emit_operand(dst, src, 0);
 7043 }
 7044 
 7045 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7046   assert(UseAVX > 0, "requires some form of AVX");
 7047   InstructionMark im(this);
 7048   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7049   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7050   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7051   emit_int8((unsigned char)0xFD);
 7052   emit_operand(dst, src, 0);
 7053 }
 7054 
 7055 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7056   assert(UseAVX > 0, "requires some form of AVX");
 7057   InstructionMark im(this);
 7058   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7059   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7060   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7061   emit_int8((unsigned char)0xFE);
 7062   emit_operand(dst, src, 0);
 7063 }
 7064 
 7065 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7066   assert(UseAVX > 0, "requires some form of AVX");
 7067   InstructionMark im(this);
 7068   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7069   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7070   attributes.set_rex_vex_w_reverted();
 7071   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7072   emit_int8((unsigned char)0xD4);
 7073   emit_operand(dst, src, 0);
 7074 }
 7075 
 7076 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 7077   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7078   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7079   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7080   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7081 }
 7082 
 7083 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 7084   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7085   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7086   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7087   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7088 }
 7089 
 7090 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 7091   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7092   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7093   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7094 }
 7095 
 7096 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 7097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7098   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7099   attributes.set_rex_vex_w_reverted();
 7100   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7101   emit_int8((unsigned char)0xFB);
 7102   emit_int8((0xC0 | encode));
 7103 }
 7104 
 7105 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7106   assert(UseAVX > 0, "requires some form of AVX");
 7107   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7108   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7109   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 7110 }
 7111 
 7112 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7113   assert(UseAVX > 0, "requires some form of AVX");
 7114   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7115   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7116   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7117 }
 7118 
 7119 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7120   assert(UseAVX > 0, "requires some form of AVX");
 7121   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7122   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7123   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7124 }
 7125 
 7126 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7127   assert(UseAVX > 0, "requires some form of AVX");
 7128   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7129   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7130   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7131 }
 7132 
 7133 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7134   assert(UseAVX > 0, "requires some form of AVX");
 7135   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7136   attributes.set_rex_vex_w_reverted();
 7137   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7138   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7139 }
 7140 
 7141 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7142   assert(UseAVX > 0, "requires some form of AVX");
 7143   InstructionMark im(this);
 7144   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7145   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7146   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7147   emit_int8((unsigned char)0xF8);
 7148   emit_operand(dst, src, 0);
 7149 }
 7150 
 7151 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7152   assert(UseAVX > 0, "requires some form of AVX");
 7153   InstructionMark im(this);
 7154   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7155   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7156   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7157   emit_int8((unsigned char)0xF9);
 7158   emit_operand(dst, src, 0);
 7159 }
 7160 
 7161 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7162   assert(UseAVX > 0, "requires some form of AVX");
 7163   InstructionMark im(this);
 7164   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7165   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7166   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7167   emit_int8((unsigned char)0xFA);
 7168   emit_operand(dst, src, 0);
 7169 }
 7170 
 7171 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7172   assert(UseAVX > 0, "requires some form of AVX");
 7173   InstructionMark im(this);
 7174   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7175   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7176   attributes.set_rex_vex_w_reverted();
 7177   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7178   emit_int8((unsigned char)0xFB);
 7179   emit_operand(dst, src, 0);
 7180 }
 7181 
 7182 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7183   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7184   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7185   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7186   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7187 }
 7188 
 7189 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7190   assert(VM_Version::supports_sse4_1(), "");
 7191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7192   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7193   emit_int16(0x40, (0xC0 | encode));
 7194 }
 7195 
 7196 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7197   assert(VM_Version::supports_sse2(), "");
 7198   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7199   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7200   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7201 }
 7202 
 7203 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7204   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7205          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7206          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7207   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7208   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7209   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7210 }
 7211 
 7212 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7213   assert(UseAVX > 0, "requires some form of AVX");
 7214   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7216   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7217 }
 7218 
 7219 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7220   assert(UseAVX > 0, "requires some form of AVX");
 7221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7222   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7223   emit_int16(0x40, (0xC0 | encode));
 7224 }
 7225 
 7226 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7227   assert(UseAVX > 2, "requires some form of EVEX");
 7228   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7229   attributes.set_is_evex_instruction();
 7230   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7231   emit_int16(0x40, (0xC0 | encode));
 7232 }
 7233 
 7234 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7235   assert(UseAVX > 0, "requires some form of AVX");
 7236   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7237   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7238   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7239 }
 7240 
 7241 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7242   assert(UseAVX > 0, "requires some form of AVX");
 7243   InstructionMark im(this);
 7244   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7245   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7246   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7247   emit_int8((unsigned char)0xD5);
 7248   emit_operand(dst, src, 0);
 7249 }
 7250 
 7251 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7252   assert(UseAVX > 0, "requires some form of AVX");
 7253   InstructionMark im(this);
 7254   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7255   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7256   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7257   emit_int8(0x40);
 7258   emit_operand(dst, src, 0);
 7259 }
 7260 
 7261 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7262   assert(UseAVX > 2, "requires some form of EVEX");
 7263   InstructionMark im(this);
 7264   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7265   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7266   attributes.set_is_evex_instruction();
 7267   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7268   emit_int8(0x40);
 7269   emit_operand(dst, src, 0);
 7270 }
 7271 
 7272 // Min, max
 7273 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7274   assert(VM_Version::supports_sse4_1(), "");
 7275   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7276   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7277   emit_int16(0x38, (0xC0 | encode));
 7278 }
 7279 
 7280 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7281   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7282         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7284   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7285   emit_int16(0x38, (0xC0 | encode));
 7286 }
 7287 
 7288 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7289   assert(VM_Version::supports_sse2(), "");
 7290   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7291   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7292   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7293 }
 7294 
 7295 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7296   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7297         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7299   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7300   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7301 }
 7302 
 7303 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7304   assert(VM_Version::supports_sse4_1(), "");
 7305   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7306   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7307   emit_int16(0x39, (0xC0 | encode));
 7308 }
 7309 
 7310 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7311   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7312         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7313   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7314   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7315   emit_int16(0x39, (0xC0 | encode));
 7316 }
 7317 
 7318 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7319   assert(UseAVX > 2, "requires AVX512F");
 7320   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7321   attributes.set_is_evex_instruction();
 7322   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7323   emit_int16(0x39, (0xC0 | encode));
 7324 }
 7325 
 7326 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7327   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7328   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7329   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7330   emit_int16(0x5D, (0xC0 | encode));
 7331 }
 7332 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7333   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7334   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7335   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7336   emit_int16(0x5D, (0xC0 | encode));
 7337 }
 7338 
 7339 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7340   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7343   emit_int16(0x5D, (0xC0 | encode));
 7344 }
 7345 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7346   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7347   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7348   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7349   emit_int16(0x5D, (0xC0 | encode));
 7350 }
 7351 
 7352 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7353   assert(VM_Version::supports_sse4_1(), "");
 7354   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7355   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7356   emit_int16(0x3C, (0xC0 | encode));
 7357 }
 7358 
 7359 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7360   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7361         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7363   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7364   emit_int16(0x3C, (0xC0 | encode));
 7365 }
 7366 
 7367 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7368   assert(VM_Version::supports_sse2(), "");
 7369   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7370   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7371   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7372 }
 7373 
 7374 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7375   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7376         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7377   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7378   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7379   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7380 }
 7381 
 7382 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7383   assert(VM_Version::supports_sse4_1(), "");
 7384   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7385   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7386   emit_int16(0x3D, (0xC0 | encode));
 7387 }
 7388 
 7389 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7390   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7391         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7392   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7393   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7394   emit_int16(0x3D, (0xC0 | encode));
 7395 }
 7396 
 7397 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7398   assert(UseAVX > 2, "requires AVX512F");
 7399   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7400   attributes.set_is_evex_instruction();
 7401   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7402   emit_int16(0x3D, (0xC0 | encode));
 7403 }
 7404 
 7405 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7406   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7407   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7408   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7409   emit_int16(0x5F, (0xC0 | encode));
 7410 }
 7411 
 7412 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7413   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7414   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7415   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7416   emit_int16(0x5F, (0xC0 | encode));
 7417 }
 7418 
 7419 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7420   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7421   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7422   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7423   emit_int16(0x5F, (0xC0 | encode));
 7424 }
 7425 
 7426 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7427   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7428   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7429   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7430   emit_int16(0x5F, (0xC0 | encode));
 7431 }
 7432 
 7433 // Shift packed integers left by specified number of bits.
 7434 void Assembler::psllw(XMMRegister dst, int shift) {
 7435   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7436   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7437   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7438   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7439   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7440 }
 7441 
 7442 void Assembler::pslld(XMMRegister dst, int shift) {
 7443   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7444   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7445   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7446   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7447   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7448 }
 7449 
 7450 void Assembler::psllq(XMMRegister dst, int shift) {
 7451   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7452   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7453   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7454   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7455   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7456 }
 7457 
 7458 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7460   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7461   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7462   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7463 }
 7464 
 7465 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7466   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7468   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7469   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7470 }
 7471 
 7472 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7473   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7474   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7475   attributes.set_rex_vex_w_reverted();
 7476   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7477   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7478 }
 7479 
 7480 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7481   assert(UseAVX > 0, "requires some form of AVX");
 7482   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7483   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7484   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7485   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7486 }
 7487 
 7488 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7489   assert(UseAVX > 0, "requires some form of AVX");
 7490   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7491   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7492   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7493   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7494   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7495 }
 7496 
 7497 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7498   assert(UseAVX > 0, "requires some form of AVX");
 7499   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7500   attributes.set_rex_vex_w_reverted();
 7501   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7502   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7503   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7504 }
 7505 
 7506 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7507   assert(UseAVX > 0, "requires some form of AVX");
 7508   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7509   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7510   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7511 }
 7512 
 7513 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7514   assert(UseAVX > 0, "requires some form of AVX");
 7515   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7516   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7517   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7518 }
 7519 
 7520 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7521   assert(UseAVX > 0, "requires some form of AVX");
 7522   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7523   attributes.set_rex_vex_w_reverted();
 7524   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7525   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7526 }
 7527 
 7528 // Shift packed integers logically right by specified number of bits.
 7529 void Assembler::psrlw(XMMRegister dst, int shift) {
 7530   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7531   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7532   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7533   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7534   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7535 }
 7536 
 7537 void Assembler::psrld(XMMRegister dst, int shift) {
 7538   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7539   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7540   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7541   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7542   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7543 }
 7544 
 7545 void Assembler::psrlq(XMMRegister dst, int shift) {
 7546   // Do not confuse it with psrldq SSE2 instruction which
 7547   // shifts 128 bit value in xmm register by number of bytes.
 7548   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7549   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7550   attributes.set_rex_vex_w_reverted();
 7551   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7552   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7553   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7554 }
 7555 
 7556 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7557   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7558   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7559   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7560   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7561 }
 7562 
 7563 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7564   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7565   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7566   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7567   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7568 }
 7569 
 7570 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7571   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7572   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7573   attributes.set_rex_vex_w_reverted();
 7574   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7575   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7576 }
 7577 
 7578 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7579   assert(UseAVX > 0, "requires some form of AVX");
 7580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7581   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7582   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7583   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7584 }
 7585 
 7586 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7587   assert(UseAVX > 0, "requires some form of AVX");
 7588   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7589   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7590   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7591   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7592 }
 7593 
 7594 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7595   assert(UseAVX > 0, "requires some form of AVX");
 7596   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7597   attributes.set_rex_vex_w_reverted();
 7598   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7599   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7600   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7601 }
 7602 
 7603 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7604   assert(UseAVX > 0, "requires some form of AVX");
 7605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7606   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7607   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7608 }
 7609 
 7610 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7611   assert(UseAVX > 0, "requires some form of AVX");
 7612   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7613   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7614   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7615 }
 7616 
 7617 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7618   assert(UseAVX > 0, "requires some form of AVX");
 7619   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7620   attributes.set_rex_vex_w_reverted();
 7621   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7622   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7623 }
 7624 
 7625 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7626   assert(VM_Version::supports_avx512bw(), "");
 7627   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7628   attributes.set_is_evex_instruction();
 7629   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7630   emit_int16(0x10, (0xC0 | encode));
 7631 }
 7632 
 7633 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7634   assert(VM_Version::supports_avx512bw(), "");
 7635   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7636   attributes.set_is_evex_instruction();
 7637   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7638   emit_int16(0x12, (0xC0 | encode));
 7639 }
 7640 
 7641 // Shift packed integers arithmetically right by specified number of bits.
 7642 void Assembler::psraw(XMMRegister dst, int shift) {
 7643   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7644   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7645   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7646   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7647   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7648 }
 7649 
 7650 void Assembler::psrad(XMMRegister dst, int shift) {
 7651   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7652   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7653   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7654   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7655   emit_int8(0x72);
 7656   emit_int8((0xC0 | encode));
 7657   emit_int8(shift & 0xFF);
 7658 }
 7659 
 7660 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7663   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7664   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7665 }
 7666 
 7667 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7669   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7670   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7671   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7672 }
 7673 
 7674 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7675   assert(UseAVX > 0, "requires some form of AVX");
 7676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7677   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7678   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7679   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7680 }
 7681 
 7682 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7683   assert(UseAVX > 0, "requires some form of AVX");
 7684   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7685   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7686   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7687   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7688 }
 7689 
 7690 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7691   assert(UseAVX > 0, "requires some form of AVX");
 7692   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7693   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7694   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7695 }
 7696 
 7697 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7698   assert(UseAVX > 0, "requires some form of AVX");
 7699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7700   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7701   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7702 }
 7703 
 7704 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7705   assert(UseAVX > 2, "requires AVX512");
 7706   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7707   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7708   attributes.set_is_evex_instruction();
 7709   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7710   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7711 }
 7712 
 7713 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7714   assert(UseAVX > 2, "requires AVX512");
 7715   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7716   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7717   attributes.set_is_evex_instruction();
 7718   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7719   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7720 }
 7721 
 7722 // logical operations packed integers
 7723 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7724   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7725   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7726   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7727   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7728 }
 7729 
 7730 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7731   assert(UseAVX > 0, "requires some form of AVX");
 7732   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7733   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7734   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7735 }
 7736 
 7737 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7738   assert(UseAVX > 0, "requires some form of AVX");
 7739   InstructionMark im(this);
 7740   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7741   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7742   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7743   emit_int8((unsigned char)0xDB);
 7744   emit_operand(dst, src, 0);
 7745 }
 7746 
 7747 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7748   assert(VM_Version::supports_evex(), "");
 7749   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7750   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7751   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7752 }
 7753 
 7754 //Variable Shift packed integers logically left.
 7755 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7756   assert(UseAVX > 1, "requires AVX2");
 7757   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7758   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7759   emit_int16(0x47, (0xC0 | encode));
 7760 }
 7761 
 7762 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7763   assert(UseAVX > 1, "requires AVX2");
 7764   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7765   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7766   emit_int16(0x47, (0xC0 | encode));
 7767 }
 7768 
 7769 //Variable Shift packed integers logically right.
 7770 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7771   assert(UseAVX > 1, "requires AVX2");
 7772   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7773   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7774   emit_int16(0x45, (0xC0 | encode));
 7775 }
 7776 
 7777 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7778   assert(UseAVX > 1, "requires AVX2");
 7779   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7780   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7781   emit_int16(0x45, (0xC0 | encode));
 7782 }
 7783 
 7784 //Variable right Shift arithmetic packed integers .
 7785 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7786   assert(UseAVX > 1, "requires AVX2");
 7787   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7788   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7789   emit_int16(0x46, (0xC0 | encode));
 7790 }
 7791 
 7792 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7793   assert(VM_Version::supports_avx512bw(), "");
 7794   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7795   attributes.set_is_evex_instruction();
 7796   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7797   emit_int16(0x11, (0xC0 | encode));
 7798 }
 7799 
 7800 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7801   assert(UseAVX > 2, "requires AVX512");
 7802   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7803   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7804   attributes.set_is_evex_instruction();
 7805   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7806   emit_int16(0x46, (0xC0 | encode));
 7807 }
 7808 
 7809 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7810   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7811   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7812   attributes.set_is_evex_instruction();
 7813   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7814   emit_int16(0x71, (0xC0 | encode));
 7815 }
 7816 
 7817 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7818   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7819   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7820   attributes.set_is_evex_instruction();
 7821   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7822   emit_int16(0x73, (0xC0 | encode));
 7823 }
 7824 
 7825 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7826   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7827   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7828   attributes.set_rex_vex_w_reverted();
 7829   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7830   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7831 }
 7832 
 7833 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7834   assert(UseAVX > 0, "requires some form of AVX");
 7835   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7836   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7837   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7838 }
 7839 
 7840 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7841   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7842   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7843   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7844   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7845 }
 7846 
 7847 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7848   assert(UseAVX > 0, "requires some form of AVX");
 7849   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7850   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7851   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7852 }
 7853 
 7854 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7855   assert(UseAVX > 0, "requires some form of AVX");
 7856   InstructionMark im(this);
 7857   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7858   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7859   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7860   emit_int8((unsigned char)0xEB);
 7861   emit_operand(dst, src, 0);
 7862 }
 7863 
 7864 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7865   assert(VM_Version::supports_evex(), "");
 7866   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7867   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7868   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7869 }
 7870 
 7871 
 7872 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7873   assert(VM_Version::supports_evex(), "");
 7874   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7875   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7876   attributes.set_is_evex_instruction();
 7877   attributes.set_embedded_opmask_register_specifier(mask);
 7878   if (merge) {
 7879     attributes.reset_is_clear_context();
 7880   }
 7881   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7882   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7883 }
 7884 
 7885 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7886   assert(VM_Version::supports_evex(), "");
 7887   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7888   InstructionMark im(this);
 7889   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7890   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7891   attributes.set_is_evex_instruction();
 7892   attributes.set_embedded_opmask_register_specifier(mask);
 7893   if (merge) {
 7894     attributes.reset_is_clear_context();
 7895   }
 7896   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7897   emit_int8((unsigned char)0xEB);
 7898   emit_operand(dst, src, 0);
 7899 }
 7900 
 7901 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7902   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7903   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7904   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7905   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7906 }
 7907 
 7908 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7909   assert(UseAVX > 0, "requires some form of AVX");
 7910   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7911          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7912          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7913   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7914   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7915   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7916 }
 7917 
 7918 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7919   assert(UseAVX > 0, "requires some form of AVX");
 7920   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7921          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 7922          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 7923   InstructionMark im(this);
 7924   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7925   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7926   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7927   emit_int8((unsigned char)0xEF);
 7928   emit_operand(dst, src, 0);
 7929 }
 7930 
 7931 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7932   assert(UseAVX > 2, "requires some form of EVEX");
 7933   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7934   attributes.set_rex_vex_w_reverted();
 7935   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7936   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7937 }
 7938 
 7939 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7940   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7941   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7942   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7943   attributes.set_is_evex_instruction();
 7944   attributes.set_embedded_opmask_register_specifier(mask);
 7945   if (merge) {
 7946     attributes.reset_is_clear_context();
 7947   }
 7948   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7949   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7950 }
 7951 
 7952 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7953   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7954   InstructionMark im(this);
 7955   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7956   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7957   attributes.set_is_evex_instruction();
 7958   attributes.set_embedded_opmask_register_specifier(mask);
 7959   if (merge) {
 7960     attributes.reset_is_clear_context();
 7961   }
 7962   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7963   emit_int8((unsigned char)0xEF);
 7964   emit_operand(dst, src, 0);
 7965 }
 7966 
 7967 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7968   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7969   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7970   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7971   attributes.set_is_evex_instruction();
 7972   attributes.set_embedded_opmask_register_specifier(mask);
 7973   if (merge) {
 7974     attributes.reset_is_clear_context();
 7975   }
 7976   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7977   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7978 }
 7979 
 7980 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7981   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7982   InstructionMark im(this);
 7983   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7984   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7985   attributes.set_is_evex_instruction();
 7986   attributes.set_embedded_opmask_register_specifier(mask);
 7987   if (merge) {
 7988     attributes.reset_is_clear_context();
 7989   }
 7990   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7991   emit_int8((unsigned char)0xEF);
 7992   emit_operand(dst, src, 0);
 7993 }
 7994 
 7995 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7996   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7997   InstructionMark im(this);
 7998   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7999   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8000   attributes.set_is_evex_instruction();
 8001   attributes.set_embedded_opmask_register_specifier(mask);
 8002   if (merge) {
 8003     attributes.reset_is_clear_context();
 8004   }
 8005   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8006   emit_int8((unsigned char)0xDB);
 8007   emit_operand(dst, src, 0);
 8008 }
 8009 
 8010 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8011   assert(VM_Version::supports_evex(), "");
 8012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8013   attributes.set_is_evex_instruction();
 8014   attributes.set_embedded_opmask_register_specifier(mask);
 8015   if (merge) {
 8016     attributes.reset_is_clear_context();
 8017   }
 8018   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8019   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 8020 }
 8021 
 8022 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8023   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8024   InstructionMark im(this);
 8025   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8026   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8027   attributes.set_is_evex_instruction();
 8028   attributes.set_embedded_opmask_register_specifier(mask);
 8029   if (merge) {
 8030     attributes.reset_is_clear_context();
 8031   }
 8032   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8033   emit_int8((unsigned char)0xDB);
 8034   emit_operand(dst, src, 0);
 8035 }
 8036 
 8037 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8038   assert(VM_Version::supports_evex(), "");
 8039   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8040   attributes.set_is_evex_instruction();
 8041   attributes.set_embedded_opmask_register_specifier(mask);
 8042   if (merge) {
 8043     attributes.reset_is_clear_context();
 8044   }
 8045   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8046   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8047 }
 8048 
 8049 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8050   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8051   InstructionMark im(this);
 8052   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8053   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8054   attributes.set_is_evex_instruction();
 8055   attributes.set_embedded_opmask_register_specifier(mask);
 8056   if (merge) {
 8057     attributes.reset_is_clear_context();
 8058   }
 8059   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8060   emit_int8((unsigned char)0xEB);
 8061   emit_operand(dst, src, 0);
 8062 }
 8063 
 8064 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8065   assert(VM_Version::supports_evex(), "requires EVEX support");
 8066   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8067   attributes.set_is_evex_instruction();
 8068   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8069   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8070 }
 8071 
 8072 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8073   assert(VM_Version::supports_evex(), "requires EVEX support");
 8074   assert(dst != xnoreg, "sanity");
 8075   InstructionMark im(this);
 8076   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8077   attributes.set_is_evex_instruction();
 8078   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8079   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8080   emit_int8((unsigned char)0xEF);
 8081   emit_operand(dst, src, 0);
 8082 }
 8083 
 8084 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8085   assert(VM_Version::supports_evex(), "requires EVEX support");
 8086   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8088   attributes.set_is_evex_instruction();
 8089   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8090   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8091 }
 8092 
 8093 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8094   assert(VM_Version::supports_evex(), "requires EVEX support");
 8095   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8096   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8097   attributes.set_is_evex_instruction();
 8098   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8099   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8100 }
 8101 
 8102 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8103   assert(VM_Version::supports_evex(), "requires EVEX support");
 8104   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8106   attributes.set_is_evex_instruction();
 8107   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8108   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8109 }
 8110 
 8111 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8112   assert(VM_Version::supports_evex(), "requires EVEX support");
 8113   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8114   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8115   attributes.set_is_evex_instruction();
 8116   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8117   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8118 }
 8119 
 8120 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8121   assert(VM_Version::supports_evex(), "requires EVEX support");
 8122   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8124   attributes.set_is_evex_instruction();
 8125   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8126   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8127 }
 8128 
 8129 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8130   assert(VM_Version::supports_evex(), "requires EVEX support");
 8131   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8132   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8133   attributes.set_is_evex_instruction();
 8134   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8135   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8136 }
 8137 
 8138 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8139   assert(VM_Version::supports_evex(), "requires EVEX support");
 8140   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8142   attributes.set_is_evex_instruction();
 8143   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8144   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8145 }
 8146 
 8147 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8148   assert(VM_Version::supports_evex(), "requires EVEX support");
 8149   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8150   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8151   attributes.set_is_evex_instruction();
 8152   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8153   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8154 }
 8155 
 8156 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8157   assert(VM_Version::supports_avx512cd(), "");
 8158   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8159   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8160   attributes.set_is_evex_instruction();
 8161   attributes.set_embedded_opmask_register_specifier(mask);
 8162   if (merge) {
 8163     attributes.reset_is_clear_context();
 8164   }
 8165   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8166   emit_int16(0x44, (0xC0 | encode));
 8167 }
 8168 
 8169 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8170   assert(VM_Version::supports_avx512cd(), "");
 8171   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8172   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8173   attributes.set_is_evex_instruction();
 8174   attributes.set_embedded_opmask_register_specifier(mask);
 8175   if (merge) {
 8176     attributes.reset_is_clear_context();
 8177   }
 8178   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8179   emit_int16(0x44, (0xC0 | encode));
 8180 }
 8181 
 8182 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8183   assert(VM_Version::supports_evex(), "requires EVEX support");
 8184   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8185   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8186   attributes.set_is_evex_instruction();
 8187   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8188   emit_int8(0x25);
 8189   emit_int8((unsigned char)(0xC0 | encode));
 8190   emit_int8(imm8);
 8191 }
 8192 
 8193 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8194   assert(VM_Version::supports_evex(), "requires EVEX support");
 8195   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8196   assert(dst != xnoreg, "sanity");
 8197   InstructionMark im(this);
 8198   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8199   attributes.set_is_evex_instruction();
 8200   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8201   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8202   emit_int8(0x25);
 8203   emit_operand(dst, src3, 1);
 8204   emit_int8(imm8);
 8205 }
 8206 
 8207 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8208   assert(VM_Version::supports_evex(), "requires EVEX support");
 8209   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8210   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8211   attributes.set_is_evex_instruction();
 8212   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8213   emit_int8(0x25);
 8214   emit_int8((unsigned char)(0xC0 | encode));
 8215   emit_int8(imm8);
 8216 }
 8217 
 8218 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8219   assert(VM_Version::supports_evex(), "");
 8220   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8222   attributes.set_is_evex_instruction();
 8223   attributes.set_embedded_opmask_register_specifier(mask);
 8224   if (merge) {
 8225     attributes.reset_is_clear_context();
 8226   }
 8227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8228   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8229 }
 8230 
 8231 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8232   assert(VM_Version::supports_evex(), "");
 8233   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8234   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8235   attributes.set_is_evex_instruction();
 8236   attributes.set_embedded_opmask_register_specifier(mask);
 8237   if (merge) {
 8238     attributes.reset_is_clear_context();
 8239   }
 8240   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8241   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8242 }
 8243 
 8244 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8245   assert(VM_Version::supports_avx512_vbmi2(), "");
 8246   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8247   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8248   attributes.set_is_evex_instruction();
 8249   attributes.set_embedded_opmask_register_specifier(mask);
 8250   if (merge) {
 8251     attributes.reset_is_clear_context();
 8252   }
 8253   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8254   emit_int16(0x62, (0xC0 | encode));
 8255 }
 8256 
 8257 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8258   assert(VM_Version::supports_avx512_vbmi2(), "");
 8259   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8260   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8261   attributes.set_is_evex_instruction();
 8262   attributes.set_embedded_opmask_register_specifier(mask);
 8263   if (merge) {
 8264     attributes.reset_is_clear_context();
 8265   }
 8266   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8267   emit_int16(0x62, (0xC0 | encode));
 8268 }
 8269 
 8270 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8271   assert(VM_Version::supports_evex(), "");
 8272   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8273   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8274   attributes.set_is_evex_instruction();
 8275   attributes.set_embedded_opmask_register_specifier(mask);
 8276   if (merge) {
 8277     attributes.reset_is_clear_context();
 8278   }
 8279   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8280   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8281 }
 8282 
 8283 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8284   assert(VM_Version::supports_evex(), "");
 8285   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8286   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8287   attributes.set_is_evex_instruction();
 8288   attributes.set_embedded_opmask_register_specifier(mask);
 8289   if (merge) {
 8290     attributes.reset_is_clear_context();
 8291   }
 8292   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8293   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8294 }
 8295 
 8296 // vinserti forms
 8297 
 8298 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8299   assert(VM_Version::supports_avx2(), "");
 8300   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8301   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8303   // last byte:
 8304   // 0x00 - insert into lower 128 bits
 8305   // 0x01 - insert into upper 128 bits
 8306   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8307 }
 8308 
 8309 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8310   assert(VM_Version::supports_avx2(), "");
 8311   assert(dst != xnoreg, "sanity");
 8312   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8313   InstructionMark im(this);
 8314   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8315   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8316   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8317   emit_int8(0x38);
 8318   emit_operand(dst, src, 1);
 8319   // 0x00 - insert into lower 128 bits
 8320   // 0x01 - insert into upper 128 bits
 8321   emit_int8(imm8 & 0x01);
 8322 }
 8323 
 8324 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8325   assert(VM_Version::supports_evex(), "");
 8326   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8327   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8328   attributes.set_is_evex_instruction();
 8329   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8330   // imm8:
 8331   // 0x00 - insert into q0 128 bits (0..127)
 8332   // 0x01 - insert into q1 128 bits (128..255)
 8333   // 0x02 - insert into q2 128 bits (256..383)
 8334   // 0x03 - insert into q3 128 bits (384..511)
 8335   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8336 }
 8337 
 8338 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8339   assert(VM_Version::supports_evex(), "");
 8340   assert(dst != xnoreg, "sanity");
 8341   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8342   InstructionMark im(this);
 8343   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8344   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8345   attributes.set_is_evex_instruction();
 8346   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8347   emit_int8(0x18);
 8348   emit_operand(dst, src, 1);
 8349   // 0x00 - insert into q0 128 bits (0..127)
 8350   // 0x01 - insert into q1 128 bits (128..255)
 8351   // 0x02 - insert into q2 128 bits (256..383)
 8352   // 0x03 - insert into q3 128 bits (384..511)
 8353   emit_int8(imm8 & 0x03);
 8354 }
 8355 
 8356 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8357   assert(VM_Version::supports_evex(), "");
 8358   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8359   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8360   attributes.set_is_evex_instruction();
 8361   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8362   //imm8:
 8363   // 0x00 - insert into lower 256 bits
 8364   // 0x01 - insert into upper 256 bits
 8365   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8366 }
 8367 
 8368 
 8369 // vinsertf forms
 8370 
 8371 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8372   assert(VM_Version::supports_avx(), "");
 8373   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8374   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8375   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8376   // imm8:
 8377   // 0x00 - insert into lower 128 bits
 8378   // 0x01 - insert into upper 128 bits
 8379   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8380 }
 8381 
 8382 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8383   assert(VM_Version::supports_avx(), "");
 8384   assert(dst != xnoreg, "sanity");
 8385   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8386   InstructionMark im(this);
 8387   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8388   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8389   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8390   emit_int8(0x18);
 8391   emit_operand(dst, src, 1);
 8392   // 0x00 - insert into lower 128 bits
 8393   // 0x01 - insert into upper 128 bits
 8394   emit_int8(imm8 & 0x01);
 8395 }
 8396 
 8397 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8398   assert(VM_Version::supports_evex(), "");
 8399   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8400   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8401   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8402   // imm8:
 8403   // 0x00 - insert into q0 128 bits (0..127)
 8404   // 0x01 - insert into q1 128 bits (128..255)
 8405   // 0x02 - insert into q0 128 bits (256..383)
 8406   // 0x03 - insert into q1 128 bits (384..512)
 8407   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8408 }
 8409 
 8410 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8411   assert(VM_Version::supports_evex(), "");
 8412   assert(dst != xnoreg, "sanity");
 8413   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8414   InstructionMark im(this);
 8415   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8416   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8417   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8418   emit_int8(0x18);
 8419   emit_operand(dst, src, 1);
 8420   // 0x00 - insert into q0 128 bits (0..127)
 8421   // 0x01 - insert into q1 128 bits (128..255)
 8422   // 0x02 - insert into q0 128 bits (256..383)
 8423   // 0x03 - insert into q1 128 bits (384..512)
 8424   emit_int8(imm8 & 0x03);
 8425 }
 8426 
 8427 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8428   assert(VM_Version::supports_evex(), "");
 8429   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8430   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8431   attributes.set_is_evex_instruction();
 8432   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8433   // imm8:
 8434   // 0x00 - insert into lower 256 bits
 8435   // 0x01 - insert into upper 256 bits
 8436   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8437 }
 8438 
 8439 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8440   assert(VM_Version::supports_evex(), "");
 8441   assert(dst != xnoreg, "sanity");
 8442   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8443   InstructionMark im(this);
 8444   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8445   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8446   attributes.set_is_evex_instruction();
 8447   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8448   emit_int8(0x1A);
 8449   emit_operand(dst, src, 1);
 8450   // 0x00 - insert into lower 256 bits
 8451   // 0x01 - insert into upper 256 bits
 8452   emit_int8(imm8 & 0x01);
 8453 }
 8454 
 8455 
 8456 // vextracti forms
 8457 
 8458 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8459   assert(VM_Version::supports_avx2(), "");
 8460   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8461   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8462   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8463   // imm8:
 8464   // 0x00 - extract from lower 128 bits
 8465   // 0x01 - extract from upper 128 bits
 8466   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8467 }
 8468 
 8469 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8470   assert(VM_Version::supports_avx2(), "");
 8471   assert(src != xnoreg, "sanity");
 8472   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8473   InstructionMark im(this);
 8474   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8475   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8476   attributes.reset_is_clear_context();
 8477   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8478   emit_int8(0x39);
 8479   emit_operand(src, dst, 1);
 8480   // 0x00 - extract from lower 128 bits
 8481   // 0x01 - extract from upper 128 bits
 8482   emit_int8(imm8 & 0x01);
 8483 }
 8484 
 8485 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8486   assert(VM_Version::supports_evex(), "");
 8487   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8488   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8489   attributes.set_is_evex_instruction();
 8490   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8491   // imm8:
 8492   // 0x00 - extract from bits 127:0
 8493   // 0x01 - extract from bits 255:128
 8494   // 0x02 - extract from bits 383:256
 8495   // 0x03 - extract from bits 511:384
 8496   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8497 }
 8498 
 8499 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8500   assert(VM_Version::supports_evex(), "");
 8501   assert(src != xnoreg, "sanity");
 8502   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8503   InstructionMark im(this);
 8504   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8505   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8506   attributes.reset_is_clear_context();
 8507   attributes.set_is_evex_instruction();
 8508   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8509   emit_int8(0x39);
 8510   emit_operand(src, dst, 1);
 8511   // 0x00 - extract from bits 127:0
 8512   // 0x01 - extract from bits 255:128
 8513   // 0x02 - extract from bits 383:256
 8514   // 0x03 - extract from bits 511:384
 8515   emit_int8(imm8 & 0x03);
 8516 }
 8517 
 8518 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8519   assert(VM_Version::supports_avx512dq(), "");
 8520   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8521   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8522   attributes.set_is_evex_instruction();
 8523   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8524   // imm8:
 8525   // 0x00 - extract from bits 127:0
 8526   // 0x01 - extract from bits 255:128
 8527   // 0x02 - extract from bits 383:256
 8528   // 0x03 - extract from bits 511:384
 8529   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8530 }
 8531 
 8532 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8533   assert(VM_Version::supports_evex(), "");
 8534   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8535   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8536   attributes.set_is_evex_instruction();
 8537   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8538   // imm8:
 8539   // 0x00 - extract from lower 256 bits
 8540   // 0x01 - extract from upper 256 bits
 8541   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8542 }
 8543 
 8544 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8545   assert(VM_Version::supports_evex(), "");
 8546   assert(src != xnoreg, "sanity");
 8547   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8548   InstructionMark im(this);
 8549   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8550   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8551   attributes.reset_is_clear_context();
 8552   attributes.set_is_evex_instruction();
 8553   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8554   emit_int8(0x38);
 8555   emit_operand(src, dst, 1);
 8556   // 0x00 - extract from lower 256 bits
 8557   // 0x01 - extract from upper 256 bits
 8558   emit_int8(imm8 & 0x01);
 8559 }
 8560 // vextractf forms
 8561 
 8562 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8563   assert(VM_Version::supports_avx(), "");
 8564   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8565   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8566   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8567   // imm8:
 8568   // 0x00 - extract from lower 128 bits
 8569   // 0x01 - extract from upper 128 bits
 8570   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8571 }
 8572 
 8573 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8574   assert(VM_Version::supports_avx(), "");
 8575   assert(src != xnoreg, "sanity");
 8576   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8577   InstructionMark im(this);
 8578   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8579   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8580   attributes.reset_is_clear_context();
 8581   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8582   emit_int8(0x19);
 8583   emit_operand(src, dst, 1);
 8584   // 0x00 - extract from lower 128 bits
 8585   // 0x01 - extract from upper 128 bits
 8586   emit_int8(imm8 & 0x01);
 8587 }
 8588 
 8589 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8590   assert(VM_Version::supports_evex(), "");
 8591   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8592   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8593   attributes.set_is_evex_instruction();
 8594   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8595   // imm8:
 8596   // 0x00 - extract from bits 127:0
 8597   // 0x01 - extract from bits 255:128
 8598   // 0x02 - extract from bits 383:256
 8599   // 0x03 - extract from bits 511:384
 8600   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8601 }
 8602 
 8603 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8604   assert(VM_Version::supports_evex(), "");
 8605   assert(src != xnoreg, "sanity");
 8606   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8607   InstructionMark im(this);
 8608   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8609   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8610   attributes.reset_is_clear_context();
 8611   attributes.set_is_evex_instruction();
 8612   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8613   emit_int8(0x19);
 8614   emit_operand(src, dst, 1);
 8615   // 0x00 - extract from bits 127:0
 8616   // 0x01 - extract from bits 255:128
 8617   // 0x02 - extract from bits 383:256
 8618   // 0x03 - extract from bits 511:384
 8619   emit_int8(imm8 & 0x03);
 8620 }
 8621 
 8622 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8623   assert(VM_Version::supports_avx512dq(), "");
 8624   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8625   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8626   attributes.set_is_evex_instruction();
 8627   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8628   // imm8:
 8629   // 0x00 - extract from bits 127:0
 8630   // 0x01 - extract from bits 255:128
 8631   // 0x02 - extract from bits 383:256
 8632   // 0x03 - extract from bits 511:384
 8633   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8634 }
 8635 
 8636 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8637   assert(VM_Version::supports_evex(), "");
 8638   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8639   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8640   attributes.set_is_evex_instruction();
 8641   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8642   // imm8:
 8643   // 0x00 - extract from lower 256 bits
 8644   // 0x01 - extract from upper 256 bits
 8645   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8646 }
 8647 
 8648 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8649   assert(VM_Version::supports_evex(), "");
 8650   assert(src != xnoreg, "sanity");
 8651   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8652   InstructionMark im(this);
 8653   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8654   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8655   attributes.reset_is_clear_context();
 8656   attributes.set_is_evex_instruction();
 8657   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8658   emit_int8(0x1B);
 8659   emit_operand(src, dst, 1);
 8660   // 0x00 - extract from lower 256 bits
 8661   // 0x01 - extract from upper 256 bits
 8662   emit_int8(imm8 & 0x01);
 8663 }
 8664 
 8665 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8666 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8667   assert(VM_Version::supports_avx2(), "");
 8668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8669   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8670   emit_int16(0x78, (0xC0 | encode));
 8671 }
 8672 
 8673 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8674   assert(VM_Version::supports_avx2(), "");
 8675   assert(dst != xnoreg, "sanity");
 8676   InstructionMark im(this);
 8677   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8678   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8679   // swap src<->dst for encoding
 8680   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8681   emit_int8(0x78);
 8682   emit_operand(dst, src, 0);
 8683 }
 8684 
 8685 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8686 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8687   assert(VM_Version::supports_avx2(), "");
 8688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8689   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8690   emit_int16(0x79, (0xC0 | encode));
 8691 }
 8692 
 8693 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8694   assert(VM_Version::supports_avx2(), "");
 8695   assert(dst != xnoreg, "sanity");
 8696   InstructionMark im(this);
 8697   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8698   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8699   // swap src<->dst for encoding
 8700   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8701   emit_int8(0x79);
 8702   emit_operand(dst, src, 0);
 8703 }
 8704 
 8705 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8706   assert(UseAVX > 0, "requires some form of AVX");
 8707   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8708   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8709   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8710 }
 8711 
 8712 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8713   assert(UseAVX > 0, "requires some form of AVX");
 8714   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8715   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8716   emit_int16(0x69, (0xC0 | encode));
 8717 }
 8718 
 8719 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8720   assert(UseAVX > 0, "requires some form of AVX");
 8721   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8722   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8723   emit_int16(0x61, (0xC0 | encode));
 8724 }
 8725 
 8726 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8727   assert(UseAVX > 0, "requires some form of AVX");
 8728   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8729   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8730   emit_int16(0x6A, (0xC0 | encode));
 8731 }
 8732 
 8733 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8734   assert(UseAVX > 0, "requires some form of AVX");
 8735   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8736   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8737   emit_int16(0x62, (0xC0 | encode));
 8738 }
 8739 
 8740 // xmm/mem sourced byte/word/dword/qword replicate
 8741 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8742   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8743   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8744   attributes.set_is_evex_instruction();
 8745   attributes.set_embedded_opmask_register_specifier(mask);
 8746   if (merge) {
 8747     attributes.reset_is_clear_context();
 8748   }
 8749   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8750   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8751 }
 8752 
 8753 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8754   InstructionMark im(this);
 8755   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8756   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8757   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8758   attributes.set_is_evex_instruction();
 8759   attributes.set_embedded_opmask_register_specifier(mask);
 8760   if (merge) {
 8761     attributes.reset_is_clear_context();
 8762   }
 8763   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8764   emit_int8((unsigned char)0xFC);
 8765   emit_operand(dst, src, 0);
 8766 }
 8767 
 8768 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8769   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8770   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8771   attributes.set_is_evex_instruction();
 8772   attributes.set_embedded_opmask_register_specifier(mask);
 8773   if (merge) {
 8774     attributes.reset_is_clear_context();
 8775   }
 8776   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8777   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8778 }
 8779 
 8780 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8781   InstructionMark im(this);
 8782   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8783   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8784   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8785   attributes.set_is_evex_instruction();
 8786   attributes.set_embedded_opmask_register_specifier(mask);
 8787   if (merge) {
 8788     attributes.reset_is_clear_context();
 8789   }
 8790   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8791   emit_int8((unsigned char)0xFD);
 8792   emit_operand(dst, src, 0);
 8793 }
 8794 
 8795 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8796   assert(VM_Version::supports_evex(), "");
 8797   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8798   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8799   attributes.set_is_evex_instruction();
 8800   attributes.set_embedded_opmask_register_specifier(mask);
 8801   if (merge) {
 8802     attributes.reset_is_clear_context();
 8803   }
 8804   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8805   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8806 }
 8807 
 8808 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8809   InstructionMark im(this);
 8810   assert(VM_Version::supports_evex(), "");
 8811   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8812   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8813   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8814   attributes.set_is_evex_instruction();
 8815   attributes.set_embedded_opmask_register_specifier(mask);
 8816   if (merge) {
 8817     attributes.reset_is_clear_context();
 8818   }
 8819   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8820   emit_int8((unsigned char)0xFE);
 8821   emit_operand(dst, src, 0);
 8822 }
 8823 
 8824 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8825   assert(VM_Version::supports_evex(), "");
 8826   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8827   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8828   attributes.set_is_evex_instruction();
 8829   attributes.set_embedded_opmask_register_specifier(mask);
 8830   if (merge) {
 8831     attributes.reset_is_clear_context();
 8832   }
 8833   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8834   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8835 }
 8836 
 8837 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8838   InstructionMark im(this);
 8839   assert(VM_Version::supports_evex(), "");
 8840   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8841   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8842   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8843   attributes.set_is_evex_instruction();
 8844   attributes.set_embedded_opmask_register_specifier(mask);
 8845   if (merge) {
 8846     attributes.reset_is_clear_context();
 8847   }
 8848   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8849   emit_int8((unsigned char)0xD4);
 8850   emit_operand(dst, src, 0);
 8851 }
 8852 
 8853 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8854   assert(VM_Version::supports_evex(), "");
 8855   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8857   attributes.set_is_evex_instruction();
 8858   attributes.set_embedded_opmask_register_specifier(mask);
 8859   if (merge) {
 8860     attributes.reset_is_clear_context();
 8861   }
 8862   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8863   emit_int16(0x58, (0xC0 | encode));
 8864 }
 8865 
 8866 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8867   InstructionMark im(this);
 8868   assert(VM_Version::supports_evex(), "");
 8869   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8870   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8871   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8872   attributes.set_is_evex_instruction();
 8873   attributes.set_embedded_opmask_register_specifier(mask);
 8874   if (merge) {
 8875     attributes.reset_is_clear_context();
 8876   }
 8877   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8878   emit_int8(0x58);
 8879   emit_operand(dst, src, 0);
 8880 }
 8881 
 8882 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8883   assert(VM_Version::supports_evex(), "");
 8884   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8885   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8886   attributes.set_is_evex_instruction();
 8887   attributes.set_embedded_opmask_register_specifier(mask);
 8888   if (merge) {
 8889     attributes.reset_is_clear_context();
 8890   }
 8891   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8892   emit_int16(0x58, (0xC0 | encode));
 8893 }
 8894 
 8895 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8896   InstructionMark im(this);
 8897   assert(VM_Version::supports_evex(), "");
 8898   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8899   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8900   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8901   attributes.set_is_evex_instruction();
 8902   attributes.set_embedded_opmask_register_specifier(mask);
 8903   if (merge) {
 8904     attributes.reset_is_clear_context();
 8905   }
 8906   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8907   emit_int8(0x58);
 8908   emit_operand(dst, src, 0);
 8909 }
 8910 
 8911 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8912   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8913   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8914   attributes.set_is_evex_instruction();
 8915   attributes.set_embedded_opmask_register_specifier(mask);
 8916   if (merge) {
 8917     attributes.reset_is_clear_context();
 8918   }
 8919   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8920   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8921 }
 8922 
 8923 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8924   InstructionMark im(this);
 8925   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8926   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8927   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8928   attributes.set_is_evex_instruction();
 8929   attributes.set_embedded_opmask_register_specifier(mask);
 8930   if (merge) {
 8931     attributes.reset_is_clear_context();
 8932   }
 8933   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8934   emit_int8((unsigned char)0xF8);
 8935   emit_operand(dst, src, 0);
 8936 }
 8937 
 8938 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8939   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8940   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8941   attributes.set_is_evex_instruction();
 8942   attributes.set_embedded_opmask_register_specifier(mask);
 8943   if (merge) {
 8944     attributes.reset_is_clear_context();
 8945   }
 8946   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8947   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8948 }
 8949 
 8950 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8951   InstructionMark im(this);
 8952   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8953   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8954   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8955   attributes.set_is_evex_instruction();
 8956   attributes.set_embedded_opmask_register_specifier(mask);
 8957   if (merge) {
 8958     attributes.reset_is_clear_context();
 8959   }
 8960   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8961   emit_int8((unsigned char)0xF9);
 8962   emit_operand(dst, src, 0);
 8963 }
 8964 
 8965 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8966   assert(VM_Version::supports_evex(), "");
 8967   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8968   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8969   attributes.set_is_evex_instruction();
 8970   attributes.set_embedded_opmask_register_specifier(mask);
 8971   if (merge) {
 8972     attributes.reset_is_clear_context();
 8973   }
 8974   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8975   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8976 }
 8977 
 8978 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8979   InstructionMark im(this);
 8980   assert(VM_Version::supports_evex(), "");
 8981   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8982   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8983   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8984   attributes.set_is_evex_instruction();
 8985   attributes.set_embedded_opmask_register_specifier(mask);
 8986   if (merge) {
 8987     attributes.reset_is_clear_context();
 8988   }
 8989   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8990   emit_int8((unsigned char)0xFA);
 8991   emit_operand(dst, src, 0);
 8992 }
 8993 
 8994 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8995   assert(VM_Version::supports_evex(), "");
 8996   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8997   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8998   attributes.set_is_evex_instruction();
 8999   attributes.set_embedded_opmask_register_specifier(mask);
 9000   if (merge) {
 9001     attributes.reset_is_clear_context();
 9002   }
 9003   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9004   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 9005 }
 9006 
 9007 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9008   InstructionMark im(this);
 9009   assert(VM_Version::supports_evex(), "");
 9010   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9011   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9012   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9013   attributes.set_is_evex_instruction();
 9014   attributes.set_embedded_opmask_register_specifier(mask);
 9015   if (merge) {
 9016     attributes.reset_is_clear_context();
 9017   }
 9018   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9019   emit_int8((unsigned char)0xFB);
 9020   emit_operand(dst, src, 0);
 9021 }
 9022 
 9023 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9024   assert(VM_Version::supports_evex(), "");
 9025   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9026   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9027   attributes.set_is_evex_instruction();
 9028   attributes.set_embedded_opmask_register_specifier(mask);
 9029   if (merge) {
 9030     attributes.reset_is_clear_context();
 9031   }
 9032   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9033   emit_int16(0x5C, (0xC0 | encode));
 9034 }
 9035 
 9036 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9037   InstructionMark im(this);
 9038   assert(VM_Version::supports_evex(), "");
 9039   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9040   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9041   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9042   attributes.set_is_evex_instruction();
 9043   attributes.set_embedded_opmask_register_specifier(mask);
 9044   if (merge) {
 9045     attributes.reset_is_clear_context();
 9046   }
 9047   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9048   emit_int8(0x5C);
 9049   emit_operand(dst, src, 0);
 9050 }
 9051 
 9052 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9053   assert(VM_Version::supports_evex(), "");
 9054   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9055   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9056   attributes.set_is_evex_instruction();
 9057   attributes.set_embedded_opmask_register_specifier(mask);
 9058   if (merge) {
 9059     attributes.reset_is_clear_context();
 9060   }
 9061   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9062   emit_int16(0x5C, (0xC0 | encode));
 9063 }
 9064 
 9065 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9066   InstructionMark im(this);
 9067   assert(VM_Version::supports_evex(), "");
 9068   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9069   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9070   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9071   attributes.set_is_evex_instruction();
 9072   attributes.set_embedded_opmask_register_specifier(mask);
 9073   if (merge) {
 9074     attributes.reset_is_clear_context();
 9075   }
 9076   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9077   emit_int8(0x5C);
 9078   emit_operand(dst, src, 0);
 9079 }
 9080 
 9081 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9082   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9083   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9084   attributes.set_is_evex_instruction();
 9085   attributes.set_embedded_opmask_register_specifier(mask);
 9086   if (merge) {
 9087     attributes.reset_is_clear_context();
 9088   }
 9089   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9090   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 9091 }
 9092 
 9093 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9094   InstructionMark im(this);
 9095   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9096   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9097   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9098   attributes.set_is_evex_instruction();
 9099   attributes.set_embedded_opmask_register_specifier(mask);
 9100   if (merge) {
 9101     attributes.reset_is_clear_context();
 9102   }
 9103   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9104   emit_int8((unsigned char)0xD5);
 9105   emit_operand(dst, src, 0);
 9106 }
 9107 
 9108 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9109   assert(VM_Version::supports_evex(), "");
 9110   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9111   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9112   attributes.set_is_evex_instruction();
 9113   attributes.set_embedded_opmask_register_specifier(mask);
 9114   if (merge) {
 9115     attributes.reset_is_clear_context();
 9116   }
 9117   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9118   emit_int16(0x40, (0xC0 | encode));
 9119 }
 9120 
 9121 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9122   InstructionMark im(this);
 9123   assert(VM_Version::supports_evex(), "");
 9124   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9125   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9126   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9127   attributes.set_is_evex_instruction();
 9128   attributes.set_embedded_opmask_register_specifier(mask);
 9129   if (merge) {
 9130     attributes.reset_is_clear_context();
 9131   }
 9132   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9133   emit_int8(0x40);
 9134   emit_operand(dst, src, 0);
 9135 }
 9136 
 9137 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9138   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9139   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9140   attributes.set_is_evex_instruction();
 9141   attributes.set_embedded_opmask_register_specifier(mask);
 9142   if (merge) {
 9143     attributes.reset_is_clear_context();
 9144   }
 9145   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9146   emit_int16(0x40, (0xC0 | encode));
 9147 }
 9148 
 9149 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9150   InstructionMark im(this);
 9151   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9152   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9153   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9154   attributes.set_is_evex_instruction();
 9155   attributes.set_embedded_opmask_register_specifier(mask);
 9156   if (merge) {
 9157     attributes.reset_is_clear_context();
 9158   }
 9159   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9160   emit_int8(0x40);
 9161   emit_operand(dst, src, 0);
 9162 }
 9163 
 9164 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9165   assert(VM_Version::supports_evex(), "");
 9166   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9167   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9168   attributes.set_is_evex_instruction();
 9169   attributes.set_embedded_opmask_register_specifier(mask);
 9170   if (merge) {
 9171     attributes.reset_is_clear_context();
 9172   }
 9173   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9174   emit_int16(0x59, (0xC0 | encode));
 9175 }
 9176 
 9177 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9178   InstructionMark im(this);
 9179   assert(VM_Version::supports_evex(), "");
 9180   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9181   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9182   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9183   attributes.set_is_evex_instruction();
 9184   attributes.set_embedded_opmask_register_specifier(mask);
 9185   if (merge) {
 9186     attributes.reset_is_clear_context();
 9187   }
 9188   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9189   emit_int8(0x59);
 9190   emit_operand(dst, src, 0);
 9191 }
 9192 
 9193 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9194   assert(VM_Version::supports_evex(), "");
 9195   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9196   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9197   attributes.set_is_evex_instruction();
 9198   attributes.set_embedded_opmask_register_specifier(mask);
 9199   if (merge) {
 9200     attributes.reset_is_clear_context();
 9201   }
 9202   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9203   emit_int16(0x59, (0xC0 | encode));
 9204 }
 9205 
 9206 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9207   InstructionMark im(this);
 9208   assert(VM_Version::supports_evex(), "");
 9209   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9210   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9211   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9212   attributes.set_is_evex_instruction();
 9213   attributes.set_embedded_opmask_register_specifier(mask);
 9214   if (merge) {
 9215     attributes.reset_is_clear_context();
 9216   }
 9217   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9218   emit_int8(0x59);
 9219   emit_operand(dst, src, 0);
 9220 }
 9221 
 9222 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9223   assert(VM_Version::supports_evex(), "");
 9224   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9225   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9226   attributes.set_is_evex_instruction();
 9227   attributes.set_embedded_opmask_register_specifier(mask);
 9228   if (merge) {
 9229     attributes.reset_is_clear_context();
 9230   }
 9231   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9232   emit_int16(0x51, (0xC0 | encode));
 9233 }
 9234 
 9235 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9236   InstructionMark im(this);
 9237   assert(VM_Version::supports_evex(), "");
 9238   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9239   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9240   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9241   attributes.set_is_evex_instruction();
 9242   attributes.set_embedded_opmask_register_specifier(mask);
 9243   if (merge) {
 9244     attributes.reset_is_clear_context();
 9245   }
 9246   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9247   emit_int8(0x51);
 9248   emit_operand(dst, src, 0);
 9249 }
 9250 
 9251 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9252   assert(VM_Version::supports_evex(), "");
 9253   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9254   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9255   attributes.set_is_evex_instruction();
 9256   attributes.set_embedded_opmask_register_specifier(mask);
 9257   if (merge) {
 9258     attributes.reset_is_clear_context();
 9259   }
 9260   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9261   emit_int16(0x51, (0xC0 | encode));
 9262 }
 9263 
 9264 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9265   InstructionMark im(this);
 9266   assert(VM_Version::supports_evex(), "");
 9267   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9268   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9269   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9270   attributes.set_is_evex_instruction();
 9271   attributes.set_embedded_opmask_register_specifier(mask);
 9272   if (merge) {
 9273     attributes.reset_is_clear_context();
 9274   }
 9275   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9276   emit_int8(0x51);
 9277   emit_operand(dst, src, 0);
 9278 }
 9279 
 9280 
 9281 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9282   assert(VM_Version::supports_evex(), "");
 9283   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9284   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9285   attributes.set_is_evex_instruction();
 9286   attributes.set_embedded_opmask_register_specifier(mask);
 9287   if (merge) {
 9288     attributes.reset_is_clear_context();
 9289   }
 9290   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9291   emit_int16(0x5E, (0xC0 | encode));
 9292 }
 9293 
 9294 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9295   InstructionMark im(this);
 9296   assert(VM_Version::supports_evex(), "");
 9297   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9298   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9299   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9300   attributes.set_is_evex_instruction();
 9301   attributes.set_embedded_opmask_register_specifier(mask);
 9302   if (merge) {
 9303     attributes.reset_is_clear_context();
 9304   }
 9305   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9306   emit_int8(0x5E);
 9307   emit_operand(dst, src, 0);
 9308 }
 9309 
 9310 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9311   assert(VM_Version::supports_evex(), "");
 9312   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9313   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9314   attributes.set_is_evex_instruction();
 9315   attributes.set_embedded_opmask_register_specifier(mask);
 9316   if (merge) {
 9317     attributes.reset_is_clear_context();
 9318   }
 9319   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9320   emit_int16(0x5E, (0xC0 | encode));
 9321 }
 9322 
 9323 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9324   InstructionMark im(this);
 9325   assert(VM_Version::supports_evex(), "");
 9326   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9327   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9328   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9329   attributes.set_is_evex_instruction();
 9330   attributes.set_embedded_opmask_register_specifier(mask);
 9331   if (merge) {
 9332     attributes.reset_is_clear_context();
 9333   }
 9334   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9335   emit_int8(0x5E);
 9336   emit_operand(dst, src, 0);
 9337 }
 9338 
 9339 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9340   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9341   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9342   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9343   attributes.set_is_evex_instruction();
 9344   attributes.set_embedded_opmask_register_specifier(mask);
 9345   if (merge) {
 9346     attributes.reset_is_clear_context();
 9347   }
 9348   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9349   emit_int16(0x1C, (0xC0 | encode));
 9350 }
 9351 
 9352 
 9353 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9354   InstructionMark im(this);
 9355   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9356   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9357   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9358   attributes.set_is_evex_instruction();
 9359   attributes.set_embedded_opmask_register_specifier(mask);
 9360   if (merge) {
 9361     attributes.reset_is_clear_context();
 9362   }
 9363   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9364   emit_int8(0x1C);
 9365   emit_operand(dst, src, 0);
 9366 }
 9367 
 9368 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9369   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9370   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9371   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9372   attributes.set_is_evex_instruction();
 9373   attributes.set_embedded_opmask_register_specifier(mask);
 9374   if (merge) {
 9375     attributes.reset_is_clear_context();
 9376   }
 9377   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9378   emit_int16(0x1D, (0xC0 | encode));
 9379 }
 9380 
 9381 
 9382 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9383   InstructionMark im(this);
 9384   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9385   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9386   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9387   attributes.set_is_evex_instruction();
 9388   attributes.set_embedded_opmask_register_specifier(mask);
 9389   if (merge) {
 9390     attributes.reset_is_clear_context();
 9391   }
 9392   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9393   emit_int8(0x1D);
 9394   emit_operand(dst, src, 0);
 9395 }
 9396 
 9397 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9398   assert(VM_Version::supports_evex(), "");
 9399   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9400   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9401   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9402   attributes.set_is_evex_instruction();
 9403   attributes.set_embedded_opmask_register_specifier(mask);
 9404   if (merge) {
 9405     attributes.reset_is_clear_context();
 9406   }
 9407   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9408   emit_int16(0x1E, (0xC0 | encode));
 9409 }
 9410 
 9411 
 9412 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9413   InstructionMark im(this);
 9414   assert(VM_Version::supports_evex(), "");
 9415   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9416   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9417   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9418   attributes.set_is_evex_instruction();
 9419   attributes.set_embedded_opmask_register_specifier(mask);
 9420   if (merge) {
 9421     attributes.reset_is_clear_context();
 9422   }
 9423   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9424   emit_int8(0x1E);
 9425   emit_operand(dst, src, 0);
 9426 }
 9427 
 9428 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9429   assert(VM_Version::supports_evex(), "");
 9430   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9431   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9432   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9433   attributes.set_is_evex_instruction();
 9434   attributes.set_embedded_opmask_register_specifier(mask);
 9435   if (merge) {
 9436     attributes.reset_is_clear_context();
 9437   }
 9438   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9439   emit_int16(0x1F, (0xC0 | encode));
 9440 }
 9441 
 9442 
 9443 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9444   InstructionMark im(this);
 9445   assert(VM_Version::supports_evex(), "");
 9446   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9447   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9448   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9449   attributes.set_is_evex_instruction();
 9450   attributes.set_embedded_opmask_register_specifier(mask);
 9451   if (merge) {
 9452     attributes.reset_is_clear_context();
 9453   }
 9454   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9455   emit_int8(0x1F);
 9456   emit_operand(dst, src, 0);
 9457 }
 9458 
 9459 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9460   assert(VM_Version::supports_evex(), "");
 9461   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9462   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9463   attributes.set_is_evex_instruction();
 9464   attributes.set_embedded_opmask_register_specifier(mask);
 9465   if (merge) {
 9466     attributes.reset_is_clear_context();
 9467   }
 9468   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9469   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9470 }
 9471 
 9472 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9473   InstructionMark im(this);
 9474   assert(VM_Version::supports_evex(), "");
 9475   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9476   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9477   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9478   attributes.set_is_evex_instruction();
 9479   attributes.set_embedded_opmask_register_specifier(mask);
 9480   if (merge) {
 9481     attributes.reset_is_clear_context();
 9482   }
 9483   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9484   emit_int8((unsigned char)0xA8);
 9485   emit_operand(dst, src, 0);
 9486 }
 9487 
 9488 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9489   assert(VM_Version::supports_evex(), "");
 9490   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9491   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9492   attributes.set_is_evex_instruction();
 9493   attributes.set_embedded_opmask_register_specifier(mask);
 9494   if (merge) {
 9495     attributes.reset_is_clear_context();
 9496   }
 9497   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9498   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9499 }
 9500 
 9501 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9502   InstructionMark im(this);
 9503   assert(VM_Version::supports_evex(), "");
 9504   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9505   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9506   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9507   attributes.set_is_evex_instruction();
 9508   attributes.set_embedded_opmask_register_specifier(mask);
 9509   if (merge) {
 9510     attributes.reset_is_clear_context();
 9511   }
 9512   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9513   emit_int8((unsigned char)0xA8);
 9514   emit_operand(dst, src, 0);
 9515 }
 9516 
 9517 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9518   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9519   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9520   attributes.set_is_evex_instruction();
 9521   attributes.set_embedded_opmask_register_specifier(mask);
 9522   if (merge) {
 9523     attributes.reset_is_clear_context();
 9524   }
 9525   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9526   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9527 }
 9528 
 9529 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9530   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9531   InstructionMark im(this);
 9532   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9533   attributes.set_is_evex_instruction();
 9534   attributes.set_embedded_opmask_register_specifier(mask);
 9535   if (merge) {
 9536     attributes.reset_is_clear_context();
 9537   }
 9538   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9539   emit_int8((unsigned char)0x8D);
 9540   emit_operand(dst, src, 0);
 9541 }
 9542 
 9543 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9544   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9545   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9546   attributes.set_is_evex_instruction();
 9547   attributes.set_embedded_opmask_register_specifier(mask);
 9548   if (merge) {
 9549     attributes.reset_is_clear_context();
 9550   }
 9551   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9552   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9553 }
 9554 
 9555 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9556   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9557   InstructionMark im(this);
 9558   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9559   attributes.set_is_evex_instruction();
 9560   attributes.set_embedded_opmask_register_specifier(mask);
 9561   if (merge) {
 9562     attributes.reset_is_clear_context();
 9563   }
 9564   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9565   emit_int8((unsigned char)0x8D);
 9566   emit_operand(dst, src, 0);
 9567 }
 9568 
 9569 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9570   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9571   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9572   attributes.set_is_evex_instruction();
 9573   attributes.set_embedded_opmask_register_specifier(mask);
 9574   if (merge) {
 9575     attributes.reset_is_clear_context();
 9576   }
 9577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9578   emit_int16(0x36, (0xC0 | encode));
 9579 }
 9580 
 9581 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9582   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9583   InstructionMark im(this);
 9584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9585   attributes.set_is_evex_instruction();
 9586   attributes.set_embedded_opmask_register_specifier(mask);
 9587   if (merge) {
 9588     attributes.reset_is_clear_context();
 9589   }
 9590   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9591   emit_int8(0x36);
 9592   emit_operand(dst, src, 0);
 9593 }
 9594 
 9595 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9596   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9597   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9598   attributes.set_is_evex_instruction();
 9599   attributes.set_embedded_opmask_register_specifier(mask);
 9600   if (merge) {
 9601     attributes.reset_is_clear_context();
 9602   }
 9603   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9604   emit_int16(0x36, (0xC0 | encode));
 9605 }
 9606 
 9607 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9608   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9609   InstructionMark im(this);
 9610   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9611   attributes.set_is_evex_instruction();
 9612   attributes.set_embedded_opmask_register_specifier(mask);
 9613   if (merge) {
 9614     attributes.reset_is_clear_context();
 9615   }
 9616   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9617   emit_int8(0x36);
 9618   emit_operand(dst, src, 0);
 9619 }
 9620 
 9621 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9622   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9623   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9624   attributes.set_is_evex_instruction();
 9625   attributes.set_embedded_opmask_register_specifier(mask);
 9626   if (merge) {
 9627     attributes.reset_is_clear_context();
 9628   }
 9629   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9630   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9631 }
 9632 
 9633 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9634   assert(VM_Version::supports_evex(), "");
 9635   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9636   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9637   attributes.set_is_evex_instruction();
 9638   attributes.set_embedded_opmask_register_specifier(mask);
 9639   if (merge) {
 9640     attributes.reset_is_clear_context();
 9641   }
 9642   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9643   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9644 }
 9645 
 9646 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9647   assert(VM_Version::supports_evex(), "");
 9648   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9649   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9650   attributes.set_is_evex_instruction();
 9651   attributes.set_embedded_opmask_register_specifier(mask);
 9652   if (merge) {
 9653     attributes.reset_is_clear_context();
 9654   }
 9655   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9656   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9657 }
 9658 
 9659 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9660   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9661   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9662   attributes.set_is_evex_instruction();
 9663   attributes.set_embedded_opmask_register_specifier(mask);
 9664   if (merge) {
 9665     attributes.reset_is_clear_context();
 9666   }
 9667   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9668   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9669 }
 9670 
 9671 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9672   assert(VM_Version::supports_evex(), "");
 9673   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9674   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9675   attributes.set_is_evex_instruction();
 9676   attributes.set_embedded_opmask_register_specifier(mask);
 9677   if (merge) {
 9678     attributes.reset_is_clear_context();
 9679   }
 9680   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9681   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9682 }
 9683 
 9684 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9685   assert(VM_Version::supports_evex(), "");
 9686   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9687   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9688   attributes.set_is_evex_instruction();
 9689   attributes.set_embedded_opmask_register_specifier(mask);
 9690   if (merge) {
 9691     attributes.reset_is_clear_context();
 9692   }
 9693   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9694   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9695 }
 9696 
 9697 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9698   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9700   attributes.set_is_evex_instruction();
 9701   attributes.set_embedded_opmask_register_specifier(mask);
 9702   if (merge) {
 9703     attributes.reset_is_clear_context();
 9704   }
 9705   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9706   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9707 }
 9708 
 9709 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9710   assert(VM_Version::supports_evex(), "");
 9711   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9713   attributes.set_is_evex_instruction();
 9714   attributes.set_embedded_opmask_register_specifier(mask);
 9715   if (merge) {
 9716     attributes.reset_is_clear_context();
 9717   }
 9718   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9719   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9720 }
 9721 
 9722 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9723   assert(VM_Version::supports_evex(), "");
 9724   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9725   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9726   attributes.set_is_evex_instruction();
 9727   attributes.set_embedded_opmask_register_specifier(mask);
 9728   if (merge) {
 9729     attributes.reset_is_clear_context();
 9730   }
 9731   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9732   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9733 }
 9734 
 9735 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9736   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9737   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9738   attributes.set_is_evex_instruction();
 9739   attributes.set_embedded_opmask_register_specifier(mask);
 9740   if (merge) {
 9741     attributes.reset_is_clear_context();
 9742   }
 9743   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9744   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9745 }
 9746 
 9747 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9748   assert(VM_Version::supports_evex(), "");
 9749   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9750   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9751   attributes.set_is_evex_instruction();
 9752   attributes.set_embedded_opmask_register_specifier(mask);
 9753   if (merge) {
 9754     attributes.reset_is_clear_context();
 9755   }
 9756   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9757   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9758 }
 9759 
 9760 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9761   assert(VM_Version::supports_evex(), "");
 9762   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9763   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9764   attributes.set_is_evex_instruction();
 9765   attributes.set_embedded_opmask_register_specifier(mask);
 9766   if (merge) {
 9767     attributes.reset_is_clear_context();
 9768   }
 9769   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9770   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9771 }
 9772 
 9773 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9774   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9775   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9776   attributes.set_is_evex_instruction();
 9777   attributes.set_embedded_opmask_register_specifier(mask);
 9778   if (merge) {
 9779     attributes.reset_is_clear_context();
 9780   }
 9781   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9782   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9783 }
 9784 
 9785 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9786   assert(VM_Version::supports_evex(), "");
 9787   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9788   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9789   attributes.set_is_evex_instruction();
 9790   attributes.set_embedded_opmask_register_specifier(mask);
 9791   if (merge) {
 9792     attributes.reset_is_clear_context();
 9793   }
 9794   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9795   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9796 }
 9797 
 9798 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9799   assert(VM_Version::supports_evex(), "");
 9800   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9801   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9802   attributes.set_is_evex_instruction();
 9803   attributes.set_embedded_opmask_register_specifier(mask);
 9804   if (merge) {
 9805     attributes.reset_is_clear_context();
 9806   }
 9807   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9808   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9809 }
 9810 
 9811 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9812   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9813   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9814   attributes.set_is_evex_instruction();
 9815   attributes.set_embedded_opmask_register_specifier(mask);
 9816   if (merge) {
 9817     attributes.reset_is_clear_context();
 9818   }
 9819   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9820   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9821 }
 9822 
 9823 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9824   assert(VM_Version::supports_evex(), "");
 9825   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9827   attributes.set_is_evex_instruction();
 9828   attributes.set_embedded_opmask_register_specifier(mask);
 9829   if (merge) {
 9830     attributes.reset_is_clear_context();
 9831   }
 9832   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9833   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9834 }
 9835 
 9836 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9837   assert(VM_Version::supports_evex(), "");
 9838   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9839   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9840   attributes.set_is_evex_instruction();
 9841   attributes.set_embedded_opmask_register_specifier(mask);
 9842   if (merge) {
 9843     attributes.reset_is_clear_context();
 9844   }
 9845   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9846   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9847 }
 9848 
 9849 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9850   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9851   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9852   attributes.set_is_evex_instruction();
 9853   attributes.set_embedded_opmask_register_specifier(mask);
 9854   if (merge) {
 9855     attributes.reset_is_clear_context();
 9856   }
 9857   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9858   emit_int16(0x12, (0xC0 | encode));
 9859 }
 9860 
 9861 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9862   assert(VM_Version::supports_evex(), "");
 9863   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9864   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9865   attributes.set_is_evex_instruction();
 9866   attributes.set_embedded_opmask_register_specifier(mask);
 9867   if (merge) {
 9868     attributes.reset_is_clear_context();
 9869   }
 9870   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9871   emit_int16(0x47, (0xC0 | encode));
 9872 }
 9873 
 9874 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9875   assert(VM_Version::supports_evex(), "");
 9876   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9877   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9878   attributes.set_is_evex_instruction();
 9879   attributes.set_embedded_opmask_register_specifier(mask);
 9880   if (merge) {
 9881     attributes.reset_is_clear_context();
 9882   }
 9883   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9884   emit_int16(0x47, (0xC0 | encode));
 9885 }
 9886 
 9887 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9888   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9889   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9890   attributes.set_is_evex_instruction();
 9891   attributes.set_embedded_opmask_register_specifier(mask);
 9892   if (merge) {
 9893     attributes.reset_is_clear_context();
 9894   }
 9895   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9896   emit_int16(0x10, (0xC0 | encode));
 9897 }
 9898 
 9899 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9900   assert(VM_Version::supports_evex(), "");
 9901   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9902   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9903   attributes.set_is_evex_instruction();
 9904   attributes.set_embedded_opmask_register_specifier(mask);
 9905   if (merge) {
 9906     attributes.reset_is_clear_context();
 9907   }
 9908   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9909   emit_int16(0x45, (0xC0 | encode));
 9910 }
 9911 
 9912 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9913   assert(VM_Version::supports_evex(), "");
 9914   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9915   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9916   attributes.set_is_evex_instruction();
 9917   attributes.set_embedded_opmask_register_specifier(mask);
 9918   if (merge) {
 9919     attributes.reset_is_clear_context();
 9920   }
 9921   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9922   emit_int16(0x45, (0xC0 | encode));
 9923 }
 9924 
 9925 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9926   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9927   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9928   attributes.set_is_evex_instruction();
 9929   attributes.set_embedded_opmask_register_specifier(mask);
 9930   if (merge) {
 9931     attributes.reset_is_clear_context();
 9932   }
 9933   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9934   emit_int16(0x11, (0xC0 | encode));
 9935 }
 9936 
 9937 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9938   assert(VM_Version::supports_evex(), "");
 9939   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9940   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9941   attributes.set_is_evex_instruction();
 9942   attributes.set_embedded_opmask_register_specifier(mask);
 9943   if (merge) {
 9944     attributes.reset_is_clear_context();
 9945   }
 9946   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9947   emit_int16(0x46, (0xC0 | encode));
 9948 }
 9949 
 9950 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9951   assert(VM_Version::supports_evex(), "");
 9952   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9953   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9954   attributes.set_is_evex_instruction();
 9955   attributes.set_embedded_opmask_register_specifier(mask);
 9956   if (merge) {
 9957     attributes.reset_is_clear_context();
 9958   }
 9959   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9960   emit_int16(0x46, (0xC0 | encode));
 9961 }
 9962 
 9963 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9964   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9965   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9966   attributes.set_is_evex_instruction();
 9967   attributes.set_embedded_opmask_register_specifier(mask);
 9968   if (merge) {
 9969     attributes.reset_is_clear_context();
 9970   }
 9971   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9972   emit_int16(0x38, (0xC0 | encode));
 9973 }
 9974 
 9975 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9976   assert(VM_Version::supports_avx512bw(), "");
 9977   InstructionMark im(this);
 9978   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9979   attributes.set_is_evex_instruction();
 9980   attributes.set_embedded_opmask_register_specifier(mask);
 9981   if (merge) {
 9982     attributes.reset_is_clear_context();
 9983   }
 9984   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9985   emit_int8(0x38);
 9986   emit_operand(dst, src, 0);
 9987 }
 9988 
 9989 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9990   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9991   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9992   attributes.set_is_evex_instruction();
 9993   attributes.set_embedded_opmask_register_specifier(mask);
 9994   if (merge) {
 9995     attributes.reset_is_clear_context();
 9996   }
 9997   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9998   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9999 }
10000 
10001 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10002   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10003   InstructionMark im(this);
10004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10005   attributes.set_is_evex_instruction();
10006   attributes.set_embedded_opmask_register_specifier(mask);
10007   if (merge) {
10008     attributes.reset_is_clear_context();
10009   }
10010   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10011   emit_int8((unsigned char)0xEA);
10012   emit_operand(dst, src, 0);
10013 }
10014 
10015 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10016   assert(VM_Version::supports_evex(), "");
10017   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10018   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10019   attributes.set_is_evex_instruction();
10020   attributes.set_embedded_opmask_register_specifier(mask);
10021   if (merge) {
10022     attributes.reset_is_clear_context();
10023   }
10024   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10025   emit_int16(0x39, (0xC0 | encode));
10026 }
10027 
10028 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10029   assert(VM_Version::supports_evex(), "");
10030   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10031   InstructionMark im(this);
10032   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10033   attributes.set_is_evex_instruction();
10034   attributes.set_embedded_opmask_register_specifier(mask);
10035   if (merge) {
10036     attributes.reset_is_clear_context();
10037   }
10038   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10039   emit_int8(0x39);
10040   emit_operand(dst, src, 0);
10041 }
10042 
10043 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10044   assert(VM_Version::supports_evex(), "");
10045   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10046   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10047   attributes.set_is_evex_instruction();
10048   attributes.set_embedded_opmask_register_specifier(mask);
10049   if (merge) {
10050     attributes.reset_is_clear_context();
10051   }
10052   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10053   emit_int16(0x39, (0xC0 | encode));
10054 }
10055 
10056 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10057   assert(VM_Version::supports_evex(), "");
10058   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10059   InstructionMark im(this);
10060   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10061   attributes.set_is_evex_instruction();
10062   attributes.set_embedded_opmask_register_specifier(mask);
10063   if (merge) {
10064     attributes.reset_is_clear_context();
10065   }
10066   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10067   emit_int8(0x39);
10068   emit_operand(dst, src, 0);
10069 }
10070 
10071 
10072 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10073   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10074   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10075   attributes.set_is_evex_instruction();
10076   attributes.set_embedded_opmask_register_specifier(mask);
10077   if (merge) {
10078     attributes.reset_is_clear_context();
10079   }
10080   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10081   emit_int16(0x3C, (0xC0 | encode));
10082 }
10083 
10084 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10085   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10086   InstructionMark im(this);
10087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10088   attributes.set_is_evex_instruction();
10089   attributes.set_embedded_opmask_register_specifier(mask);
10090   if (merge) {
10091     attributes.reset_is_clear_context();
10092   }
10093   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10094   emit_int8(0x3C);
10095   emit_operand(dst, src, 0);
10096 }
10097 
10098 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10099   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10100   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10101   attributes.set_is_evex_instruction();
10102   attributes.set_embedded_opmask_register_specifier(mask);
10103   if (merge) {
10104     attributes.reset_is_clear_context();
10105   }
10106   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10107   emit_int16((unsigned char)0xEE, (0xC0 | encode));
10108 }
10109 
10110 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10111   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10112   InstructionMark im(this);
10113   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10114   attributes.set_is_evex_instruction();
10115   attributes.set_embedded_opmask_register_specifier(mask);
10116   if (merge) {
10117     attributes.reset_is_clear_context();
10118   }
10119   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10120   emit_int8((unsigned char)0xEE);
10121   emit_operand(dst, src, 0);
10122 }
10123 
10124 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10125   assert(VM_Version::supports_evex(), "");
10126   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10127   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10128   attributes.set_is_evex_instruction();
10129   attributes.set_embedded_opmask_register_specifier(mask);
10130   if (merge) {
10131     attributes.reset_is_clear_context();
10132   }
10133   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10134   emit_int16(0x3D, (0xC0 | encode));
10135 }
10136 
10137 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10138   assert(VM_Version::supports_evex(), "");
10139   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10140   InstructionMark im(this);
10141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10142   attributes.set_is_evex_instruction();
10143   attributes.set_embedded_opmask_register_specifier(mask);
10144   if (merge) {
10145     attributes.reset_is_clear_context();
10146   }
10147   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10148   emit_int8(0x3D);
10149   emit_operand(dst, src, 0);
10150 }
10151 
10152 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10153   assert(VM_Version::supports_evex(), "");
10154   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10155   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10156   attributes.set_is_evex_instruction();
10157   attributes.set_embedded_opmask_register_specifier(mask);
10158   if (merge) {
10159     attributes.reset_is_clear_context();
10160   }
10161   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10162   emit_int16(0x3D, (0xC0 | encode));
10163 }
10164 
10165 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10166   assert(VM_Version::supports_evex(), "");
10167   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10168   InstructionMark im(this);
10169   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10170   attributes.set_is_evex_instruction();
10171   attributes.set_embedded_opmask_register_specifier(mask);
10172   if (merge) {
10173     attributes.reset_is_clear_context();
10174   }
10175   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10176   emit_int8(0x3D);
10177   emit_operand(dst, src, 0);
10178 }
10179 
10180 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10181   assert(VM_Version::supports_evex(), "requires EVEX support");
10182   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10184   attributes.set_is_evex_instruction();
10185   attributes.set_embedded_opmask_register_specifier(mask);
10186   if (merge) {
10187     attributes.reset_is_clear_context();
10188   }
10189   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10190   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10191 }
10192 
10193 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10194   assert(VM_Version::supports_evex(), "requires EVEX support");
10195   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10196   assert(dst != xnoreg, "sanity");
10197   InstructionMark im(this);
10198   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10199   attributes.set_is_evex_instruction();
10200   attributes.set_embedded_opmask_register_specifier(mask);
10201   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10202   if (merge) {
10203     attributes.reset_is_clear_context();
10204   }
10205   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10206   emit_int8(0x25);
10207   emit_operand(dst, src3, 1);
10208   emit_int8(imm8);
10209 }
10210 
10211 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10212   assert(VM_Version::supports_evex(), "requires EVEX support");
10213   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10214   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10215   attributes.set_is_evex_instruction();
10216   attributes.set_embedded_opmask_register_specifier(mask);
10217   if (merge) {
10218     attributes.reset_is_clear_context();
10219   }
10220   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10221   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10222 }
10223 
10224 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10225   assert(VM_Version::supports_evex(), "requires EVEX support");
10226   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10227   assert(dst != xnoreg, "sanity");
10228   InstructionMark im(this);
10229   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10230   attributes.set_is_evex_instruction();
10231   attributes.set_embedded_opmask_register_specifier(mask);
10232   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10233   if (merge) {
10234     attributes.reset_is_clear_context();
10235   }
10236   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10237   emit_int8(0x25);
10238   emit_operand(dst, src3, 1);
10239   emit_int8(imm8);
10240 }
10241 
10242 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10243   assert(VM_Version::supports_gfni(), "");
10244   assert(VM_Version::supports_sse(), "");
10245   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10246   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10247   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10248 }
10249 
10250 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10251   assert(VM_Version::supports_gfni(), "requires GFNI support");
10252   assert(VM_Version::supports_sse(), "");
10253   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10254   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10255   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10256 }
10257 
10258 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10259 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10260   assert(UseAVX >= 2, "");
10261   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10263   emit_int16(0x58, (0xC0 | encode));
10264 }
10265 
10266 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10267   assert(VM_Version::supports_avx2(), "");
10268   assert(dst != xnoreg, "sanity");
10269   InstructionMark im(this);
10270   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10271   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10272   // swap src<->dst for encoding
10273   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10274   emit_int8(0x58);
10275   emit_operand(dst, src, 0);
10276 }
10277 
10278 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10279 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10280   assert(VM_Version::supports_avx2(), "");
10281   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10282   attributes.set_rex_vex_w_reverted();
10283   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10284   emit_int16(0x59, (0xC0 | encode));
10285 }
10286 
10287 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10288   assert(VM_Version::supports_avx2(), "");
10289   assert(dst != xnoreg, "sanity");
10290   InstructionMark im(this);
10291   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10292   attributes.set_rex_vex_w_reverted();
10293   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10294   // swap src<->dst for encoding
10295   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10296   emit_int8(0x59);
10297   emit_operand(dst, src, 0);
10298 }
10299 
10300 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10301   assert(vector_len != Assembler::AVX_128bit, "");
10302   assert(VM_Version::supports_evex(), "");
10303   assert(dst != xnoreg, "sanity");
10304   InstructionMark im(this);
10305   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10306   attributes.set_rex_vex_w_reverted();
10307   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10308   // swap src<->dst for encoding
10309   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10310   emit_int8(0x5A);
10311   emit_operand(dst, src, 0);
10312 }
10313 
10314 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10315   assert(vector_len != Assembler::AVX_128bit, "");
10316   assert(VM_Version::supports_avx512dq(), "");
10317   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10318   attributes.set_rex_vex_w_reverted();
10319   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10320   emit_int16(0x5A, (0xC0 | encode));
10321 }
10322 
10323 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10324   assert(vector_len != Assembler::AVX_128bit, "");
10325   assert(VM_Version::supports_avx512dq(), "");
10326   assert(dst != xnoreg, "sanity");
10327   InstructionMark im(this);
10328   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10329   attributes.set_rex_vex_w_reverted();
10330   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10331   // swap src<->dst for encoding
10332   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10333   emit_int8(0x5A);
10334   emit_operand(dst, src, 0);
10335 }
10336 
10337 // scalar single/double precision replicate
10338 
10339 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10340 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10341   assert(VM_Version::supports_avx2(), "");
10342   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10343   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10344   emit_int16(0x18, (0xC0 | encode));
10345 }
10346 
10347 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10348   assert(VM_Version::supports_avx(), "");
10349   assert(dst != xnoreg, "sanity");
10350   InstructionMark im(this);
10351   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10352   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10353   // swap src<->dst for encoding
10354   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10355   emit_int8(0x18);
10356   emit_operand(dst, src, 0);
10357 }
10358 
10359 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10360 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10361   assert(VM_Version::supports_avx2(), "");
10362   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10363   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10364   attributes.set_rex_vex_w_reverted();
10365   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10366   emit_int16(0x19, (0xC0 | encode));
10367 }
10368 
10369 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10370   assert(VM_Version::supports_avx(), "");
10371   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10372   assert(dst != xnoreg, "sanity");
10373   InstructionMark im(this);
10374   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10375   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10376   attributes.set_rex_vex_w_reverted();
10377   // swap src<->dst for encoding
10378   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10379   emit_int8(0x19);
10380   emit_operand(dst, src, 0);
10381 }
10382 
10383 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10384   assert(VM_Version::supports_avx(), "");
10385   assert(vector_len == AVX_256bit, "");
10386   assert(dst != xnoreg, "sanity");
10387   InstructionMark im(this);
10388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10389   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10390   // swap src<->dst for encoding
10391   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10392   emit_int8(0x1A);
10393   emit_operand(dst, src, 0);
10394 }
10395 
10396 // gpr source broadcast forms
10397 
10398 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10399 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10400   assert(VM_Version::supports_avx512bw(), "");
10401   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10402   attributes.set_is_evex_instruction();
10403   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10404   emit_int16(0x7A, (0xC0 | encode));
10405 }
10406 
10407 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10408 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10409   assert(VM_Version::supports_avx512bw(), "");
10410   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10411   attributes.set_is_evex_instruction();
10412   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10413   emit_int16(0x7B, (0xC0 | encode));
10414 }
10415 
10416 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10417 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10418   assert(VM_Version::supports_evex(), "");
10419   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10420   attributes.set_is_evex_instruction();
10421   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10422   emit_int16(0x7C, (0xC0 | encode));
10423 }
10424 
10425 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10426 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10427   assert(VM_Version::supports_evex(), "");
10428   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10429   attributes.set_is_evex_instruction();
10430   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10431   emit_int16(0x7C, (0xC0 | encode));
10432 }
10433 
10434 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10435   assert(VM_Version::supports_avx2(), "");
10436   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10437   assert(dst != xnoreg, "sanity");
10438   assert(src.isxmmindex(),"expected to be xmm index");
10439   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10440   InstructionMark im(this);
10441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10442   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10443   emit_int8((unsigned char)0x90);
10444   emit_operand(dst, src, 0);
10445 }
10446 
10447 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10448   assert(VM_Version::supports_avx2(), "");
10449   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10450   assert(dst != xnoreg, "sanity");
10451   assert(src.isxmmindex(),"expected to be xmm index");
10452   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10453   InstructionMark im(this);
10454   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10455   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10456   emit_int8((unsigned char)0x90);
10457   emit_operand(dst, src, 0);
10458 }
10459 
10460 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10461   assert(VM_Version::supports_avx2(), "");
10462   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10463   assert(dst != xnoreg, "sanity");
10464   assert(src.isxmmindex(),"expected to be xmm index");
10465   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10466   InstructionMark im(this);
10467   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10468   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10469   emit_int8((unsigned char)0x92);
10470   emit_operand(dst, src, 0);
10471 }
10472 
10473 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10474   assert(VM_Version::supports_avx2(), "");
10475   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10476   assert(dst != xnoreg, "sanity");
10477   assert(src.isxmmindex(),"expected to be xmm index");
10478   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10479   InstructionMark im(this);
10480   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10481   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10482   emit_int8((unsigned char)0x92);
10483   emit_operand(dst, src, 0);
10484 }
10485 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10486   assert(VM_Version::supports_evex(), "");
10487   assert(dst != xnoreg, "sanity");
10488   assert(src.isxmmindex(),"expected to be xmm index");
10489   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10490   assert(mask != k0, "instruction will #UD if mask is in k0");
10491   InstructionMark im(this);
10492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10493   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10494   attributes.reset_is_clear_context();
10495   attributes.set_embedded_opmask_register_specifier(mask);
10496   attributes.set_is_evex_instruction();
10497   // swap src<->dst for encoding
10498   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10499   emit_int8((unsigned char)0x90);
10500   emit_operand(dst, src, 0);
10501 }
10502 
10503 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10504   assert(VM_Version::supports_evex(), "");
10505   assert(dst != xnoreg, "sanity");
10506   assert(src.isxmmindex(),"expected to be xmm index");
10507   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10508   assert(mask != k0, "instruction will #UD if mask is in k0");
10509   InstructionMark im(this);
10510   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10511   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10512   attributes.reset_is_clear_context();
10513   attributes.set_embedded_opmask_register_specifier(mask);
10514   attributes.set_is_evex_instruction();
10515   // swap src<->dst for encoding
10516   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10517   emit_int8((unsigned char)0x90);
10518   emit_operand(dst, src, 0);
10519 }
10520 
10521 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10522   assert(VM_Version::supports_evex(), "");
10523   assert(dst != xnoreg, "sanity");
10524   assert(src.isxmmindex(),"expected to be xmm index");
10525   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10526   assert(mask != k0, "instruction will #UD if mask is in k0");
10527   InstructionMark im(this);
10528   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10529   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10530   attributes.reset_is_clear_context();
10531   attributes.set_embedded_opmask_register_specifier(mask);
10532   attributes.set_is_evex_instruction();
10533   // swap src<->dst for encoding
10534   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10535   emit_int8((unsigned char)0x92);
10536   emit_operand(dst, src, 0);
10537 }
10538 
10539 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10540   assert(VM_Version::supports_evex(), "");
10541   assert(dst != xnoreg, "sanity");
10542   assert(src.isxmmindex(),"expected to be xmm index");
10543   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10544   assert(mask != k0, "instruction will #UD if mask is in k0");
10545   InstructionMark im(this);
10546   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10547   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10548   attributes.reset_is_clear_context();
10549   attributes.set_embedded_opmask_register_specifier(mask);
10550   attributes.set_is_evex_instruction();
10551   // swap src<->dst for encoding
10552   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10553   emit_int8((unsigned char)0x92);
10554   emit_operand(dst, src, 0);
10555 }
10556 
10557 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10558   assert(VM_Version::supports_evex(), "");
10559   assert(mask != k0, "instruction will #UD if mask is in k0");
10560   InstructionMark im(this);
10561   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10562   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10563   attributes.reset_is_clear_context();
10564   attributes.set_embedded_opmask_register_specifier(mask);
10565   attributes.set_is_evex_instruction();
10566   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10567   emit_int8((unsigned char)0xA0);
10568   emit_operand(src, dst, 0);
10569 }
10570 
10571 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10572   assert(VM_Version::supports_evex(), "");
10573   assert(mask != k0, "instruction will #UD if mask is in k0");
10574   InstructionMark im(this);
10575   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10576   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10577   attributes.reset_is_clear_context();
10578   attributes.set_embedded_opmask_register_specifier(mask);
10579   attributes.set_is_evex_instruction();
10580   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10581   emit_int8((unsigned char)0xA0);
10582   emit_operand(src, dst, 0);
10583 }
10584 
10585 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10586   assert(VM_Version::supports_evex(), "");
10587   assert(mask != k0, "instruction will #UD if mask is in k0");
10588   InstructionMark im(this);
10589   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10590   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10591   attributes.reset_is_clear_context();
10592   attributes.set_embedded_opmask_register_specifier(mask);
10593   attributes.set_is_evex_instruction();
10594   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10595   emit_int8((unsigned char)0xA2);
10596   emit_operand(src, dst, 0);
10597 }
10598 
10599 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10600   assert(VM_Version::supports_evex(), "");
10601   assert(mask != k0, "instruction will #UD if mask is in k0");
10602   InstructionMark im(this);
10603   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10604   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10605   attributes.reset_is_clear_context();
10606   attributes.set_embedded_opmask_register_specifier(mask);
10607   attributes.set_is_evex_instruction();
10608   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10609   emit_int8((unsigned char)0xA2);
10610   emit_operand(src, dst, 0);
10611 }
10612 // Carry-Less Multiplication Quadword
10613 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10614   assert(VM_Version::supports_clmul(), "");
10615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10616   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10617   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10618 }
10619 
10620 // Carry-Less Multiplication Quadword
10621 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10622   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10623   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10624   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10625   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10626 }
10627 
10628 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10629   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10630   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10631   attributes.set_is_evex_instruction();
10632   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10633   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10634 }
10635 
10636 void Assembler::vzeroupper_uncached() {
10637   if (VM_Version::supports_vzeroupper()) {
10638     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10639     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10640     emit_int8(0x77);
10641   }
10642 }
10643 
10644 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10645   // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10646   assert(VM_Version::supports_evex(), "");
10647   assert(VM_Version::supports_avx512dq(), "");
10648   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10649   attributes.set_is_evex_instruction();
10650   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10651   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10652 }
10653 
10654 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10655   // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10656   assert(VM_Version::supports_evex(), "");
10657   assert(VM_Version::supports_avx512dq(), "");
10658   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10659   attributes.set_is_evex_instruction();
10660   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10661   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10662 }
10663 
10664 void Assembler::fld_x(Address adr) {
10665   InstructionMark im(this);
10666   emit_int8((unsigned char)0xDB);
10667   emit_operand32(rbp, adr, 0);
10668 }
10669 
10670 void Assembler::fstp_x(Address adr) {
10671   InstructionMark im(this);
10672   emit_int8((unsigned char)0xDB);
10673   emit_operand32(rdi, adr, 0);
10674 }
10675 
10676 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10677   assert(reg->encoding() < 8, "no extended registers");
10678   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10679   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10680 }
10681 
10682 #ifndef _LP64
10683 // 32bit only pieces of the assembler
10684 
10685 void Assembler::emms() {
10686   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10687   emit_int16(0x0F, 0x77);
10688 }
10689 
10690 void Assembler::vzeroupper() {
10691   vzeroupper_uncached();
10692 }
10693 
10694 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10695   // NO PREFIX AS NEVER 64BIT
10696   InstructionMark im(this);
10697   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10698   emit_data(imm32, rspec, 0);
10699 }
10700 
10701 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10702   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10703   InstructionMark im(this);
10704   emit_int8((unsigned char)0x81);
10705   emit_operand(rdi, src1, 4);
10706   emit_data(imm32, rspec, 0);
10707 }
10708 
10709 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10710 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10711 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10712 void Assembler::cmpxchg8(Address adr) {
10713   InstructionMark im(this);
10714   emit_int16(0x0F, (unsigned char)0xC7);
10715   emit_operand(rcx, adr, 0);
10716 }
10717 
10718 void Assembler::decl(Register dst) {
10719   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10720  emit_int8(0x48 | dst->encoding());
10721 }
10722 
10723 // 64bit doesn't use the x87
10724 
10725 void Assembler::emit_farith(int b1, int b2, int i) {
10726   assert(isByte(b1) && isByte(b2), "wrong opcode");
10727   assert(0 <= i &&  i < 8, "illegal stack offset");
10728   emit_int16(b1, b2 + i);
10729 }
10730 
10731 void Assembler::fabs() {
10732   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10733 }
10734 
10735 void Assembler::fadd(int i) {
10736   emit_farith(0xD8, 0xC0, i);
10737 }
10738 
10739 void Assembler::fadd_d(Address src) {
10740   InstructionMark im(this);
10741   emit_int8((unsigned char)0xDC);
10742   emit_operand32(rax, src, 0);
10743 }
10744 
10745 void Assembler::fadd_s(Address src) {
10746   InstructionMark im(this);
10747   emit_int8((unsigned char)0xD8);
10748   emit_operand32(rax, src, 0);
10749 }
10750 
10751 void Assembler::fadda(int i) {
10752   emit_farith(0xDC, 0xC0, i);
10753 }
10754 
10755 void Assembler::faddp(int i) {
10756   emit_farith(0xDE, 0xC0, i);
10757 }
10758 
10759 void Assembler::fchs() {
10760   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10761 }
10762 
10763 void Assembler::fcom(int i) {
10764   emit_farith(0xD8, 0xD0, i);
10765 }
10766 
10767 void Assembler::fcomp(int i) {
10768   emit_farith(0xD8, 0xD8, i);
10769 }
10770 
10771 void Assembler::fcomp_d(Address src) {
10772   InstructionMark im(this);
10773   emit_int8((unsigned char)0xDC);
10774   emit_operand32(rbx, src, 0);
10775 }
10776 
10777 void Assembler::fcomp_s(Address src) {
10778   InstructionMark im(this);
10779   emit_int8((unsigned char)0xD8);
10780   emit_operand32(rbx, src, 0);
10781 }
10782 
10783 void Assembler::fcompp() {
10784   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10785 }
10786 
10787 void Assembler::fcos() {
10788   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10789 }
10790 
10791 void Assembler::fdecstp() {
10792   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10793 }
10794 
10795 void Assembler::fdiv(int i) {
10796   emit_farith(0xD8, 0xF0, i);
10797 }
10798 
10799 void Assembler::fdiv_d(Address src) {
10800   InstructionMark im(this);
10801   emit_int8((unsigned char)0xDC);
10802   emit_operand32(rsi, src, 0);
10803 }
10804 
10805 void Assembler::fdiv_s(Address src) {
10806   InstructionMark im(this);
10807   emit_int8((unsigned char)0xD8);
10808   emit_operand32(rsi, src, 0);
10809 }
10810 
10811 void Assembler::fdiva(int i) {
10812   emit_farith(0xDC, 0xF8, i);
10813 }
10814 
10815 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10816 //       is erroneous for some of the floating-point instructions below.
10817 
10818 void Assembler::fdivp(int i) {
10819   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10820 }
10821 
10822 void Assembler::fdivr(int i) {
10823   emit_farith(0xD8, 0xF8, i);
10824 }
10825 
10826 void Assembler::fdivr_d(Address src) {
10827   InstructionMark im(this);
10828   emit_int8((unsigned char)0xDC);
10829   emit_operand32(rdi, src, 0);
10830 }
10831 
10832 void Assembler::fdivr_s(Address src) {
10833   InstructionMark im(this);
10834   emit_int8((unsigned char)0xD8);
10835   emit_operand32(rdi, src, 0);
10836 }
10837 
10838 void Assembler::fdivra(int i) {
10839   emit_farith(0xDC, 0xF0, i);
10840 }
10841 
10842 void Assembler::fdivrp(int i) {
10843   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10844 }
10845 
10846 void Assembler::ffree(int i) {
10847   emit_farith(0xDD, 0xC0, i);
10848 }
10849 
10850 void Assembler::fild_d(Address adr) {
10851   InstructionMark im(this);
10852   emit_int8((unsigned char)0xDF);
10853   emit_operand32(rbp, adr, 0);
10854 }
10855 
10856 void Assembler::fild_s(Address adr) {
10857   InstructionMark im(this);
10858   emit_int8((unsigned char)0xDB);
10859   emit_operand32(rax, adr, 0);
10860 }
10861 
10862 void Assembler::fincstp() {
10863   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10864 }
10865 
10866 void Assembler::finit() {
10867   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10868 }
10869 
10870 void Assembler::fist_s(Address adr) {
10871   InstructionMark im(this);
10872   emit_int8((unsigned char)0xDB);
10873   emit_operand32(rdx, adr, 0);
10874 }
10875 
10876 void Assembler::fistp_d(Address adr) {
10877   InstructionMark im(this);
10878   emit_int8((unsigned char)0xDF);
10879   emit_operand32(rdi, adr, 0);
10880 }
10881 
10882 void Assembler::fistp_s(Address adr) {
10883   InstructionMark im(this);
10884   emit_int8((unsigned char)0xDB);
10885   emit_operand32(rbx, adr, 0);
10886 }
10887 
10888 void Assembler::fld1() {
10889   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10890 }
10891 
10892 void Assembler::fld_d(Address adr) {
10893   InstructionMark im(this);
10894   emit_int8((unsigned char)0xDD);
10895   emit_operand32(rax, adr, 0);
10896 }
10897 
10898 void Assembler::fld_s(Address adr) {
10899   InstructionMark im(this);
10900   emit_int8((unsigned char)0xD9);
10901   emit_operand32(rax, adr, 0);
10902 }
10903 
10904 
10905 void Assembler::fld_s(int index) {
10906   emit_farith(0xD9, 0xC0, index);
10907 }
10908 
10909 void Assembler::fldcw(Address src) {
10910   InstructionMark im(this);
10911   emit_int8((unsigned char)0xD9);
10912   emit_operand32(rbp, src, 0);
10913 }
10914 
10915 void Assembler::fldenv(Address src) {
10916   InstructionMark im(this);
10917   emit_int8((unsigned char)0xD9);
10918   emit_operand32(rsp, src, 0);
10919 }
10920 
10921 void Assembler::fldlg2() {
10922   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10923 }
10924 
10925 void Assembler::fldln2() {
10926   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10927 }
10928 
10929 void Assembler::fldz() {
10930   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10931 }
10932 
10933 void Assembler::flog() {
10934   fldln2();
10935   fxch();
10936   fyl2x();
10937 }
10938 
10939 void Assembler::flog10() {
10940   fldlg2();
10941   fxch();
10942   fyl2x();
10943 }
10944 
10945 void Assembler::fmul(int i) {
10946   emit_farith(0xD8, 0xC8, i);
10947 }
10948 
10949 void Assembler::fmul_d(Address src) {
10950   InstructionMark im(this);
10951   emit_int8((unsigned char)0xDC);
10952   emit_operand32(rcx, src, 0);
10953 }
10954 
10955 void Assembler::fmul_s(Address src) {
10956   InstructionMark im(this);
10957   emit_int8((unsigned char)0xD8);
10958   emit_operand32(rcx, src, 0);
10959 }
10960 
10961 void Assembler::fmula(int i) {
10962   emit_farith(0xDC, 0xC8, i);
10963 }
10964 
10965 void Assembler::fmulp(int i) {
10966   emit_farith(0xDE, 0xC8, i);
10967 }
10968 
10969 void Assembler::fnsave(Address dst) {
10970   InstructionMark im(this);
10971   emit_int8((unsigned char)0xDD);
10972   emit_operand32(rsi, dst, 0);
10973 }
10974 
10975 void Assembler::fnstcw(Address src) {
10976   InstructionMark im(this);
10977   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10978   emit_operand32(rdi, src, 0);
10979 }
10980 
10981 void Assembler::fnstsw_ax() {
10982   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10983 }
10984 
10985 void Assembler::fprem() {
10986   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10987 }
10988 
10989 void Assembler::fprem1() {
10990   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10991 }
10992 
10993 void Assembler::frstor(Address src) {
10994   InstructionMark im(this);
10995   emit_int8((unsigned char)0xDD);
10996   emit_operand32(rsp, src, 0);
10997 }
10998 
10999 void Assembler::fsin() {
11000   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
11001 }
11002 
11003 void Assembler::fsqrt() {
11004   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11005 }
11006 
11007 void Assembler::fst_d(Address adr) {
11008   InstructionMark im(this);
11009   emit_int8((unsigned char)0xDD);
11010   emit_operand32(rdx, adr, 0);
11011 }
11012 
11013 void Assembler::fst_s(Address adr) {
11014   InstructionMark im(this);
11015   emit_int8((unsigned char)0xD9);
11016   emit_operand32(rdx, adr, 0);
11017 }
11018 
11019 void Assembler::fstp_d(Address adr) {
11020   InstructionMark im(this);
11021   emit_int8((unsigned char)0xDD);
11022   emit_operand32(rbx, adr, 0);
11023 }
11024 
11025 void Assembler::fstp_d(int index) {
11026   emit_farith(0xDD, 0xD8, index);
11027 }
11028 
11029 void Assembler::fstp_s(Address adr) {
11030   InstructionMark im(this);
11031   emit_int8((unsigned char)0xD9);
11032   emit_operand32(rbx, adr, 0);
11033 }
11034 
11035 void Assembler::fsub(int i) {
11036   emit_farith(0xD8, 0xE0, i);
11037 }
11038 
11039 void Assembler::fsub_d(Address src) {
11040   InstructionMark im(this);
11041   emit_int8((unsigned char)0xDC);
11042   emit_operand32(rsp, src, 0);
11043 }
11044 
11045 void Assembler::fsub_s(Address src) {
11046   InstructionMark im(this);
11047   emit_int8((unsigned char)0xD8);
11048   emit_operand32(rsp, src, 0);
11049 }
11050 
11051 void Assembler::fsuba(int i) {
11052   emit_farith(0xDC, 0xE8, i);
11053 }
11054 
11055 void Assembler::fsubp(int i) {
11056   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11057 }
11058 
11059 void Assembler::fsubr(int i) {
11060   emit_farith(0xD8, 0xE8, i);
11061 }
11062 
11063 void Assembler::fsubr_d(Address src) {
11064   InstructionMark im(this);
11065   emit_int8((unsigned char)0xDC);
11066   emit_operand32(rbp, src, 0);
11067 }
11068 
11069 void Assembler::fsubr_s(Address src) {
11070   InstructionMark im(this);
11071   emit_int8((unsigned char)0xD8);
11072   emit_operand32(rbp, src, 0);
11073 }
11074 
11075 void Assembler::fsubra(int i) {
11076   emit_farith(0xDC, 0xE0, i);
11077 }
11078 
11079 void Assembler::fsubrp(int i) {
11080   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11081 }
11082 
11083 void Assembler::ftan() {
11084   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11085 }
11086 
11087 void Assembler::ftst() {
11088   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11089 }
11090 
11091 void Assembler::fucomi(int i) {
11092   // make sure the instruction is supported (introduced for P6, together with cmov)
11093   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11094   emit_farith(0xDB, 0xE8, i);
11095 }
11096 
11097 void Assembler::fucomip(int i) {
11098   // make sure the instruction is supported (introduced for P6, together with cmov)
11099   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11100   emit_farith(0xDF, 0xE8, i);
11101 }
11102 
11103 void Assembler::fwait() {
11104   emit_int8((unsigned char)0x9B);
11105 }
11106 
11107 void Assembler::fxch(int i) {
11108   emit_farith(0xD9, 0xC8, i);
11109 }
11110 
11111 void Assembler::fyl2x() {
11112   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11113 }
11114 
11115 void Assembler::frndint() {
11116   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11117 }
11118 
11119 void Assembler::f2xm1() {
11120   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11121 }
11122 
11123 void Assembler::fldl2e() {
11124   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11125 }
11126 #endif // !_LP64
11127 
11128 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11129 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11130 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11131 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
11132 
11133 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11134 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11135   if (pre > 0) {
11136     emit_int8(simd_pre[pre]);
11137   }
11138   if (rex_w) {
11139     prefixq(adr, xreg);
11140   } else {
11141     prefix(adr, xreg);
11142   }
11143   if (opc > 0) {
11144     emit_int8(0x0F);
11145     int opc2 = simd_opc[opc];
11146     if (opc2 > 0) {
11147       emit_int8(opc2);
11148     }
11149   }
11150 }
11151 
11152 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11153   if (pre > 0) {
11154     emit_int8(simd_pre[pre]);
11155   }
11156   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11157   if (opc > 0) {
11158     emit_int8(0x0F);
11159     int opc2 = simd_opc[opc];
11160     if (opc2 > 0) {
11161       emit_int8(opc2);
11162     }
11163   }
11164   return encode;
11165 }
11166 
11167 
11168 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11169   int vector_len = _attributes->get_vector_len();
11170   bool vex_w = _attributes->is_rex_vex_w();
11171   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11172     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11173     byte1 = (~byte1) & 0xE0;
11174     byte1 |= opc;
11175 
11176     int byte2 = ((~nds_enc) & 0xf) << 3;
11177     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11178 
11179     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11180   } else {
11181     int byte1 = vex_r ? VEX_R : 0;
11182     byte1 = (~byte1) & 0x80;
11183     byte1 |= ((~nds_enc) & 0xf) << 3;
11184     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11185     emit_int16((unsigned char)VEX_2bytes, byte1);
11186   }
11187 }
11188 
11189 // This is a 4 byte encoding
11190 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11191   // EVEX 0x62 prefix
11192   // byte1 = EVEX_4bytes;
11193 
11194   bool vex_w = _attributes->is_rex_vex_w();
11195   int evex_encoding = (vex_w ? VEX_W : 0);
11196   // EVEX.b is not currently used for broadcast of single element or data rounding modes
11197   _attributes->set_evex_encoding(evex_encoding);
11198 
11199   // P0: byte 2, initialized to RXBR`00mm
11200   // instead of not'd
11201   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11202   byte2 = (~byte2) & 0xF0;
11203   // confine opc opcode extensions in mm bits to lower two bits
11204   // of form {0F, 0F_38, 0F_3A}
11205   byte2 |= opc;
11206 
11207   // P1: byte 3 as Wvvvv1pp
11208   int byte3 = ((~nds_enc) & 0xf) << 3;
11209   // p[10] is always 1
11210   byte3 |= EVEX_F;
11211   byte3 |= (vex_w & 1) << 7;
11212   // confine pre opcode extensions in pp bits to lower two bits
11213   // of form {66, F3, F2}
11214   byte3 |= pre;
11215 
11216   // P2: byte 4 as zL'Lbv'aaa
11217   // kregs are implemented in the low 3 bits as aaa
11218   int byte4 = (_attributes->is_no_reg_mask()) ?
11219               0 :
11220               _attributes->get_embedded_opmask_register_specifier();
11221   // EVEX.v` for extending EVEX.vvvv or VIDX
11222   byte4 |= (evex_v ? 0: EVEX_V);
11223   // third EXEC.b for broadcast actions
11224   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11225   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11226   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11227   // last is EVEX.z for zero/merge actions
11228   if (_attributes->is_no_reg_mask() == false &&
11229       _attributes->get_embedded_opmask_register_specifier() != 0) {
11230     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11231   }
11232 
11233   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11234 }
11235 
11236 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11237   bool vex_r = (xreg_enc & 8) == 8;
11238   bool vex_b = adr.base_needs_rex();
11239   bool vex_x;
11240   if (adr.isxmmindex()) {
11241     vex_x = adr.xmmindex_needs_rex();
11242   } else {
11243     vex_x = adr.index_needs_rex();
11244   }
11245   set_attributes(attributes);
11246   attributes->set_current_assembler(this);
11247 
11248   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11249   // is allowed in legacy mode and has resources which will fit in it.
11250   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11251   if (!attributes->is_legacy_mode()) {
11252     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11253       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11254           attributes->set_is_legacy_mode();
11255       }
11256     }
11257   }
11258 
11259   if (UseAVX > 2) {
11260     assert(((!attributes->uses_vl()) ||
11261             (attributes->get_vector_len() == AVX_512bit) ||
11262             (!_legacy_mode_vl) ||
11263             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11264     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11265   }
11266 
11267   clear_managed();
11268   if (UseAVX > 2 && !attributes->is_legacy_mode())
11269   {
11270     bool evex_r = (xreg_enc >= 16);
11271     bool evex_v;
11272     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11273     if (adr.isxmmindex())  {
11274       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11275     } else {
11276       evex_v = (nds_enc >= 16);
11277     }
11278     attributes->set_is_evex_instruction();
11279     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11280   } else {
11281     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11282       attributes->set_rex_vex_w(false);
11283     }
11284     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11285   }
11286 }
11287 
11288 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11289   bool vex_r = (dst_enc & 8) == 8;
11290   bool vex_b = (src_enc & 8) == 8;
11291   bool vex_x = false;
11292   set_attributes(attributes);
11293   attributes->set_current_assembler(this);
11294 
11295   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11296   // is allowed in legacy mode and has resources which will fit in it.
11297   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11298   if (!attributes->is_legacy_mode()) {
11299     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11300       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11301           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11302           attributes->set_is_legacy_mode();
11303       }
11304     }
11305   }
11306 
11307   if (UseAVX > 2) {
11308     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11309     // Instruction with uses_vl true are vector instructions
11310     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11311     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11312     // Rest all should have legacy_mode set as true
11313     assert(((!attributes->uses_vl()) ||
11314             (attributes->get_vector_len() == AVX_512bit) ||
11315             (!_legacy_mode_vl) ||
11316             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11317     // Instruction with legacy_mode true should have dst, nds and src < 15
11318     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11319   }
11320 
11321   clear_managed();
11322   if (UseAVX > 2 && !attributes->is_legacy_mode())
11323   {
11324     bool evex_r = (dst_enc >= 16);
11325     bool evex_v = (nds_enc >= 16);
11326     // can use vex_x as bank extender on rm encoding
11327     vex_x = (src_enc >= 16);
11328     attributes->set_is_evex_instruction();
11329     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11330   } else {
11331     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11332       attributes->set_rex_vex_w(false);
11333     }
11334     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11335   }
11336 
11337   // return modrm byte components for operands
11338   return (((dst_enc & 7) << 3) | (src_enc & 7));
11339 }
11340 
11341 
11342 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11343                             VexOpcode opc, InstructionAttr *attributes) {
11344   if (UseAVX > 0) {
11345     int xreg_enc = xreg->encoding();
11346     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11347     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11348   } else {
11349     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11350     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11351   }
11352 }
11353 
11354 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11355                                       VexOpcode opc, InstructionAttr *attributes) {
11356   int dst_enc = dst->encoding();
11357   int src_enc = src->encoding();
11358   if (UseAVX > 0) {
11359     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11360     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11361   } else {
11362     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11363     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11364   }
11365 }
11366 
11367 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11368   assert(VM_Version::supports_avx(), "");
11369   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11370   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11371   emit_int16(0x5F, (0xC0 | encode));
11372 }
11373 
11374 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11375   assert(VM_Version::supports_avx(), "");
11376   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11377   attributes.set_rex_vex_w_reverted();
11378   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11379   emit_int16(0x5F, (0xC0 | encode));
11380 }
11381 
11382 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11383   assert(VM_Version::supports_avx(), "");
11384   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11385   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11386   emit_int16(0x5D, (0xC0 | encode));
11387 }
11388 
11389 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11390   assert(VM_Version::supports_avx(), "");
11391   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11392   attributes.set_rex_vex_w_reverted();
11393   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11394   emit_int16(0x5D, (0xC0 | encode));
11395 }
11396 
11397 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11398   assert(VM_Version::supports_avx(), "");
11399   assert(vector_len <= AVX_256bit, "");
11400   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11401   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11402   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11403 }
11404 
11405 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11406   assert(VM_Version::supports_avx(), "");
11407   assert(vector_len <= AVX_256bit, "");
11408   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11409   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11410   int src2_enc = src2->encoding();
11411   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11412 }
11413 
11414 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11415   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11416   assert(vector_len <= AVX_256bit, "");
11417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11418   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11419   int src2_enc = src2->encoding();
11420   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11421 }
11422 
11423 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11424   assert(VM_Version::supports_avx2(), "");
11425   assert(vector_len <= AVX_256bit, "");
11426   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11427   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11428   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11429 }
11430 
11431 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11432   assert(VM_Version::supports_avx(), "");
11433   assert(vector_len <= AVX_256bit, "");
11434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11435   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11436   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11437 }
11438 
11439 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11440                         ComparisonPredicateFP comparison, int vector_len) {
11441   assert(VM_Version::supports_evex(), "");
11442   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11443   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11444   attributes.set_is_evex_instruction();
11445   attributes.set_embedded_opmask_register_specifier(mask);
11446   attributes.reset_is_clear_context();
11447   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11448   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11449 }
11450 
11451 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11452                         ComparisonPredicateFP comparison, int vector_len) {
11453   assert(VM_Version::supports_evex(), "");
11454   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11455   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11456   attributes.set_is_evex_instruction();
11457   attributes.set_embedded_opmask_register_specifier(mask);
11458   attributes.reset_is_clear_context();
11459   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11460   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11461 }
11462 
11463 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11464   assert(VM_Version::supports_sse4_1(), "");
11465   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11467   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11468   emit_int16(0x14, (0xC0 | encode));
11469 }
11470 
11471 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11472   assert(VM_Version::supports_sse4_1(), "");
11473   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11474   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11475   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11476   emit_int16(0x15, (0xC0 | encode));
11477 }
11478 
11479 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11480   assert(VM_Version::supports_sse4_1(), "");
11481   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11482   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11483   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11484   emit_int16(0x10, (0xC0 | encode));
11485 }
11486 
11487 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11488   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11489   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11490   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11491   int src2_enc = src2->encoding();
11492   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11493 }
11494 
11495 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11496   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11497   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11498   emit_int24(0x0C, (0xC0 | encode), imm8);
11499 }
11500 
11501 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11502   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11503   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11504   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11505   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11506   emit_int16(0x64, (0xC0 | encode));
11507 }
11508 
11509 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11510   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11511   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11512   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11513   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11514   emit_int16(0x65, (0xC0 | encode));
11515 }
11516 
11517 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11518   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11519   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11520   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11521   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11522   emit_int16(0x66, (0xC0 | encode));
11523 }
11524 
11525 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11526   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11527   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11528   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11529   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11530   emit_int16(0x37, (0xC0 | encode));
11531 }
11532 
11533 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11534                         int comparison, bool is_signed, int vector_len) {
11535   assert(VM_Version::supports_evex(), "");
11536   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11537   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11538   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11539   attributes.set_is_evex_instruction();
11540   attributes.set_embedded_opmask_register_specifier(mask);
11541   attributes.reset_is_clear_context();
11542   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11543   int opcode = is_signed ? 0x1F : 0x1E;
11544   emit_int24(opcode, (0xC0 | encode), comparison);
11545 }
11546 
11547 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11548                         int comparison, bool is_signed, int vector_len) {
11549   assert(VM_Version::supports_evex(), "");
11550   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11551   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11552   InstructionMark im(this);
11553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11554   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11555   attributes.set_is_evex_instruction();
11556   attributes.set_embedded_opmask_register_specifier(mask);
11557   attributes.reset_is_clear_context();
11558   int dst_enc = kdst->encoding();
11559   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11560   int opcode = is_signed ? 0x1F : 0x1E;
11561   emit_int8((unsigned char)opcode);
11562   emit_operand(as_Register(dst_enc), src, 1);
11563   emit_int8((unsigned char)comparison);
11564 }
11565 
11566 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11567                         int comparison, bool is_signed, int vector_len) {
11568   assert(VM_Version::supports_evex(), "");
11569   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11570   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11571   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11572   attributes.set_is_evex_instruction();
11573   attributes.set_embedded_opmask_register_specifier(mask);
11574   attributes.reset_is_clear_context();
11575   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11576   int opcode = is_signed ? 0x1F : 0x1E;
11577   emit_int24(opcode, (0xC0 | encode), comparison);
11578 }
11579 
11580 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11581                         int comparison, bool is_signed, int vector_len) {
11582   assert(VM_Version::supports_evex(), "");
11583   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11584   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11585   InstructionMark im(this);
11586   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11587   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11588   attributes.set_is_evex_instruction();
11589   attributes.set_embedded_opmask_register_specifier(mask);
11590   attributes.reset_is_clear_context();
11591   int dst_enc = kdst->encoding();
11592   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11593   int opcode = is_signed ? 0x1F : 0x1E;
11594   emit_int8((unsigned char)opcode);
11595   emit_operand(as_Register(dst_enc), src, 1);
11596   emit_int8((unsigned char)comparison);
11597 }
11598 
11599 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11600                         int comparison, bool is_signed, int vector_len) {
11601   assert(VM_Version::supports_evex(), "");
11602   assert(VM_Version::supports_avx512bw(), "");
11603   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11604   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11606   attributes.set_is_evex_instruction();
11607   attributes.set_embedded_opmask_register_specifier(mask);
11608   attributes.reset_is_clear_context();
11609   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11610   int opcode = is_signed ? 0x3F : 0x3E;
11611   emit_int24(opcode, (0xC0 | encode), comparison);
11612 }
11613 
11614 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11615                         int comparison, bool is_signed, int vector_len) {
11616   assert(VM_Version::supports_evex(), "");
11617   assert(VM_Version::supports_avx512bw(), "");
11618   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11619   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11620   InstructionMark im(this);
11621   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11622   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11623   attributes.set_is_evex_instruction();
11624   attributes.set_embedded_opmask_register_specifier(mask);
11625   attributes.reset_is_clear_context();
11626   int dst_enc = kdst->encoding();
11627   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11628   int opcode = is_signed ? 0x3F : 0x3E;
11629   emit_int8((unsigned char)opcode);
11630   emit_operand(as_Register(dst_enc), src, 1);
11631   emit_int8((unsigned char)comparison);
11632 }
11633 
11634 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11635                         int comparison, bool is_signed, int vector_len) {
11636   assert(VM_Version::supports_evex(), "");
11637   assert(VM_Version::supports_avx512bw(), "");
11638   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11639   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11640   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11641   attributes.set_is_evex_instruction();
11642   attributes.set_embedded_opmask_register_specifier(mask);
11643   attributes.reset_is_clear_context();
11644   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11645   int opcode = is_signed ? 0x3F : 0x3E;
11646   emit_int24(opcode, (0xC0 | encode), comparison);
11647 }
11648 
11649 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11650                         int comparison, bool is_signed, int vector_len) {
11651   assert(VM_Version::supports_evex(), "");
11652   assert(VM_Version::supports_avx512bw(), "");
11653   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11654   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11655   InstructionMark im(this);
11656   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11657   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11658   attributes.set_is_evex_instruction();
11659   attributes.set_embedded_opmask_register_specifier(mask);
11660   attributes.reset_is_clear_context();
11661   int dst_enc = kdst->encoding();
11662   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11663   int opcode = is_signed ? 0x3F : 0x3E;
11664   emit_int8((unsigned char)opcode);
11665   emit_operand(as_Register(dst_enc), src, 1);
11666   emit_int8((unsigned char)comparison);
11667 }
11668 
11669 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11670   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11672   attributes.set_is_evex_instruction();
11673   attributes.set_embedded_opmask_register_specifier(mask);
11674   if (merge) {
11675     attributes.reset_is_clear_context();
11676   }
11677   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11678   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11679 }
11680 
11681 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11682   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11683   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11684   attributes.set_is_evex_instruction();
11685   attributes.set_embedded_opmask_register_specifier(mask);
11686   if (merge) {
11687     attributes.reset_is_clear_context();
11688   }
11689   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11690   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11691 }
11692 
11693 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11694   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11696   attributes.set_is_evex_instruction();
11697   attributes.set_embedded_opmask_register_specifier(mask);
11698   if (merge) {
11699     attributes.reset_is_clear_context();
11700   }
11701   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11702   emit_int16(0x14, (0xC0 | encode));
11703 }
11704 
11705 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11706   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11707   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11708   attributes.set_is_evex_instruction();
11709   attributes.set_embedded_opmask_register_specifier(mask);
11710   if (merge) {
11711     attributes.reset_is_clear_context();
11712   }
11713   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11714   emit_int16(0x14, (0xC0 | encode));
11715 }
11716 
11717 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11718   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11719   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11720   attributes.set_is_evex_instruction();
11721   attributes.set_embedded_opmask_register_specifier(mask);
11722   if (merge) {
11723     attributes.reset_is_clear_context();
11724   }
11725   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11726   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11727 }
11728 
11729 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11730   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11731   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11732   attributes.set_is_evex_instruction();
11733   attributes.set_embedded_opmask_register_specifier(mask);
11734   if (merge) {
11735     attributes.reset_is_clear_context();
11736   }
11737   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11738   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11739 }
11740 
11741 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11742   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11743   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11744   attributes.set_is_evex_instruction();
11745   attributes.set_embedded_opmask_register_specifier(mask);
11746   if (merge) {
11747     attributes.reset_is_clear_context();
11748   }
11749   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11750   emit_int16(0x15, (0xC0 | encode));
11751 }
11752 
11753 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11754   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11755   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11756   attributes.set_is_evex_instruction();
11757   attributes.set_embedded_opmask_register_specifier(mask);
11758   if (merge) {
11759     attributes.reset_is_clear_context();
11760   }
11761   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11762   emit_int16(0x15, (0xC0 | encode));
11763 }
11764 
11765 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11766   assert(VM_Version::supports_avx(), "");
11767   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11768   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11769   int mask_enc = mask->encoding();
11770   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11771 }
11772 
11773 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11774   assert(VM_Version::supports_evex(), "");
11775   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11776   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11777   attributes.set_is_evex_instruction();
11778   attributes.set_embedded_opmask_register_specifier(mask);
11779   if (merge) {
11780     attributes.reset_is_clear_context();
11781   }
11782   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11783   emit_int16(0x65, (0xC0 | encode));
11784 }
11785 
11786 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11787   assert(VM_Version::supports_evex(), "");
11788   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11789   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11790   attributes.set_is_evex_instruction();
11791   attributes.set_embedded_opmask_register_specifier(mask);
11792   if (merge) {
11793     attributes.reset_is_clear_context();
11794   }
11795   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11796   emit_int16(0x65, (0xC0 | encode));
11797 }
11798 
11799 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11800   assert(VM_Version::supports_evex(), "");
11801   assert(VM_Version::supports_avx512bw(), "");
11802   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11803   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11804   attributes.set_is_evex_instruction();
11805   attributes.set_embedded_opmask_register_specifier(mask);
11806   if (merge) {
11807     attributes.reset_is_clear_context();
11808   }
11809   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11810   emit_int16(0x66, (0xC0 | encode));
11811 }
11812 
11813 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11814   assert(VM_Version::supports_evex(), "");
11815   assert(VM_Version::supports_avx512bw(), "");
11816   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11817   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11818   attributes.set_is_evex_instruction();
11819   attributes.set_embedded_opmask_register_specifier(mask);
11820   if (merge) {
11821     attributes.reset_is_clear_context();
11822   }
11823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11824   emit_int16(0x66, (0xC0 | encode));
11825 }
11826 
11827 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11828   assert(VM_Version::supports_evex(), "");
11829   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11830   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11831   attributes.set_is_evex_instruction();
11832   attributes.set_embedded_opmask_register_specifier(mask);
11833   if (merge) {
11834     attributes.reset_is_clear_context();
11835   }
11836   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11837   emit_int16(0x64, (0xC0 | encode));
11838 }
11839 
11840 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11841   assert(VM_Version::supports_evex(), "");
11842   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11843   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11844   attributes.set_is_evex_instruction();
11845   attributes.set_embedded_opmask_register_specifier(mask);
11846   if (merge) {
11847     attributes.reset_is_clear_context();
11848   }
11849   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11850   emit_int16(0x64, (0xC0 | encode));
11851 }
11852 
11853 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11854   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11855   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11856   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11857   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11858 }
11859 
11860 void Assembler::pextl(Register dst, Register src1, Register src2) {
11861   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11862   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11863   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11864   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11865 }
11866 
11867 void Assembler::pdepl(Register dst, Register src1, Register src2) {
11868   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11869   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11870   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11871   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11872 }
11873 
11874 void Assembler::pextq(Register dst, Register src1, Register src2) {
11875   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11876   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11877   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11878   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11879 }
11880 
11881 void Assembler::pdepq(Register dst, Register src1, Register src2) {
11882   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11883   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11884   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11885   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11886 }
11887 
11888 void Assembler::pextl(Register dst, Register src1, Address src2) {
11889   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11890   InstructionMark im(this);
11891   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11892   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11893   emit_int8((unsigned char)0xF5);
11894   emit_operand(dst, src2, 0);
11895 }
11896 
11897 void Assembler::pdepl(Register dst, Register src1, Address src2) {
11898   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11899   InstructionMark im(this);
11900   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11901   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11902   emit_int8((unsigned char)0xF5);
11903   emit_operand(dst, src2, 0);
11904 }
11905 
11906 void Assembler::pextq(Register dst, Register src1, Address src2) {
11907   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11908   InstructionMark im(this);
11909   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11910   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11911   emit_int8((unsigned char)0xF5);
11912   emit_operand(dst, src2, 0);
11913 }
11914 
11915 void Assembler::pdepq(Register dst, Register src1, Address src2) {
11916   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11917   InstructionMark im(this);
11918   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11919   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11920   emit_int8((unsigned char)0xF5);
11921   emit_operand(dst, src2, 0);
11922 }
11923 
11924 void Assembler::sarxl(Register dst, Register src1, Register src2) {
11925   assert(VM_Version::supports_bmi2(), "");
11926   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11927   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11928   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11929 }
11930 
11931 void Assembler::sarxl(Register dst, Address src1, Register src2) {
11932   assert(VM_Version::supports_bmi2(), "");
11933   InstructionMark im(this);
11934   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11935   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11936   emit_int8((unsigned char)0xF7);
11937   emit_operand(dst, src1, 0);
11938 }
11939 
11940 void Assembler::sarxq(Register dst, Register src1, Register src2) {
11941   assert(VM_Version::supports_bmi2(), "");
11942   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11943   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11944   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11945 }
11946 
11947 void Assembler::sarxq(Register dst, Address src1, Register src2) {
11948   assert(VM_Version::supports_bmi2(), "");
11949   InstructionMark im(this);
11950   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11951   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11952   emit_int8((unsigned char)0xF7);
11953   emit_operand(dst, src1, 0);
11954 }
11955 
11956 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11957   assert(VM_Version::supports_bmi2(), "");
11958   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11959   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11960   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11961 }
11962 
11963 void Assembler::shlxl(Register dst, Address src1, Register src2) {
11964   assert(VM_Version::supports_bmi2(), "");
11965   InstructionMark im(this);
11966   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11967   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11968   emit_int8((unsigned char)0xF7);
11969   emit_operand(dst, src1, 0);
11970 }
11971 
11972 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11973   assert(VM_Version::supports_bmi2(), "");
11974   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11975   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11976   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11977 }
11978 
11979 void Assembler::shlxq(Register dst, Address src1, Register src2) {
11980   assert(VM_Version::supports_bmi2(), "");
11981   InstructionMark im(this);
11982   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11983   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11984   emit_int8((unsigned char)0xF7);
11985   emit_operand(dst, src1, 0);
11986 }
11987 
11988 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11989   assert(VM_Version::supports_bmi2(), "");
11990   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11991   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11992   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11993 }
11994 
11995 void Assembler::shrxl(Register dst, Address src1, Register src2) {
11996   assert(VM_Version::supports_bmi2(), "");
11997   InstructionMark im(this);
11998   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11999   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12000   emit_int8((unsigned char)0xF7);
12001   emit_operand(dst, src1, 0);
12002 }
12003 
12004 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12005   assert(VM_Version::supports_bmi2(), "");
12006   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12007   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12008   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12009 }
12010 
12011 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12012   assert(VM_Version::supports_bmi2(), "");
12013   InstructionMark im(this);
12014   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12015   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12016   emit_int8((unsigned char)0xF7);
12017   emit_operand(dst, src1, 0);
12018 }
12019 
12020 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12021   assert(VM_Version::supports_avx512vldq(), "");
12022   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12023   attributes.set_is_evex_instruction();
12024   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12025   emit_int16(0x39, (0xC0 | encode));
12026 }
12027 
12028 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12029   assert(VM_Version::supports_avx512vldq(), "");
12030   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12031   attributes.set_is_evex_instruction();
12032   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12033   emit_int16(0x39, (0xC0 | encode));
12034 }
12035 
12036 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12037   assert(VM_Version::supports_avx512vlbw(), "");
12038   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12039   attributes.set_is_evex_instruction();
12040   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12041   emit_int16(0x29, (0xC0 | encode));
12042 }
12043 
12044 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12045   assert(VM_Version::supports_avx512vlbw(), "");
12046   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12047   attributes.set_is_evex_instruction();
12048   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12049   emit_int16(0x29, (0xC0 | encode));
12050 }
12051 
12052 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12053   assert(VM_Version::supports_avx512vldq(), "");
12054   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12055   attributes.set_is_evex_instruction();
12056   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12057   emit_int16(0x38, (0xC0 | encode));
12058 }
12059 
12060 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12061   assert(VM_Version::supports_avx512vldq(), "");
12062   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12063   attributes.set_is_evex_instruction();
12064   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12065   emit_int16(0x38, (0xC0 | encode));
12066 }
12067 
12068 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12069   assert(VM_Version::supports_avx512vlbw(), "");
12070   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12071   attributes.set_is_evex_instruction();
12072   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12073   emit_int16(0x28, (0xC0 | encode));
12074 }
12075 
12076 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12077   assert(VM_Version::supports_avx512vlbw(), "");
12078   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12079   attributes.set_is_evex_instruction();
12080   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12081   emit_int16(0x28, (0xC0 | encode));
12082 }
12083 
12084 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12085   assert(VM_Version::supports_avx512_vbmi2(), "");
12086   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12088   attributes.set_embedded_opmask_register_specifier(mask);
12089   attributes.set_is_evex_instruction();
12090   if (merge) {
12091     attributes.reset_is_clear_context();
12092   }
12093   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12094   emit_int16((unsigned char)0x63, (0xC0 | encode));
12095 }
12096 
12097 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12098   assert(VM_Version::supports_avx512_vbmi2(), "");
12099   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12100   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12101   attributes.set_embedded_opmask_register_specifier(mask);
12102   attributes.set_is_evex_instruction();
12103   if (merge) {
12104     attributes.reset_is_clear_context();
12105   }
12106   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12107   emit_int16((unsigned char)0x63, (0xC0 | encode));
12108 }
12109 
12110 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12111   assert(VM_Version::supports_evex(), "");
12112   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12113   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12114   attributes.set_embedded_opmask_register_specifier(mask);
12115   attributes.set_is_evex_instruction();
12116   if (merge) {
12117     attributes.reset_is_clear_context();
12118   }
12119   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12120   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12121 }
12122 
12123 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12124   assert(VM_Version::supports_evex(), "");
12125   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12126   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12127   attributes.set_embedded_opmask_register_specifier(mask);
12128   attributes.set_is_evex_instruction();
12129   if (merge) {
12130     attributes.reset_is_clear_context();
12131   }
12132   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12133   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12134 }
12135 
12136 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12137   assert(VM_Version::supports_evex(), "");
12138   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12140   attributes.set_embedded_opmask_register_specifier(mask);
12141   attributes.set_is_evex_instruction();
12142   if (merge) {
12143     attributes.reset_is_clear_context();
12144   }
12145   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12146   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12147 }
12148 
12149 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12150   assert(VM_Version::supports_evex(), "");
12151   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12152   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12153   attributes.set_embedded_opmask_register_specifier(mask);
12154   attributes.set_is_evex_instruction();
12155   if (merge) {
12156     attributes.reset_is_clear_context();
12157   }
12158   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12159   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12160 }
12161 
12162 #ifndef _LP64
12163 
12164 void Assembler::incl(Register dst) {
12165   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12166   emit_int8(0x40 | dst->encoding());
12167 }
12168 
12169 void Assembler::lea(Register dst, Address src) {
12170   leal(dst, src);
12171 }
12172 
12173 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12174   InstructionMark im(this);
12175   emit_int8((unsigned char)0xC7);
12176   emit_operand(rax, dst, 4);
12177   emit_data((int)imm32, rspec, 0);
12178 }
12179 
12180 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12181   InstructionMark im(this);
12182   int encode = prefix_and_encode(dst->encoding());
12183   emit_int8((0xB8 | encode));
12184   emit_data((int)imm32, rspec, 0);
12185 }
12186 
12187 void Assembler::popa() { // 32bit
12188   emit_int8(0x61);
12189 }
12190 
12191 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12192   InstructionMark im(this);
12193   emit_int8(0x68);
12194   emit_data(imm32, rspec, 0);
12195 }
12196 
12197 void Assembler::pusha() { // 32bit
12198   emit_int8(0x60);
12199 }
12200 
12201 void Assembler::set_byte_if_not_zero(Register dst) {
12202   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12203 }
12204 
12205 #else // LP64
12206 
12207 // 64bit only pieces of the assembler
12208 
12209 void Assembler::set_byte_if_not_zero(Register dst) {
12210   int enc = prefix_and_encode(dst->encoding(), true);
12211   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12212 }
12213 
12214 // This should only be used by 64bit instructions that can use rip-relative
12215 // it cannot be used by instructions that want an immediate value.
12216 
12217 // Determine whether an address is always reachable in rip-relative addressing mode
12218 // when accessed from the code cache.
12219 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12220   switch (reloc_type) {
12221     // This should be rip-relative and easily reachable.
12222     case relocInfo::internal_word_type: {
12223       return true;
12224     }
12225     // This should be rip-relative within the code cache and easily
12226     // reachable until we get huge code caches. (At which point
12227     // IC code is going to have issues).
12228     case relocInfo::virtual_call_type:
12229     case relocInfo::opt_virtual_call_type:
12230     case relocInfo::static_call_type:
12231     case relocInfo::static_stub_type: {
12232       return true;
12233     }
12234     case relocInfo::runtime_call_type:
12235     case relocInfo::external_word_type:
12236     case relocInfo::poll_return_type: // these are really external_word but need special
12237     case relocInfo::poll_type: {      // relocs to identify them
12238       return CodeCache::contains(target);
12239     }
12240     default: {
12241       return false;
12242     }
12243   }
12244 }
12245 
12246 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12247 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12248   if (is_always_reachable(target, reloc_type)) {
12249     return true;
12250   }
12251   switch (reloc_type) {
12252     // None will force a 64bit literal to the code stream. Likely a placeholder
12253     // for something that will be patched later and we need to certain it will
12254     // always be reachable.
12255     case relocInfo::none: {
12256       return false;
12257     }
12258     case relocInfo::runtime_call_type:
12259     case relocInfo::external_word_type:
12260     case relocInfo::poll_return_type: // these are really external_word but need special
12261     case relocInfo::poll_type: {      // relocs to identify them
12262       assert(!CodeCache::contains(target), "always reachable");
12263       if (ForceUnreachable) {
12264         return false; // stress the correction code
12265       }
12266       // For external_word_type/runtime_call_type if it is reachable from where we
12267       // are now (possibly a temp buffer) and where we might end up
12268       // anywhere in the code cache then we are always reachable.
12269       // This would have to change if we ever save/restore shared code to be more pessimistic.
12270       // Code buffer has to be allocated in the code cache, so check against
12271       // code cache boundaries cover that case.
12272       //
12273       // In rip-relative addressing mode, an effective address is formed by adding displacement
12274       // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12275       // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12276       // to account for that.
12277       return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12278              Assembler::is_simm32(target - CodeCache::high_bound());
12279     }
12280     default: {
12281       return false;
12282     }
12283   }
12284 }
12285 
12286 bool Assembler::reachable(AddressLiteral adr) {
12287   assert(CodeCache::contains(pc()), "required");
12288   if (adr.is_lval()) {
12289     return false;
12290   }
12291   return is_reachable(adr.target(), adr.reloc());
12292 }
12293 
12294 bool Assembler::always_reachable(AddressLiteral adr) {
12295   assert(CodeCache::contains(pc()), "required");
12296   if (adr.is_lval()) {
12297     return false;
12298   }
12299   return is_always_reachable(adr.target(), adr.reloc());
12300 }
12301 
12302 void Assembler::emit_data64(jlong data,
12303                             relocInfo::relocType rtype,
12304                             int format) {
12305   if (rtype == relocInfo::none) {
12306     emit_int64(data);
12307   } else {
12308     emit_data64(data, Relocation::spec_simple(rtype), format);
12309   }
12310 }
12311 
12312 void Assembler::emit_data64(jlong data,
12313                             RelocationHolder const& rspec,
12314                             int format) {
12315   assert(imm_operand == 0, "default format must be immediate in this file");
12316   assert(imm_operand == format, "must be immediate");
12317   assert(inst_mark() != NULL, "must be inside InstructionMark");
12318   // Do not use AbstractAssembler::relocate, which is not intended for
12319   // embedded words.  Instead, relocate to the enclosing instruction.
12320   code_section()->relocate(inst_mark(), rspec, format);
12321 #ifdef ASSERT
12322   check_relocation(rspec, format);
12323 #endif
12324   emit_int64(data);
12325 }
12326 
12327 void Assembler::prefix(Register reg) {
12328   if (reg->encoding() >= 8) {
12329     prefix(REX_B);
12330   }
12331 }
12332 
12333 void Assembler::prefix(Register dst, Register src, Prefix p) {
12334   if (src->encoding() >= 8) {
12335     p = (Prefix)(p | REX_B);
12336   }
12337   if (dst->encoding() >= 8) {
12338     p = (Prefix)(p | REX_R);
12339   }
12340   if (p != Prefix_EMPTY) {
12341     // do not generate an empty prefix
12342     prefix(p);
12343   }
12344 }
12345 
12346 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12347   if (adr.base_needs_rex()) {
12348     if (adr.index_needs_rex()) {
12349       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12350     } else {
12351       prefix(REX_B);
12352     }
12353   } else {
12354     if (adr.index_needs_rex()) {
12355       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12356     }
12357   }
12358   if (dst->encoding() >= 8) {
12359     p = (Prefix)(p | REX_R);
12360   }
12361   if (p != Prefix_EMPTY) {
12362     // do not generate an empty prefix
12363     prefix(p);
12364   }
12365 }
12366 
12367 void Assembler::prefix(Address adr) {
12368   if (adr.base_needs_rex()) {
12369     if (adr.index_needs_rex()) {
12370       prefix(REX_XB);
12371     } else {
12372       prefix(REX_B);
12373     }
12374   } else {
12375     if (adr.index_needs_rex()) {
12376       prefix(REX_X);
12377     }
12378   }
12379 }
12380 
12381 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12382   if (reg->encoding() < 8) {
12383     if (adr.base_needs_rex()) {
12384       if (adr.index_needs_rex()) {
12385         prefix(REX_XB);
12386       } else {
12387         prefix(REX_B);
12388       }
12389     } else {
12390       if (adr.index_needs_rex()) {
12391         prefix(REX_X);
12392       } else if (byteinst && reg->encoding() >= 4) {
12393         prefix(REX);
12394       }
12395     }
12396   } else {
12397     if (adr.base_needs_rex()) {
12398       if (adr.index_needs_rex()) {
12399         prefix(REX_RXB);
12400       } else {
12401         prefix(REX_RB);
12402       }
12403     } else {
12404       if (adr.index_needs_rex()) {
12405         prefix(REX_RX);
12406       } else {
12407         prefix(REX_R);
12408       }
12409     }
12410   }
12411 }
12412 
12413 void Assembler::prefix(Address adr, XMMRegister reg) {
12414   if (reg->encoding() < 8) {
12415     if (adr.base_needs_rex()) {
12416       if (adr.index_needs_rex()) {
12417         prefix(REX_XB);
12418       } else {
12419         prefix(REX_B);
12420       }
12421     } else {
12422       if (adr.index_needs_rex()) {
12423         prefix(REX_X);
12424       }
12425     }
12426   } else {
12427     if (adr.base_needs_rex()) {
12428       if (adr.index_needs_rex()) {
12429         prefix(REX_RXB);
12430       } else {
12431         prefix(REX_RB);
12432       }
12433     } else {
12434       if (adr.index_needs_rex()) {
12435         prefix(REX_RX);
12436       } else {
12437         prefix(REX_R);
12438       }
12439     }
12440   }
12441 }
12442 
12443 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12444   if (reg_enc >= 8) {
12445     prefix(REX_B);
12446     reg_enc -= 8;
12447   } else if (byteinst && reg_enc >= 4) {
12448     prefix(REX);
12449   }
12450   return reg_enc;
12451 }
12452 
12453 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12454   if (dst_enc < 8) {
12455     if (src_enc >= 8) {
12456       prefix(REX_B);
12457       src_enc -= 8;
12458     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12459       prefix(REX);
12460     }
12461   } else {
12462     if (src_enc < 8) {
12463       prefix(REX_R);
12464     } else {
12465       prefix(REX_RB);
12466       src_enc -= 8;
12467     }
12468     dst_enc -= 8;
12469   }
12470   return dst_enc << 3 | src_enc;
12471 }
12472 
12473 int8_t Assembler::get_prefixq(Address adr) {
12474   int8_t prfx = get_prefixq(adr, rax);
12475   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12476   return prfx;
12477 }
12478 
12479 int8_t Assembler::get_prefixq(Address adr, Register src) {
12480   int8_t prfx = (int8_t)(REX_W +
12481                          ((int)adr.base_needs_rex()) +
12482                          ((int)adr.index_needs_rex() << 1) +
12483                          ((int)(src->encoding() >= 8) << 2));
12484 #ifdef ASSERT
12485   if (src->encoding() < 8) {
12486     if (adr.base_needs_rex()) {
12487       if (adr.index_needs_rex()) {
12488         assert(prfx == REX_WXB, "must be");
12489       } else {
12490         assert(prfx == REX_WB, "must be");
12491       }
12492     } else {
12493       if (adr.index_needs_rex()) {
12494         assert(prfx == REX_WX, "must be");
12495       } else {
12496         assert(prfx == REX_W, "must be");
12497       }
12498     }
12499   } else {
12500     if (adr.base_needs_rex()) {
12501       if (adr.index_needs_rex()) {
12502         assert(prfx == REX_WRXB, "must be");
12503       } else {
12504         assert(prfx == REX_WRB, "must be");
12505       }
12506     } else {
12507       if (adr.index_needs_rex()) {
12508         assert(prfx == REX_WRX, "must be");
12509       } else {
12510         assert(prfx == REX_WR, "must be");
12511       }
12512     }
12513   }
12514 #endif
12515   return prfx;
12516 }
12517 
12518 void Assembler::prefixq(Address adr) {
12519   emit_int8(get_prefixq(adr));
12520 }
12521 
12522 void Assembler::prefixq(Address adr, Register src) {
12523   emit_int8(get_prefixq(adr, src));
12524 }
12525 
12526 void Assembler::prefixq(Address adr, XMMRegister src) {
12527   if (src->encoding() < 8) {
12528     if (adr.base_needs_rex()) {
12529       if (adr.index_needs_rex()) {
12530         prefix(REX_WXB);
12531       } else {
12532         prefix(REX_WB);
12533       }
12534     } else {
12535       if (adr.index_needs_rex()) {
12536         prefix(REX_WX);
12537       } else {
12538         prefix(REX_W);
12539       }
12540     }
12541   } else {
12542     if (adr.base_needs_rex()) {
12543       if (adr.index_needs_rex()) {
12544         prefix(REX_WRXB);
12545       } else {
12546         prefix(REX_WRB);
12547       }
12548     } else {
12549       if (adr.index_needs_rex()) {
12550         prefix(REX_WRX);
12551       } else {
12552         prefix(REX_WR);
12553       }
12554     }
12555   }
12556 }
12557 
12558 int Assembler::prefixq_and_encode(int reg_enc) {
12559   if (reg_enc < 8) {
12560     prefix(REX_W);
12561   } else {
12562     prefix(REX_WB);
12563     reg_enc -= 8;
12564   }
12565   return reg_enc;
12566 }
12567 
12568 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12569   if (dst_enc < 8) {
12570     if (src_enc < 8) {
12571       prefix(REX_W);
12572     } else {
12573       prefix(REX_WB);
12574       src_enc -= 8;
12575     }
12576   } else {
12577     if (src_enc < 8) {
12578       prefix(REX_WR);
12579     } else {
12580       prefix(REX_WRB);
12581       src_enc -= 8;
12582     }
12583     dst_enc -= 8;
12584   }
12585   return dst_enc << 3 | src_enc;
12586 }
12587 
12588 void Assembler::adcq(Register dst, int32_t imm32) {
12589   (void) prefixq_and_encode(dst->encoding());
12590   emit_arith(0x81, 0xD0, dst, imm32);
12591 }
12592 
12593 void Assembler::adcq(Register dst, Address src) {
12594   InstructionMark im(this);
12595   emit_int16(get_prefixq(src, dst), 0x13);
12596   emit_operand(dst, src, 0);
12597 }
12598 
12599 void Assembler::adcq(Register dst, Register src) {
12600   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12601   emit_arith(0x13, 0xC0, dst, src);
12602 }
12603 
12604 void Assembler::addq(Address dst, int32_t imm32) {
12605   InstructionMark im(this);
12606   prefixq(dst);
12607   emit_arith_operand(0x81, rax, dst, imm32);
12608 }
12609 
12610 void Assembler::addq(Address dst, Register src) {
12611   InstructionMark im(this);
12612   emit_int16(get_prefixq(dst, src), 0x01);
12613   emit_operand(src, dst, 0);
12614 }
12615 
12616 void Assembler::addq(Register dst, int32_t imm32) {
12617   (void) prefixq_and_encode(dst->encoding());
12618   emit_arith(0x81, 0xC0, dst, imm32);
12619 }
12620 
12621 void Assembler::addq(Register dst, Address src) {
12622   InstructionMark im(this);
12623   emit_int16(get_prefixq(src, dst), 0x03);
12624   emit_operand(dst, src, 0);
12625 }
12626 
12627 void Assembler::addq(Register dst, Register src) {
12628   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12629   emit_arith(0x03, 0xC0, dst, src);
12630 }
12631 
12632 void Assembler::adcxq(Register dst, Register src) {
12633   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12634   emit_int8(0x66);
12635   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12636   emit_int32(0x0F,
12637              0x38,
12638              (unsigned char)0xF6,
12639              (0xC0 | encode));
12640 }
12641 
12642 void Assembler::adoxq(Register dst, Register src) {
12643   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12644   emit_int8((unsigned char)0xF3);
12645   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12646   emit_int32(0x0F,
12647              0x38,
12648              (unsigned char)0xF6,
12649              (0xC0 | encode));
12650 }
12651 
12652 void Assembler::andq(Address dst, int32_t imm32) {
12653   InstructionMark im(this);
12654   prefixq(dst);
12655   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12656 }
12657 
12658 void Assembler::andq(Register dst, int32_t imm32) {
12659   (void) prefixq_and_encode(dst->encoding());
12660   emit_arith(0x81, 0xE0, dst, imm32);
12661 }
12662 
12663 void Assembler::andq(Register dst, Address src) {
12664   InstructionMark im(this);
12665   emit_int16(get_prefixq(src, dst), 0x23);
12666   emit_operand(dst, src, 0);
12667 }
12668 
12669 void Assembler::andq(Register dst, Register src) {
12670   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12671   emit_arith(0x23, 0xC0, dst, src);
12672 }
12673 
12674 void Assembler::andq(Address dst, Register src) {
12675   InstructionMark im(this);
12676   emit_int16(get_prefixq(dst, src), 0x21);
12677   emit_operand(src, dst, 0);
12678 }
12679 
12680 void Assembler::andnq(Register dst, Register src1, Register src2) {
12681   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12682   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12683   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12684   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12685 }
12686 
12687 void Assembler::andnq(Register dst, Register src1, Address src2) {
12688   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12689   InstructionMark im(this);
12690   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12691   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12692   emit_int8((unsigned char)0xF2);
12693   emit_operand(dst, src2, 0);
12694 }
12695 
12696 void Assembler::bsfq(Register dst, Register src) {
12697   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12698   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12699 }
12700 
12701 void Assembler::bsrq(Register dst, Register src) {
12702   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12703   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12704 }
12705 
12706 void Assembler::bswapq(Register reg) {
12707   int encode = prefixq_and_encode(reg->encoding());
12708   emit_int16(0x0F, (0xC8 | encode));
12709 }
12710 
12711 void Assembler::blsiq(Register dst, Register src) {
12712   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12713   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12714   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12715   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12716 }
12717 
12718 void Assembler::blsiq(Register dst, Address src) {
12719   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12720   InstructionMark im(this);
12721   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12722   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12723   emit_int8((unsigned char)0xF3);
12724   emit_operand(rbx, src, 0);
12725 }
12726 
12727 void Assembler::blsmskq(Register dst, Register src) {
12728   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12729   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12730   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12731   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12732 }
12733 
12734 void Assembler::blsmskq(Register dst, Address src) {
12735   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12736   InstructionMark im(this);
12737   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12738   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12739   emit_int8((unsigned char)0xF3);
12740   emit_operand(rdx, src, 0);
12741 }
12742 
12743 void Assembler::blsrq(Register dst, Register src) {
12744   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12745   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12746   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12747   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12748 }
12749 
12750 void Assembler::blsrq(Register dst, Address src) {
12751   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12752   InstructionMark im(this);
12753   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12754   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12755   emit_int8((unsigned char)0xF3);
12756   emit_operand(rcx, src, 0);
12757 }
12758 
12759 void Assembler::cdqq() {
12760   emit_int16(REX_W, (unsigned char)0x99);
12761 }
12762 
12763 void Assembler::clflush(Address adr) {
12764   assert(VM_Version::supports_clflush(), "should do");
12765   prefix(adr);
12766   emit_int16(0x0F, (unsigned char)0xAE);
12767   emit_operand(rdi, adr, 0);
12768 }
12769 
12770 void Assembler::clflushopt(Address adr) {
12771   assert(VM_Version::supports_clflushopt(), "should do!");
12772   // adr should be base reg only with no index or offset
12773   assert(adr.index() == noreg, "index should be noreg");
12774   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12775   assert(adr.disp() == 0, "displacement should be 0");
12776   // instruction prefix is 0x66
12777   emit_int8(0x66);
12778   prefix(adr);
12779   // opcode family is 0x0F 0xAE
12780   emit_int16(0x0F, (unsigned char)0xAE);
12781   // extended opcode byte is 7 == rdi
12782   emit_operand(rdi, adr, 0);
12783 }
12784 
12785 void Assembler::clwb(Address adr) {
12786   assert(VM_Version::supports_clwb(), "should do!");
12787   // adr should be base reg only with no index or offset
12788   assert(adr.index() == noreg, "index should be noreg");
12789   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12790   assert(adr.disp() == 0, "displacement should be 0");
12791   // instruction prefix is 0x66
12792   emit_int8(0x66);
12793   prefix(adr);
12794   // opcode family is 0x0f 0xAE
12795   emit_int16(0x0F, (unsigned char)0xAE);
12796   // extended opcode byte is 6 == rsi
12797   emit_operand(rsi, adr, 0);
12798 }
12799 
12800 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12801   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12802   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12803 }
12804 
12805 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12806   InstructionMark im(this);
12807   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12808   emit_operand(dst, src, 0);
12809 }
12810 
12811 void Assembler::cmpq(Address dst, int32_t imm32) {
12812   InstructionMark im(this);
12813   prefixq(dst);
12814   emit_arith_operand(0x81, as_Register(7), dst, imm32);
12815 }
12816 
12817 void Assembler::cmpq(Register dst, int32_t imm32) {
12818   (void) prefixq_and_encode(dst->encoding());
12819   emit_arith(0x81, 0xF8, dst, imm32);
12820 }
12821 
12822 void Assembler::cmpq(Address dst, Register src) {
12823   InstructionMark im(this);
12824   emit_int16(get_prefixq(dst, src), 0x39);
12825   emit_operand(src, dst, 0);
12826 }
12827 
12828 void Assembler::cmpq(Register dst, Register src) {
12829   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12830   emit_arith(0x3B, 0xC0, dst, src);
12831 }
12832 
12833 void Assembler::cmpq(Register dst, Address src) {
12834   InstructionMark im(this);
12835   emit_int16(get_prefixq(src, dst), 0x3B);
12836   emit_operand(dst, src, 0);
12837 }
12838 
12839 void Assembler::cmpxchgq(Register reg, Address adr) {
12840   InstructionMark im(this);
12841   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12842   emit_operand(reg, adr, 0);
12843 }
12844 
12845 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12847   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12848   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12849   emit_int16(0x2A, (0xC0 | encode));
12850 }
12851 
12852 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12853   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12854   InstructionMark im(this);
12855   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12856   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12857   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12858   emit_int8(0x2A);
12859   emit_operand(dst, src, 0);
12860 }
12861 
12862 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12863   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12864   InstructionMark im(this);
12865   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12866   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12867   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12868   emit_int8(0x2A);
12869   emit_operand(dst, src, 0);
12870 }
12871 
12872 void Assembler::cvttsd2siq(Register dst, Address src) {
12873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12874   // F2 REX.W 0F 2C /r
12875   // CVTTSD2SI r64, xmm1/m64
12876   InstructionMark im(this);
12877   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12878   emit_operand(dst, src, 0);
12879 }
12880 
12881 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12883   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12884   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12885   emit_int16(0x2C, (0xC0 | encode));
12886 }
12887 
12888 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12889   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12890   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12891   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12892   emit_int16(0x2D, (0xC0 | encode));
12893 }
12894 
12895 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12896   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12897   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12898   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12899   emit_int16(0x2C, (0xC0 | encode));
12900 }
12901 
12902 void Assembler::decl(Register dst) {
12903   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12904   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12905   int encode = prefix_and_encode(dst->encoding());
12906   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12907 }
12908 
12909 void Assembler::decq(Register dst) {
12910   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12911   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12912   int encode = prefixq_and_encode(dst->encoding());
12913   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12914 }
12915 
12916 void Assembler::decq(Address dst) {
12917   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12918   InstructionMark im(this);
12919   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12920   emit_operand(rcx, dst, 0);
12921 }
12922 
12923 void Assembler::fxrstor(Address src) {
12924   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12925   emit_operand(as_Register(1), src, 0);
12926 }
12927 
12928 void Assembler::xrstor(Address src) {
12929   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12930   emit_operand(as_Register(5), src, 0);
12931 }
12932 
12933 void Assembler::fxsave(Address dst) {
12934   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12935   emit_operand(as_Register(0), dst, 0);
12936 }
12937 
12938 void Assembler::xsave(Address dst) {
12939   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12940   emit_operand(as_Register(4), dst, 0);
12941 }
12942 
12943 void Assembler::idivq(Register src) {
12944   int encode = prefixq_and_encode(src->encoding());
12945   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12946 }
12947 
12948 void Assembler::divq(Register src) {
12949   int encode = prefixq_and_encode(src->encoding());
12950   emit_int16((unsigned char)0xF7, (0xF0 | encode));
12951 }
12952 
12953 void Assembler::imulq(Register dst, Register src) {
12954   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12955   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12956 }
12957 
12958 void Assembler::imulq(Register src) {
12959   int encode = prefixq_and_encode(src->encoding());
12960   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12961 }
12962 
12963 void Assembler::imulq(Register dst, Address src, int32_t value) {
12964   InstructionMark im(this);
12965   prefixq(src, dst);
12966   if (is8bit(value)) {
12967     emit_int8((unsigned char)0x6B);
12968     emit_operand(dst, src, 1);
12969     emit_int8(value);
12970   } else {
12971     emit_int8((unsigned char)0x69);
12972     emit_operand(dst, src, 4);
12973     emit_int32(value);
12974   }
12975 }
12976 
12977 void Assembler::imulq(Register dst, Register src, int value) {
12978   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12979   if (is8bit(value)) {
12980     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12981   } else {
12982     emit_int16(0x69, (0xC0 | encode));
12983     emit_int32(value);
12984   }
12985 }
12986 
12987 void Assembler::imulq(Register dst, Address src) {
12988   InstructionMark im(this);
12989   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12990   emit_operand(dst, src, 0);
12991 }
12992 
12993 void Assembler::incl(Register dst) {
12994   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12995   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12996   int encode = prefix_and_encode(dst->encoding());
12997   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12998 }
12999 
13000 void Assembler::incq(Register dst) {
13001   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13002   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13003   int encode = prefixq_and_encode(dst->encoding());
13004   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13005 }
13006 
13007 void Assembler::incq(Address dst) {
13008   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13009   InstructionMark im(this);
13010   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13011   emit_operand(rax, dst, 0);
13012 }
13013 
13014 void Assembler::lea(Register dst, Address src) {
13015   leaq(dst, src);
13016 }
13017 
13018 void Assembler::leaq(Register dst, Address src) {
13019   InstructionMark im(this);
13020   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13021   emit_operand(dst, src, 0);
13022 }
13023 
13024 void Assembler::mov64(Register dst, int64_t imm64) {
13025   InstructionMark im(this);
13026   int encode = prefixq_and_encode(dst->encoding());
13027   emit_int8(0xB8 | encode);
13028   emit_int64(imm64);
13029 }
13030 
13031 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13032   InstructionMark im(this);
13033   int encode = prefixq_and_encode(dst->encoding());
13034   emit_int8(0xB8 | encode);
13035   emit_data64(imm64, rtype, format);
13036 }
13037 
13038 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13039   InstructionMark im(this);
13040   int encode = prefixq_and_encode(dst->encoding());
13041   emit_int8(0xB8 | encode);
13042   emit_data64(imm64, rspec);
13043 }
13044 
13045 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13046   InstructionMark im(this);
13047   int encode = prefix_and_encode(dst->encoding());
13048   emit_int8(0xB8 | encode);
13049   emit_data((int)imm32, rspec, narrow_oop_operand);
13050 }
13051 
13052 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
13053   InstructionMark im(this);
13054   prefix(dst);
13055   emit_int8((unsigned char)0xC7);
13056   emit_operand(rax, dst, 4);
13057   emit_data((int)imm32, rspec, narrow_oop_operand);
13058 }
13059 
13060 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13061   InstructionMark im(this);
13062   int encode = prefix_and_encode(src1->encoding());
13063   emit_int16((unsigned char)0x81, (0xF8 | encode));
13064   emit_data((int)imm32, rspec, narrow_oop_operand);
13065 }
13066 
13067 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13068   InstructionMark im(this);
13069   prefix(src1);
13070   emit_int8((unsigned char)0x81);
13071   emit_operand(rax, src1, 4);
13072   emit_data((int)imm32, rspec, narrow_oop_operand);
13073 }
13074 
13075 void Assembler::lzcntq(Register dst, Register src) {
13076   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13077   emit_int8((unsigned char)0xF3);
13078   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13079   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13080 }
13081 
13082 void Assembler::lzcntq(Register dst, Address src) {
13083   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13084   InstructionMark im(this);
13085   emit_int8((unsigned char)0xF3);
13086   prefixq(src, dst);
13087   emit_int16(0x0F, (unsigned char)0xBD);
13088   emit_operand(dst, src, 0);
13089 }
13090 
13091 void Assembler::movdq(XMMRegister dst, Register src) {
13092   // table D-1 says MMX/SSE2
13093   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13094   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13095   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13096   emit_int16(0x6E, (0xC0 | encode));
13097 }
13098 
13099 void Assembler::movdq(Register dst, XMMRegister src) {
13100   // table D-1 says MMX/SSE2
13101   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13102   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13103   // swap src/dst to get correct prefix
13104   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13105   emit_int16(0x7E,
13106              (0xC0 | encode));
13107 }
13108 
13109 void Assembler::movq(Register dst, Register src) {
13110   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13111   emit_int16((unsigned char)0x8B,
13112              (0xC0 | encode));
13113 }
13114 
13115 void Assembler::movq(Register dst, Address src) {
13116   InstructionMark im(this);
13117   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13118   emit_operand(dst, src, 0);
13119 }
13120 
13121 void Assembler::movq(Address dst, Register src) {
13122   InstructionMark im(this);
13123   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13124   emit_operand(src, dst, 0);
13125 }
13126 
13127 void Assembler::movq(Address dst, int32_t imm32) {
13128   InstructionMark im(this);
13129   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13130   emit_operand(as_Register(0), dst, 4);
13131   emit_int32(imm32);
13132 }
13133 
13134 void Assembler::movq(Register dst, int32_t imm32) {
13135   int encode = prefixq_and_encode(dst->encoding());
13136   emit_int16((unsigned char)0xC7, (0xC0 | encode));
13137   emit_int32(imm32);
13138 }
13139 
13140 void Assembler::movsbq(Register dst, Address src) {
13141   InstructionMark im(this);
13142   emit_int24(get_prefixq(src, dst),
13143              0x0F,
13144              (unsigned char)0xBE);
13145   emit_operand(dst, src, 0);
13146 }
13147 
13148 void Assembler::movsbq(Register dst, Register src) {
13149   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13150   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13151 }
13152 
13153 void Assembler::movslq(Register dst, int32_t imm32) {
13154   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
13155   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
13156   // as a result we shouldn't use until tested at runtime...
13157   ShouldNotReachHere();
13158   InstructionMark im(this);
13159   int encode = prefixq_and_encode(dst->encoding());
13160   emit_int8(0xC7 | encode);
13161   emit_int32(imm32);
13162 }
13163 
13164 void Assembler::movslq(Address dst, int32_t imm32) {
13165   assert(is_simm32(imm32), "lost bits");
13166   InstructionMark im(this);
13167   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13168   emit_operand(rax, dst, 4);
13169   emit_int32(imm32);
13170 }
13171 
13172 void Assembler::movslq(Register dst, Address src) {
13173   InstructionMark im(this);
13174   emit_int16(get_prefixq(src, dst), 0x63);
13175   emit_operand(dst, src, 0);
13176 }
13177 
13178 void Assembler::movslq(Register dst, Register src) {
13179   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13180   emit_int16(0x63, (0xC0 | encode));
13181 }
13182 
13183 void Assembler::movswq(Register dst, Address src) {
13184   InstructionMark im(this);
13185   emit_int24(get_prefixq(src, dst),
13186              0x0F,
13187              (unsigned char)0xBF);
13188   emit_operand(dst, src, 0);
13189 }
13190 
13191 void Assembler::movswq(Register dst, Register src) {
13192   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13193   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13194 }
13195 
13196 void Assembler::movzbq(Register dst, Address src) {
13197   InstructionMark im(this);
13198   emit_int24(get_prefixq(src, dst),
13199              0x0F,
13200              (unsigned char)0xB6);
13201   emit_operand(dst, src, 0);
13202 }
13203 
13204 void Assembler::movzbq(Register dst, Register src) {
13205   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13206   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13207 }
13208 
13209 void Assembler::movzwq(Register dst, Address src) {
13210   InstructionMark im(this);
13211   emit_int24(get_prefixq(src, dst),
13212              0x0F,
13213              (unsigned char)0xB7);
13214   emit_operand(dst, src, 0);
13215 }
13216 
13217 void Assembler::movzwq(Register dst, Register src) {
13218   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13219   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13220 }
13221 
13222 void Assembler::mulq(Address src) {
13223   InstructionMark im(this);
13224   emit_int16(get_prefixq(src), (unsigned char)0xF7);
13225   emit_operand(rsp, src, 0);
13226 }
13227 
13228 void Assembler::mulq(Register src) {
13229   int encode = prefixq_and_encode(src->encoding());
13230   emit_int16((unsigned char)0xF7, (0xE0 | encode));
13231 }
13232 
13233 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13234   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13235   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13236   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13237   emit_int16((unsigned char)0xF6, (0xC0 | encode));
13238 }
13239 
13240 void Assembler::negq(Register dst) {
13241   int encode = prefixq_and_encode(dst->encoding());
13242   emit_int16((unsigned char)0xF7, (0xD8 | encode));
13243 }
13244 
13245 void Assembler::negq(Address dst) {
13246   InstructionMark im(this);
13247   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13248   emit_operand(as_Register(3), dst, 0);
13249 }
13250 
13251 void Assembler::notq(Register dst) {
13252   int encode = prefixq_and_encode(dst->encoding());
13253   emit_int16((unsigned char)0xF7, (0xD0 | encode));
13254 }
13255 
13256 void Assembler::btsq(Address dst, int imm8) {
13257   assert(isByte(imm8), "not a byte");
13258   InstructionMark im(this);
13259   emit_int24(get_prefixq(dst),
13260              0x0F,
13261              (unsigned char)0xBA);
13262   emit_operand(rbp /* 5 */, dst, 1);
13263   emit_int8(imm8);
13264 }
13265 
13266 void Assembler::btrq(Address dst, int imm8) {
13267   assert(isByte(imm8), "not a byte");
13268   InstructionMark im(this);
13269   emit_int24(get_prefixq(dst),
13270              0x0F,
13271              (unsigned char)0xBA);
13272   emit_operand(rsi /* 6 */, dst, 1);
13273   emit_int8(imm8);
13274 }
13275 
13276 void Assembler::orq(Address dst, int32_t imm32) {
13277   InstructionMark im(this);
13278   prefixq(dst);
13279   emit_arith_operand(0x81, as_Register(1), dst, imm32);
13280 }
13281 
13282 void Assembler::orq(Address dst, Register src) {
13283   InstructionMark im(this);
13284   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13285   emit_operand(src, dst, 0);
13286 }
13287 
13288 void Assembler::orq(Register dst, int32_t imm32) {
13289   (void) prefixq_and_encode(dst->encoding());
13290   emit_arith(0x81, 0xC8, dst, imm32);
13291 }
13292 
13293 void Assembler::orq(Register dst, Address src) {
13294   InstructionMark im(this);
13295   emit_int16(get_prefixq(src, dst), 0x0B);
13296   emit_operand(dst, src, 0);
13297 }
13298 
13299 void Assembler::orq(Register dst, Register src) {
13300   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13301   emit_arith(0x0B, 0xC0, dst, src);
13302 }
13303 
13304 void Assembler::popcntq(Register dst, Address src) {
13305   assert(VM_Version::supports_popcnt(), "must support");
13306   InstructionMark im(this);
13307   emit_int32((unsigned char)0xF3,
13308              get_prefixq(src, dst),
13309              0x0F,
13310              (unsigned char)0xB8);
13311   emit_operand(dst, src, 0);
13312 }
13313 
13314 void Assembler::popcntq(Register dst, Register src) {
13315   assert(VM_Version::supports_popcnt(), "must support");
13316   emit_int8((unsigned char)0xF3);
13317   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13318   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13319 }
13320 
13321 void Assembler::popq(Address dst) {
13322   InstructionMark im(this);
13323   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13324   emit_operand(rax, dst, 0);
13325 }
13326 
13327 void Assembler::popq(Register dst) {
13328   emit_int8((unsigned char)0x58 | dst->encoding());
13329 }
13330 
13331 // Precomputable: popa, pusha, vzeroupper
13332 
13333 // The result of these routines are invariant from one invocation to another
13334 // invocation for the duration of a run. Caching the result on bootstrap
13335 // and copying it out on subsequent invocations can thus be beneficial
13336 static bool     precomputed = false;
13337 
13338 static u_char* popa_code  = NULL;
13339 static int     popa_len   = 0;
13340 
13341 static u_char* pusha_code = NULL;
13342 static int     pusha_len  = 0;
13343 
13344 static u_char* vzup_code  = NULL;
13345 static int     vzup_len   = 0;
13346 
13347 void Assembler::precompute_instructions() {
13348   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13349   guarantee(!precomputed, "only once");
13350   precomputed = true;
13351   ResourceMark rm;
13352 
13353   // Make a temporary buffer big enough for the routines we're capturing
13354   int size = 256;
13355   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13356   CodeBuffer buffer((address)tmp_code, size);
13357   MacroAssembler masm(&buffer);
13358 
13359   address begin_popa  = masm.code_section()->end();
13360   masm.popa_uncached();
13361   address end_popa    = masm.code_section()->end();
13362   masm.pusha_uncached();
13363   address end_pusha   = masm.code_section()->end();
13364   masm.vzeroupper_uncached();
13365   address end_vzup    = masm.code_section()->end();
13366 
13367   // Save the instructions to permanent buffers.
13368   popa_len = (int)(end_popa - begin_popa);
13369   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13370   memcpy(popa_code, begin_popa, popa_len);
13371 
13372   pusha_len = (int)(end_pusha - end_popa);
13373   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13374   memcpy(pusha_code, end_popa, pusha_len);
13375 
13376   vzup_len = (int)(end_vzup - end_pusha);
13377   if (vzup_len > 0) {
13378     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13379     memcpy(vzup_code, end_pusha, vzup_len);
13380   } else {
13381     vzup_code = pusha_code; // dummy
13382   }
13383 
13384   assert(masm.code()->total_oop_size() == 0 &&
13385          masm.code()->total_metadata_size() == 0 &&
13386          masm.code()->total_relocation_size() == 0,
13387          "pre-computed code can't reference oops, metadata or contain relocations");
13388 }
13389 
13390 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13391   assert(src != NULL, "code to copy must have been pre-computed");
13392   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13393   address end = code_section->end();
13394   memcpy(end, src, src_len);
13395   code_section->set_end(end + src_len);
13396 }
13397 
13398 void Assembler::popa() { // 64bit
13399   emit_copy(code_section(), popa_code, popa_len);
13400 }
13401 
13402 void Assembler::popa_uncached() { // 64bit
13403   movq(r15, Address(rsp, 0));
13404   movq(r14, Address(rsp, wordSize));
13405   movq(r13, Address(rsp, 2 * wordSize));
13406   movq(r12, Address(rsp, 3 * wordSize));
13407   movq(r11, Address(rsp, 4 * wordSize));
13408   movq(r10, Address(rsp, 5 * wordSize));
13409   movq(r9,  Address(rsp, 6 * wordSize));
13410   movq(r8,  Address(rsp, 7 * wordSize));
13411   movq(rdi, Address(rsp, 8 * wordSize));
13412   movq(rsi, Address(rsp, 9 * wordSize));
13413   movq(rbp, Address(rsp, 10 * wordSize));
13414   // Skip rsp as it is restored automatically to the value
13415   // before the corresponding pusha when popa is done.
13416   movq(rbx, Address(rsp, 12 * wordSize));
13417   movq(rdx, Address(rsp, 13 * wordSize));
13418   movq(rcx, Address(rsp, 14 * wordSize));
13419   movq(rax, Address(rsp, 15 * wordSize));
13420 
13421   addq(rsp, 16 * wordSize);
13422 }
13423 
13424 // Does not actually store the value of rsp on the stack.
13425 // The slot for rsp just contains an arbitrary value.
13426 void Assembler::pusha() { // 64bit
13427   emit_copy(code_section(), pusha_code, pusha_len);
13428 }
13429 
13430 // Does not actually store the value of rsp on the stack.
13431 // The slot for rsp just contains an arbitrary value.
13432 void Assembler::pusha_uncached() { // 64bit
13433   subq(rsp, 16 * wordSize);
13434 
13435   movq(Address(rsp, 15 * wordSize), rax);
13436   movq(Address(rsp, 14 * wordSize), rcx);
13437   movq(Address(rsp, 13 * wordSize), rdx);
13438   movq(Address(rsp, 12 * wordSize), rbx);
13439   // Skip rsp as the value is normally not used. There are a few places where
13440   // the original value of rsp needs to be known but that can be computed
13441   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13442   movq(Address(rsp, 10 * wordSize), rbp);
13443   movq(Address(rsp, 9 * wordSize), rsi);
13444   movq(Address(rsp, 8 * wordSize), rdi);
13445   movq(Address(rsp, 7 * wordSize), r8);
13446   movq(Address(rsp, 6 * wordSize), r9);
13447   movq(Address(rsp, 5 * wordSize), r10);
13448   movq(Address(rsp, 4 * wordSize), r11);
13449   movq(Address(rsp, 3 * wordSize), r12);
13450   movq(Address(rsp, 2 * wordSize), r13);
13451   movq(Address(rsp, wordSize), r14);
13452   movq(Address(rsp, 0), r15);
13453 }
13454 
13455 void Assembler::vzeroupper() {
13456   emit_copy(code_section(), vzup_code, vzup_len);
13457 }
13458 
13459 void Assembler::pushq(Address src) {
13460   InstructionMark im(this);
13461   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13462   emit_operand(rsi, src, 0);
13463 }
13464 
13465 void Assembler::rclq(Register dst, int imm8) {
13466   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13467   int encode = prefixq_and_encode(dst->encoding());
13468   if (imm8 == 1) {
13469     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13470   } else {
13471     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13472   }
13473 }
13474 
13475 void Assembler::rcrq(Register dst, int imm8) {
13476   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13477   int encode = prefixq_and_encode(dst->encoding());
13478   if (imm8 == 1) {
13479     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13480   } else {
13481     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13482   }
13483 }
13484 
13485 void Assembler::rorxl(Register dst, Register src, int imm8) {
13486   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13487   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13488   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13489   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13490 }
13491 
13492 void Assembler::rorxl(Register dst, Address src, int imm8) {
13493   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13494   InstructionMark im(this);
13495   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13496   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13497   emit_int8((unsigned char)0xF0);
13498   emit_operand(dst, src, 1);
13499   emit_int8(imm8);
13500 }
13501 
13502 void Assembler::rorxq(Register dst, Register src, int imm8) {
13503   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13504   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13505   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13506   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13507 }
13508 
13509 void Assembler::rorxq(Register dst, Address src, int imm8) {
13510   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13511   InstructionMark im(this);
13512   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13513   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13514   emit_int8((unsigned char)0xF0);
13515   emit_operand(dst, src, 1);
13516   emit_int8(imm8);
13517 }
13518 
13519 #ifdef _LP64
13520 void Assembler::salq(Address dst, int imm8) {
13521   InstructionMark im(this);
13522   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13523   if (imm8 == 1) {
13524     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13525     emit_operand(as_Register(4), dst, 0);
13526   }
13527   else {
13528     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13529     emit_operand(as_Register(4), dst, 1);
13530     emit_int8(imm8);
13531   }
13532 }
13533 
13534 void Assembler::salq(Address dst) {
13535   InstructionMark im(this);
13536   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13537   emit_operand(as_Register(4), dst, 0);
13538 }
13539 
13540 void Assembler::salq(Register dst, int imm8) {
13541   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13542   int encode = prefixq_and_encode(dst->encoding());
13543   if (imm8 == 1) {
13544     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13545   } else {
13546     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13547   }
13548 }
13549 
13550 void Assembler::salq(Register dst) {
13551   int encode = prefixq_and_encode(dst->encoding());
13552   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13553 }
13554 
13555 void Assembler::sarq(Address dst, int imm8) {
13556   InstructionMark im(this);
13557   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13558   if (imm8 == 1) {
13559     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13560     emit_operand(as_Register(7), dst, 0);
13561   }
13562   else {
13563     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13564     emit_operand(as_Register(7), dst, 1);
13565     emit_int8(imm8);
13566   }
13567 }
13568 
13569 void Assembler::sarq(Address dst) {
13570   InstructionMark im(this);
13571   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13572   emit_operand(as_Register(7), dst, 0);
13573 }
13574 
13575 void Assembler::sarq(Register dst, int imm8) {
13576   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13577   int encode = prefixq_and_encode(dst->encoding());
13578   if (imm8 == 1) {
13579     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13580   } else {
13581     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13582   }
13583 }
13584 
13585 void Assembler::sarq(Register dst) {
13586   int encode = prefixq_and_encode(dst->encoding());
13587   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13588 }
13589 #endif
13590 
13591 void Assembler::sbbq(Address dst, int32_t imm32) {
13592   InstructionMark im(this);
13593   prefixq(dst);
13594   emit_arith_operand(0x81, rbx, dst, imm32);
13595 }
13596 
13597 void Assembler::sbbq(Register dst, int32_t imm32) {
13598   (void) prefixq_and_encode(dst->encoding());
13599   emit_arith(0x81, 0xD8, dst, imm32);
13600 }
13601 
13602 void Assembler::sbbq(Register dst, Address src) {
13603   InstructionMark im(this);
13604   emit_int16(get_prefixq(src, dst), 0x1B);
13605   emit_operand(dst, src, 0);
13606 }
13607 
13608 void Assembler::sbbq(Register dst, Register src) {
13609   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13610   emit_arith(0x1B, 0xC0, dst, src);
13611 }
13612 
13613 void Assembler::shlq(Register dst, int imm8) {
13614   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13615   int encode = prefixq_and_encode(dst->encoding());
13616   if (imm8 == 1) {
13617     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13618   } else {
13619     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13620   }
13621 }
13622 
13623 void Assembler::shlq(Register dst) {
13624   int encode = prefixq_and_encode(dst->encoding());
13625   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13626 }
13627 
13628 void Assembler::shrq(Register dst, int imm8) {
13629   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13630   int encode = prefixq_and_encode(dst->encoding());
13631   if (imm8 == 1) {
13632     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13633   }
13634   else {
13635     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13636   }
13637 }
13638 
13639 void Assembler::shrq(Register dst) {
13640   int encode = prefixq_and_encode(dst->encoding());
13641   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13642 }
13643 
13644 void Assembler::shrq(Address dst) {
13645   InstructionMark im(this);
13646   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13647   emit_operand(as_Register(5), dst, 0);
13648 }
13649 
13650 void Assembler::shrq(Address dst, int imm8) {
13651   InstructionMark im(this);
13652   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13653   if (imm8 == 1) {
13654     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13655     emit_operand(as_Register(5), dst, 0);
13656   }
13657   else {
13658     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13659     emit_operand(as_Register(5), dst, 1);
13660     emit_int8(imm8);
13661   }
13662 }
13663 
13664 void Assembler::subq(Address dst, int32_t imm32) {
13665   InstructionMark im(this);
13666   prefixq(dst);
13667   emit_arith_operand(0x81, rbp, dst, imm32);
13668 }
13669 
13670 void Assembler::subq(Address dst, Register src) {
13671   InstructionMark im(this);
13672   emit_int16(get_prefixq(dst, src), 0x29);
13673   emit_operand(src, dst, 0);
13674 }
13675 
13676 void Assembler::subq(Register dst, int32_t imm32) {
13677   (void) prefixq_and_encode(dst->encoding());
13678   emit_arith(0x81, 0xE8, dst, imm32);
13679 }
13680 
13681 // Force generation of a 4 byte immediate value even if it fits into 8bit
13682 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13683   (void) prefixq_and_encode(dst->encoding());
13684   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13685 }
13686 
13687 void Assembler::subq(Register dst, Address src) {
13688   InstructionMark im(this);
13689   emit_int16(get_prefixq(src, dst), 0x2B);
13690   emit_operand(dst, src, 0);
13691 }
13692 
13693 void Assembler::subq(Register dst, Register src) {
13694   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13695   emit_arith(0x2B, 0xC0, dst, src);
13696 }
13697 
13698 void Assembler::testq(Address dst, int32_t imm32) {
13699   InstructionMark im(this);
13700   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13701   emit_operand(as_Register(0), dst, 4);
13702   emit_int32(imm32);
13703 }
13704 
13705 void Assembler::testq(Register dst, int32_t imm32) {
13706   // not using emit_arith because test
13707   // doesn't support sign-extension of
13708   // 8bit operands
13709   if (dst == rax) {
13710     prefix(REX_W);
13711     emit_int8((unsigned char)0xA9);
13712     emit_int32(imm32);
13713   } else {
13714     int encode = dst->encoding();
13715     encode = prefixq_and_encode(encode);
13716     emit_int16((unsigned char)0xF7, (0xC0 | encode));
13717     emit_int32(imm32);
13718   }
13719 }
13720 
13721 void Assembler::testq(Register dst, Register src) {
13722   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13723   emit_arith(0x85, 0xC0, dst, src);
13724 }
13725 
13726 void Assembler::testq(Register dst, Address src) {
13727   InstructionMark im(this);
13728   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13729   emit_operand(dst, src, 0);
13730 }
13731 
13732 void Assembler::xaddq(Address dst, Register src) {
13733   InstructionMark im(this);
13734   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13735   emit_operand(src, dst, 0);
13736 }
13737 
13738 void Assembler::xchgq(Register dst, Address src) {
13739   InstructionMark im(this);
13740   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13741   emit_operand(dst, src, 0);
13742 }
13743 
13744 void Assembler::xchgq(Register dst, Register src) {
13745   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13746   emit_int16((unsigned char)0x87, (0xc0 | encode));
13747 }
13748 
13749 void Assembler::xorq(Register dst, Register src) {
13750   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13751   emit_arith(0x33, 0xC0, dst, src);
13752 }
13753 
13754 void Assembler::xorq(Register dst, Address src) {
13755   InstructionMark im(this);
13756   emit_int16(get_prefixq(src, dst), 0x33);
13757   emit_operand(dst, src, 0);
13758 }
13759 
13760 void Assembler::xorq(Register dst, int32_t imm32) {
13761   (void) prefixq_and_encode(dst->encoding());
13762   emit_arith(0x81, 0xF0, dst, imm32);
13763 }
13764 
13765 void Assembler::xorq(Address dst, int32_t imm32) {
13766   InstructionMark im(this);
13767   prefixq(dst);
13768   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13769 }
13770 
13771 void Assembler::xorq(Address dst, Register src) {
13772   InstructionMark im(this);
13773   emit_int16(get_prefixq(dst, src), 0x31);
13774   emit_operand(src, dst, 0);
13775 }
13776 
13777 #endif // !LP64
13778 
13779 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13780   if (VM_Version::supports_evex()) {
13781     _tuple_type = tuple_type;
13782     _input_size_in_bits = input_size_in_bits;
13783   }
13784 }