1 /*
    2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec = RelocationHolder::none;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert(op1 == 0x81, "Unexpected opcode");
  304   if (is8bit(imm32)) {
  305     emit_int24(op1 | 0x02,        // set sign bit
  306                op2 | encode(dst),
  307                imm32 & 0xFF);
  308   } else if (dst == rax) {
  309     switch (op2) {
  310       case 0xD0: emit_int8(0x15); break; // adc
  311       case 0xC0: emit_int8(0x05); break; // add
  312       case 0xE0: emit_int8(0x25); break; // and
  313       case 0xF8: emit_int8(0x3D); break; // cmp
  314       case 0xC8: emit_int8(0x0D); break; // or
  315       case 0xD8: emit_int8(0x1D); break; // sbb
  316       case 0xE8: emit_int8(0x2D); break; // sub
  317       case 0xF0: emit_int8(0x35); break; // xor
  318       default: ShouldNotReachHere();
  319     }
  320     emit_int32(imm32);
  321   } else {
  322     emit_int16(op1, (op2 | encode(dst)));
  323     emit_int32(imm32);
  324   }
  325 }
  326 
  327 // Force generation of a 4 byte immediate value even if it fits into 8bit
  328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  329   assert(isByte(op1) && isByte(op2), "wrong opcode");
  330   assert((op1 & 0x01) == 1, "should be 32bit operation");
  331   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  332   emit_int16(op1, (op2 | encode(dst)));
  333   emit_int32(imm32);
  334 }
  335 
  336 // immediate-to-memory forms
  337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  338   assert((op1 & 0x01) == 1, "should be 32bit operation");
  339   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  340   if (is8bit(imm32)) {
  341     emit_int8(op1 | 0x02); // set sign bit
  342     emit_operand(rm, adr, 1);
  343     emit_int8(imm32 & 0xFF);
  344   } else {
  345     emit_int8(op1);
  346     emit_operand(rm, adr, 4);
  347     emit_int32(imm32);
  348   }
  349 }
  350 
  351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
  352   assert(op1 == 0x81, "unexpected opcode");
  353   emit_int8(op1);
  354   emit_operand(rm, adr, 4);
  355   emit_int32(imm32);
  356 }
  357 
  358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  359   assert(isByte(op1) && isByte(op2), "wrong opcode");
  360   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  361 }
  362 
  363 
  364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  365                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  366   int mod_idx = 0;
  367   // We will test if the displacement fits the compressed format and if so
  368   // apply the compression to the displacement iff the result is8bit.
  369   if (VM_Version::supports_evex() && is_evex_inst) {
  370     switch (cur_tuple_type) {
  371     case EVEX_FV:
  372       if ((cur_encoding & VEX_W) == VEX_W) {
  373         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  374       } else {
  375         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  376       }
  377       break;
  378 
  379     case EVEX_HV:
  380       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  381       break;
  382 
  383     case EVEX_FVM:
  384       break;
  385 
  386     case EVEX_T1S:
  387       switch (in_size_in_bits) {
  388       case EVEX_8bit:
  389         break;
  390 
  391       case EVEX_16bit:
  392         mod_idx = 1;
  393         break;
  394 
  395       case EVEX_32bit:
  396         mod_idx = 2;
  397         break;
  398 
  399       case EVEX_64bit:
  400         mod_idx = 3;
  401         break;
  402       }
  403       break;
  404 
  405     case EVEX_T1F:
  406     case EVEX_T2:
  407     case EVEX_T4:
  408       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  409       break;
  410 
  411     case EVEX_T8:
  412       break;
  413 
  414     case EVEX_HVM:
  415       break;
  416 
  417     case EVEX_QVM:
  418       break;
  419 
  420     case EVEX_OVM:
  421       break;
  422 
  423     case EVEX_M128:
  424       break;
  425 
  426     case EVEX_DUP:
  427       break;
  428 
  429     default:
  430       assert(0, "no valid evex tuple_table entry");
  431       break;
  432     }
  433 
  434     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  435       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  436       if ((disp % disp_factor) == 0) {
  437         int new_disp = disp / disp_factor;
  438         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  439           disp = new_disp;
  440         }
  441       } else {
  442         return false;
  443       }
  444     }
  445   }
  446   return (-0x80 <= disp && disp < 0x80);
  447 }
  448 
  449 
  450 bool Assembler::emit_compressed_disp_byte(int &disp) {
  451   int mod_idx = 0;
  452   // We will test if the displacement fits the compressed format and if so
  453   // apply the compression to the displacement iff the result is8bit.
  454   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  455     int evex_encoding = _attributes->get_evex_encoding();
  456     int tuple_type = _attributes->get_tuple_type();
  457     switch (tuple_type) {
  458     case EVEX_FV:
  459       if ((evex_encoding & VEX_W) == VEX_W) {
  460         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  461       } else {
  462         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  463       }
  464       break;
  465 
  466     case EVEX_HV:
  467       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  468       break;
  469 
  470     case EVEX_FVM:
  471       break;
  472 
  473     case EVEX_T1S:
  474       switch (_attributes->get_input_size()) {
  475       case EVEX_8bit:
  476         break;
  477 
  478       case EVEX_16bit:
  479         mod_idx = 1;
  480         break;
  481 
  482       case EVEX_32bit:
  483         mod_idx = 2;
  484         break;
  485 
  486       case EVEX_64bit:
  487         mod_idx = 3;
  488         break;
  489       }
  490       break;
  491 
  492     case EVEX_T1F:
  493     case EVEX_T2:
  494     case EVEX_T4:
  495       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  496       break;
  497 
  498     case EVEX_T8:
  499       break;
  500 
  501     case EVEX_HVM:
  502       break;
  503 
  504     case EVEX_QVM:
  505       break;
  506 
  507     case EVEX_OVM:
  508       break;
  509 
  510     case EVEX_M128:
  511       break;
  512 
  513     case EVEX_DUP:
  514       break;
  515 
  516     default:
  517       assert(0, "no valid evex tuple_table entry");
  518       break;
  519     }
  520 
  521     int vector_len = _attributes->get_vector_len();
  522     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  523       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  524       if ((disp % disp_factor) == 0) {
  525         int new_disp = disp / disp_factor;
  526         if (is8bit(new_disp)) {
  527           disp = new_disp;
  528         }
  529       } else {
  530         return false;
  531       }
  532     }
  533   }
  534   return is8bit(disp);
  535 }
  536 
  537 static bool is_valid_encoding(int reg_enc) {
  538   return reg_enc >= 0;
  539 }
  540 
  541 static int raw_encode(Register reg) {
  542   assert(reg == noreg || reg->is_valid(), "sanity");
  543   int reg_enc = reg->raw_encoding();
  544   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  545   return reg_enc;
  546 }
  547 
  548 static int raw_encode(XMMRegister xmmreg) {
  549   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  550   int xmmreg_enc = xmmreg->raw_encoding();
  551   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  552   return xmmreg_enc;
  553 }
  554 
  555 static int raw_encode(KRegister kreg) {
  556   assert(kreg == knoreg || kreg->is_valid(), "sanity");
  557   int kreg_enc = kreg->raw_encoding();
  558   assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
  559   return kreg_enc;
  560 }
  561 
  562 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  563   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  564 }
  565 
  566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  567   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  568 }
  569 
  570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  571   assert((mod & 3) != 0b11, "forbidden");
  572   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  573   emit_int8(modrm);
  574 }
  575 
  576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  577                                         int disp) {
  578   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  579   emit_int16(modrm, disp & 0xFF);
  580 }
  581 
  582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  583                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  584   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  585   int sib = sib_encoding(scale, index_enc, base_enc);
  586   emit_int16(modrm, sib);
  587 }
  588 
  589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  590                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  591                                             int disp) {
  592   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  593   int sib = sib_encoding(scale, index_enc, base_enc);
  594   emit_int24(modrm, sib, disp & 0xFF);
  595 }
  596 
  597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  598                                     Address::ScaleFactor scale, int disp,
  599                                     RelocationHolder const& rspec,
  600                                     int post_addr_length) {
  601   bool no_relocation = (rspec.type() == relocInfo::none);
  602 
  603   if (is_valid_encoding(base_enc)) {
  604     if (is_valid_encoding(index_enc)) {
  605       assert(scale != Address::no_scale, "inconsistent address");
  606       // [base + index*scale + disp]
  607       if (disp == 0 && no_relocation &&
  608           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  609         // [base + index*scale]
  610         // [00 reg 100][ss index base]
  611         emit_modrm_sib(0b00, reg_enc, 0b100,
  612                        scale, index_enc, base_enc);
  613       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  614         // [base + index*scale + imm8]
  615         // [01 reg 100][ss index base] imm8
  616         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  617                              scale, index_enc, base_enc,
  618                              disp);
  619       } else {
  620         // [base + index*scale + disp32]
  621         // [10 reg 100][ss index base] disp32
  622         emit_modrm_sib(0b10, reg_enc, 0b100,
  623                        scale, index_enc, base_enc);
  624         emit_data(disp, rspec, disp32_operand);
  625       }
  626     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  627       // [rsp + disp]
  628       if (disp == 0 && no_relocation) {
  629         // [rsp]
  630         // [00 reg 100][00 100 100]
  631         emit_modrm_sib(0b00, reg_enc, 0b100,
  632                        Address::times_1, 0b100, 0b100);
  633       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  634         // [rsp + imm8]
  635         // [01 reg 100][00 100 100] disp8
  636         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  637                              Address::times_1, 0b100, 0b100,
  638                              disp);
  639       } else {
  640         // [rsp + imm32]
  641         // [10 reg 100][00 100 100] disp32
  642         emit_modrm_sib(0b10, reg_enc, 0b100,
  643                        Address::times_1, 0b100, 0b100);
  644         emit_data(disp, rspec, disp32_operand);
  645       }
  646     } else {
  647       // [base + disp]
  648       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  649       if (disp == 0 && no_relocation &&
  650           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  651         // [base]
  652         // [00 reg base]
  653         emit_modrm(0, reg_enc, base_enc);
  654       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  655         // [base + disp8]
  656         // [01 reg base] disp8
  657         emit_modrm_disp8(0b01, reg_enc, base_enc,
  658                          disp);
  659       } else {
  660         // [base + disp32]
  661         // [10 reg base] disp32
  662         emit_modrm(0b10, reg_enc, base_enc);
  663         emit_data(disp, rspec, disp32_operand);
  664       }
  665     }
  666   } else {
  667     if (is_valid_encoding(index_enc)) {
  668       assert(scale != Address::no_scale, "inconsistent address");
  669       // base == noreg
  670       // [index*scale + disp]
  671       // [00 reg 100][ss index 101] disp32
  672       emit_modrm_sib(0b00, reg_enc, 0b100,
  673                      scale, index_enc, 0b101 /* no base */);
  674       emit_data(disp, rspec, disp32_operand);
  675     } else if (!no_relocation) {
  676       // base == noreg, index == noreg
  677       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  678       // [00 reg 101] disp32
  679 
  680       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  681       // Note that the RIP-rel. correction applies to the generated
  682       // disp field, but _not_ to the target address in the rspec.
  683 
  684       // disp was created by converting the target address minus the pc
  685       // at the start of the instruction. That needs more correction here.
  686       // intptr_t disp = target - next_ip;
  687       assert(inst_mark() != NULL, "must be inside InstructionMark");
  688       address next_ip = pc() + sizeof(int32_t) + post_addr_length;
  689       int64_t adjusted = disp;
  690       // Do rip-rel adjustment for 64bit
  691       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  692       assert(is_simm32(adjusted),
  693              "must be 32bit offset (RIP relative address)");
  694       emit_data((int32_t) adjusted, rspec, disp32_operand);
  695 
  696     } else {
  697       // base == noreg, index == noreg, no_relocation == true
  698       // 32bit never did this, did everything as the rip-rel/disp code above
  699       // [disp] ABSOLUTE
  700       // [00 reg 100][00 100 101] disp32
  701       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  702                      Address::times_1, 0b100, 0b101);
  703       emit_data(disp, rspec, disp32_operand);
  704     }
  705   }
  706 }
  707 
  708 void Assembler::emit_operand(Register reg, Register base, Register index,
  709                              Address::ScaleFactor scale, int disp,
  710                              RelocationHolder const& rspec,
  711                              int post_addr_length) {
  712   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  713   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  714                       scale, disp, rspec, post_addr_length);
  715 
  716 }
  717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  718                              Address::ScaleFactor scale, int disp,
  719                              RelocationHolder const& rspec,
  720                              int post_addr_length) {
  721   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  722   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  723   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  724                       scale, disp, rspec, post_addr_length);
  725 }
  726 
  727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  728                              Address::ScaleFactor scale, int disp,
  729                              RelocationHolder const& rspec,
  730                              int post_addr_length) {
  731   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  732   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  733   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  734                       scale, disp, rspec, post_addr_length);
  735 }
  736 
  737 void Assembler::emit_operand(KRegister kreg, Address adr,
  738                              int post_addr_length) {
  739   emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
  740                adr._rspec,
  741                post_addr_length);
  742 }
  743 
  744 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
  745                              Address::ScaleFactor scale, int disp,
  746                              RelocationHolder const& rspec,
  747                              int post_addr_length) {
  748   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  749   emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
  750                       scale, disp, rspec, post_addr_length);
  751 }
  752 
  753 // Secret local extension to Assembler::WhichOperand:
  754 #define end_pc_operand (_WhichOperand_limit)
  755 
  756 address Assembler::locate_operand(address inst, WhichOperand which) {
  757   // Decode the given instruction, and return the address of
  758   // an embedded 32-bit operand word.
  759 
  760   // If "which" is disp32_operand, selects the displacement portion
  761   // of an effective address specifier.
  762   // If "which" is imm64_operand, selects the trailing immediate constant.
  763   // If "which" is call32_operand, selects the displacement of a call or jump.
  764   // Caller is responsible for ensuring that there is such an operand,
  765   // and that it is 32/64 bits wide.
  766 
  767   // If "which" is end_pc_operand, find the end of the instruction.
  768 
  769   address ip = inst;
  770   bool is_64bit = false;
  771 
  772   debug_only(bool has_disp32 = false);
  773   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  774 
  775   again_after_prefix:
  776   switch (0xFF & *ip++) {
  777 
  778   // These convenience macros generate groups of "case" labels for the switch.
  779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  781              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  782 #define REP16(x) REP8((x)+0): \
  783               case REP8((x)+8)
  784 
  785   case CS_segment:
  786   case SS_segment:
  787   case DS_segment:
  788   case ES_segment:
  789   case FS_segment:
  790   case GS_segment:
  791     // Seems dubious
  792     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  793     assert(ip == inst+1, "only one prefix allowed");
  794     goto again_after_prefix;
  795 
  796   case 0x67:
  797   case REX:
  798   case REX_B:
  799   case REX_X:
  800   case REX_XB:
  801   case REX_R:
  802   case REX_RB:
  803   case REX_RX:
  804   case REX_RXB:
  805     NOT_LP64(assert(false, "64bit prefixes"));
  806     goto again_after_prefix;
  807 
  808   case REX_W:
  809   case REX_WB:
  810   case REX_WX:
  811   case REX_WXB:
  812   case REX_WR:
  813   case REX_WRB:
  814   case REX_WRX:
  815   case REX_WRXB:
  816     NOT_LP64(assert(false, "64bit prefixes"));
  817     is_64bit = true;
  818     goto again_after_prefix;
  819 
  820   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  821   case 0x88: // movb a, r
  822   case 0x89: // movl a, r
  823   case 0x8A: // movb r, a
  824   case 0x8B: // movl r, a
  825   case 0x8F: // popl a
  826     debug_only(has_disp32 = true);
  827     break;
  828 
  829   case 0x68: // pushq #32
  830     if (which == end_pc_operand) {
  831       return ip + 4;
  832     }
  833     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  834     return ip;                  // not produced by emit_operand
  835 
  836   case 0x66: // movw ... (size prefix)
  837     again_after_size_prefix2:
  838     switch (0xFF & *ip++) {
  839     case REX:
  840     case REX_B:
  841     case REX_X:
  842     case REX_XB:
  843     case REX_R:
  844     case REX_RB:
  845     case REX_RX:
  846     case REX_RXB:
  847     case REX_W:
  848     case REX_WB:
  849     case REX_WX:
  850     case REX_WXB:
  851     case REX_WR:
  852     case REX_WRB:
  853     case REX_WRX:
  854     case REX_WRXB:
  855       NOT_LP64(assert(false, "64bit prefix found"));
  856       goto again_after_size_prefix2;
  857     case 0x8B: // movw r, a
  858     case 0x89: // movw a, r
  859       debug_only(has_disp32 = true);
  860       break;
  861     case 0xC7: // movw a, #16
  862       debug_only(has_disp32 = true);
  863       tail_size = 2;  // the imm16
  864       break;
  865     case 0x0F: // several SSE/SSE2 variants
  866       ip--;    // reparse the 0x0F
  867       goto again_after_prefix;
  868     default:
  869       ShouldNotReachHere();
  870     }
  871     break;
  872 
  873   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  874     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  875     // these asserts are somewhat nonsensical
  876 #ifndef _LP64
  877     assert(which == imm_operand || which == disp32_operand,
  878            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  879 #else
  880     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  881            which == narrow_oop_operand && !is_64bit,
  882            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  883 #endif // _LP64
  884     return ip;
  885 
  886   case 0x69: // imul r, a, #32
  887   case 0xC7: // movl a, #32(oop?)
  888     tail_size = 4;
  889     debug_only(has_disp32 = true); // has both kinds of operands!
  890     break;
  891 
  892   case 0x0F: // movx..., etc.
  893     switch (0xFF & *ip++) {
  894     case 0x3A: // pcmpestri
  895       tail_size = 1;
  896     case 0x38: // ptest, pmovzxbw
  897       ip++; // skip opcode
  898       debug_only(has_disp32 = true); // has both kinds of operands!
  899       break;
  900 
  901     case 0x70: // pshufd r, r/a, #8
  902       debug_only(has_disp32 = true); // has both kinds of operands!
  903     case 0x73: // psrldq r, #8
  904       tail_size = 1;
  905       break;
  906 
  907     case 0x10: // movups
  908     case 0x11: // movups
  909     case 0x12: // movlps
  910     case 0x28: // movaps
  911     case 0x2E: // ucomiss
  912     case 0x2F: // comiss
  913     case 0x54: // andps
  914     case 0x55: // andnps
  915     case 0x56: // orps
  916     case 0x57: // xorps
  917     case 0x58: // addpd
  918     case 0x59: // mulpd
  919     case 0x6E: // movd
  920     case 0x7E: // movd
  921     case 0x6F: // movdq
  922     case 0x7F: // movdq
  923     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  924     case 0xFE: // paddd
  925       debug_only(has_disp32 = true);
  926       break;
  927 
  928     case 0xAD: // shrd r, a, %cl
  929     case 0xAF: // imul r, a
  930     case 0xBE: // movsbl r, a (movsxb)
  931     case 0xBF: // movswl r, a (movsxw)
  932     case 0xB6: // movzbl r, a (movzxb)
  933     case 0xB7: // movzwl r, a (movzxw)
  934     case REP16(0x40): // cmovl cc, r, a
  935     case 0xB0: // cmpxchgb
  936     case 0xB1: // cmpxchg
  937     case 0xC1: // xaddl
  938     case 0xC7: // cmpxchg8
  939     case REP16(0x90): // setcc a
  940       debug_only(has_disp32 = true);
  941       // fall out of the switch to decode the address
  942       break;
  943 
  944     case 0xC4: // pinsrw r, a, #8
  945       debug_only(has_disp32 = true);
  946     case 0xC5: // pextrw r, r, #8
  947       tail_size = 1;  // the imm8
  948       break;
  949 
  950     case 0xAC: // shrd r, a, #8
  951       debug_only(has_disp32 = true);
  952       tail_size = 1;  // the imm8
  953       break;
  954 
  955     case REP16(0x80): // jcc rdisp32
  956       if (which == end_pc_operand)  return ip + 4;
  957       assert(which == call32_operand, "jcc has no disp32 or imm");
  958       return ip;
  959     default:
  960       ShouldNotReachHere();
  961     }
  962     break;
  963 
  964   case 0x81: // addl a, #32; addl r, #32
  965     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  966     // on 32bit in the case of cmpl, the imm might be an oop
  967     tail_size = 4;
  968     debug_only(has_disp32 = true); // has both kinds of operands!
  969     break;
  970 
  971   case 0x83: // addl a, #8; addl r, #8
  972     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  973     debug_only(has_disp32 = true); // has both kinds of operands!
  974     tail_size = 1;
  975     break;
  976 
  977   case 0x15: // adc rax, #32
  978   case 0x05: // add rax, #32
  979   case 0x25: // and rax, #32
  980   case 0x3D: // cmp rax, #32
  981   case 0x0D: // or  rax, #32
  982   case 0x1D: // sbb rax, #32
  983   case 0x2D: // sub rax, #32
  984   case 0x35: // xor rax, #32
  985     return which == end_pc_operand ? ip + 4 : ip;
  986 
  987   case 0x9B:
  988     switch (0xFF & *ip++) {
  989     case 0xD9: // fnstcw a
  990       debug_only(has_disp32 = true);
  991       break;
  992     default:
  993       ShouldNotReachHere();
  994     }
  995     break;
  996 
  997   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  998   case REP4(0x10): // adc...
  999   case REP4(0x20): // and...
 1000   case REP4(0x30): // xor...
 1001   case REP4(0x08): // or...
 1002   case REP4(0x18): // sbb...
 1003   case REP4(0x28): // sub...
 1004   case 0xF7: // mull a
 1005   case 0x8D: // lea r, a
 1006   case 0x87: // xchg r, a
 1007   case REP4(0x38): // cmp...
 1008   case 0x85: // test r, a
 1009     debug_only(has_disp32 = true); // has both kinds of operands!
 1010     break;
 1011 
 1012   case 0xA8: // testb rax, #8
 1013     return which == end_pc_operand ? ip + 1 : ip;
 1014   case 0xA9: // testl/testq rax, #32
 1015     return which == end_pc_operand ? ip + 4 : ip;
 1016 
 1017   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 1018   case 0xC6: // movb a, #8
 1019   case 0x80: // cmpb a, #8
 1020   case 0x6B: // imul r, a, #8
 1021     debug_only(has_disp32 = true); // has both kinds of operands!
 1022     tail_size = 1; // the imm8
 1023     break;
 1024 
 1025   case 0xC4: // VEX_3bytes
 1026   case 0xC5: // VEX_2bytes
 1027     assert((UseAVX > 0), "shouldn't have VEX prefix");
 1028     assert(ip == inst+1, "no prefixes allowed");
 1029     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 1030     // but they have prefix 0x0F and processed when 0x0F processed above.
 1031     //
 1032     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1033     // instructions (these instructions are not supported in 64-bit mode).
 1034     // To distinguish them bits [7:6] are set in the VEX second byte since
 1035     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1036     // those VEX bits REX and vvvv bits are inverted.
 1037     //
 1038     // Fortunately C2 doesn't generate these instructions so we don't need
 1039     // to check for them in product version.
 1040 
 1041     // Check second byte
 1042     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1043 
 1044     int vex_opcode;
 1045     // First byte
 1046     if ((0xFF & *inst) == VEX_3bytes) {
 1047       vex_opcode = VEX_OPCODE_MASK & *ip;
 1048       ip++; // third byte
 1049       is_64bit = ((VEX_W & *ip) == VEX_W);
 1050     } else {
 1051       vex_opcode = VEX_OPCODE_0F;
 1052     }
 1053     ip++; // opcode
 1054     // To find the end of instruction (which == end_pc_operand).
 1055     switch (vex_opcode) {
 1056       case VEX_OPCODE_0F:
 1057         switch (0xFF & *ip) {
 1058         case 0x70: // pshufd r, r/a, #8
 1059         case 0x71: // ps[rl|ra|ll]w r, #8
 1060         case 0x72: // ps[rl|ra|ll]d r, #8
 1061         case 0x73: // ps[rl|ra|ll]q r, #8
 1062         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1063         case 0xC4: // pinsrw r, r, r/a, #8
 1064         case 0xC5: // pextrw r/a, r, #8
 1065         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1066           tail_size = 1;  // the imm8
 1067           break;
 1068         }
 1069         break;
 1070       case VEX_OPCODE_0F_3A:
 1071         tail_size = 1;
 1072         break;
 1073     }
 1074     ip++; // skip opcode
 1075     debug_only(has_disp32 = true); // has both kinds of operands!
 1076     break;
 1077 
 1078   case 0x62: // EVEX_4bytes
 1079     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1080     assert(ip == inst+1, "no prefixes allowed");
 1081     // no EVEX collisions, all instructions that have 0x62 opcodes
 1082     // have EVEX versions and are subopcodes of 0x66
 1083     ip++; // skip P0 and examine W in P1
 1084     is_64bit = ((VEX_W & *ip) == VEX_W);
 1085     ip++; // move to P2
 1086     ip++; // skip P2, move to opcode
 1087     // To find the end of instruction (which == end_pc_operand).
 1088     switch (0xFF & *ip) {
 1089     case 0x22: // pinsrd r, r/a, #8
 1090     case 0x61: // pcmpestri r, r/a, #8
 1091     case 0x70: // pshufd r, r/a, #8
 1092     case 0x73: // psrldq r, #8
 1093     case 0x1f: // evpcmpd/evpcmpq
 1094     case 0x3f: // evpcmpb/evpcmpw
 1095       tail_size = 1;  // the imm8
 1096       break;
 1097     default:
 1098       break;
 1099     }
 1100     ip++; // skip opcode
 1101     debug_only(has_disp32 = true); // has both kinds of operands!
 1102     break;
 1103 
 1104   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1105   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1106   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1107   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1108   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1109   case 0xDF: // fild_d a; fistp_d a
 1110   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1111   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1112   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1113     debug_only(has_disp32 = true);
 1114     break;
 1115 
 1116   case 0xE8: // call rdisp32
 1117   case 0xE9: // jmp  rdisp32
 1118     if (which == end_pc_operand)  return ip + 4;
 1119     assert(which == call32_operand, "call has no disp32 or imm");
 1120     return ip;
 1121 
 1122   case 0xF0:                    // Lock
 1123     goto again_after_prefix;
 1124 
 1125   case 0xF3:                    // For SSE
 1126   case 0xF2:                    // For SSE2
 1127     switch (0xFF & *ip++) {
 1128     case REX:
 1129     case REX_B:
 1130     case REX_X:
 1131     case REX_XB:
 1132     case REX_R:
 1133     case REX_RB:
 1134     case REX_RX:
 1135     case REX_RXB:
 1136     case REX_W:
 1137     case REX_WB:
 1138     case REX_WX:
 1139     case REX_WXB:
 1140     case REX_WR:
 1141     case REX_WRB:
 1142     case REX_WRX:
 1143     case REX_WRXB:
 1144       NOT_LP64(assert(false, "found 64bit prefix"));
 1145       ip++;
 1146     default:
 1147       ip++;
 1148     }
 1149     debug_only(has_disp32 = true); // has both kinds of operands!
 1150     break;
 1151 
 1152   default:
 1153     ShouldNotReachHere();
 1154 
 1155 #undef REP8
 1156 #undef REP16
 1157   }
 1158 
 1159   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1160 #ifdef _LP64
 1161   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1162 #else
 1163   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1164   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1165 #endif // LP64
 1166   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1167 
 1168   // parse the output of emit_operand
 1169   int op2 = 0xFF & *ip++;
 1170   int base = op2 & 0x07;
 1171   int op3 = -1;
 1172   const int b100 = 4;
 1173   const int b101 = 5;
 1174   if (base == b100 && (op2 >> 6) != 3) {
 1175     op3 = 0xFF & *ip++;
 1176     base = op3 & 0x07;   // refetch the base
 1177   }
 1178   // now ip points at the disp (if any)
 1179 
 1180   switch (op2 >> 6) {
 1181   case 0:
 1182     // [00 reg  100][ss index base]
 1183     // [00 reg  100][00   100  esp]
 1184     // [00 reg base]
 1185     // [00 reg  100][ss index  101][disp32]
 1186     // [00 reg  101]               [disp32]
 1187 
 1188     if (base == b101) {
 1189       if (which == disp32_operand)
 1190         return ip;              // caller wants the disp32
 1191       ip += 4;                  // skip the disp32
 1192     }
 1193     break;
 1194 
 1195   case 1:
 1196     // [01 reg  100][ss index base][disp8]
 1197     // [01 reg  100][00   100  esp][disp8]
 1198     // [01 reg base]               [disp8]
 1199     ip += 1;                    // skip the disp8
 1200     break;
 1201 
 1202   case 2:
 1203     // [10 reg  100][ss index base][disp32]
 1204     // [10 reg  100][00   100  esp][disp32]
 1205     // [10 reg base]               [disp32]
 1206     if (which == disp32_operand)
 1207       return ip;                // caller wants the disp32
 1208     ip += 4;                    // skip the disp32
 1209     break;
 1210 
 1211   case 3:
 1212     // [11 reg base]  (not a memory addressing mode)
 1213     break;
 1214   }
 1215 
 1216   if (which == end_pc_operand) {
 1217     return ip + tail_size;
 1218   }
 1219 
 1220 #ifdef _LP64
 1221   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1222 #else
 1223   assert(which == imm_operand, "instruction has only an imm field");
 1224 #endif // LP64
 1225   return ip;
 1226 }
 1227 
 1228 address Assembler::locate_next_instruction(address inst) {
 1229   // Secretly share code with locate_operand:
 1230   return locate_operand(inst, end_pc_operand);
 1231 }
 1232 
 1233 
 1234 #ifdef ASSERT
 1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1236   address inst = inst_mark();
 1237   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1238   address opnd;
 1239 
 1240   Relocation* r = rspec.reloc();
 1241   if (r->type() == relocInfo::none) {
 1242     return;
 1243   } else if (r->is_call() || format == call32_operand) {
 1244     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1245     opnd = locate_operand(inst, call32_operand);
 1246   } else if (r->is_data()) {
 1247     assert(format == imm_operand || format == disp32_operand
 1248            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1249     opnd = locate_operand(inst, (WhichOperand)format);
 1250   } else {
 1251     assert(format == imm_operand, "cannot specify a format");
 1252     return;
 1253   }
 1254   assert(opnd == pc(), "must put operand where relocs can find it");
 1255 }
 1256 #endif // ASSERT
 1257 
 1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
 1259   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1260 }
 1261 
 1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
 1263   if (adr.isxmmindex()) {
 1264      emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1265   } else {
 1266      emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1267   }
 1268 }
 1269 
 1270 // Now the Assembler instructions (identical for 32/64 bits)
 1271 
 1272 void Assembler::adcl(Address dst, int32_t imm32) {
 1273   InstructionMark im(this);
 1274   prefix(dst);
 1275   emit_arith_operand(0x81, rdx, dst, imm32);
 1276 }
 1277 
 1278 void Assembler::adcl(Address dst, Register src) {
 1279   InstructionMark im(this);
 1280   prefix(dst, src);
 1281   emit_int8(0x11);
 1282   emit_operand(src, dst, 0);
 1283 }
 1284 
 1285 void Assembler::adcl(Register dst, int32_t imm32) {
 1286   prefix(dst);
 1287   emit_arith(0x81, 0xD0, dst, imm32);
 1288 }
 1289 
 1290 void Assembler::adcl(Register dst, Address src) {
 1291   InstructionMark im(this);
 1292   prefix(src, dst);
 1293   emit_int8(0x13);
 1294   emit_operand(dst, src, 0);
 1295 }
 1296 
 1297 void Assembler::adcl(Register dst, Register src) {
 1298   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1299   emit_arith(0x13, 0xC0, dst, src);
 1300 }
 1301 
 1302 void Assembler::addl(Address dst, int32_t imm32) {
 1303   InstructionMark im(this);
 1304   prefix(dst);
 1305   emit_arith_operand(0x81, rax, dst, imm32);
 1306 }
 1307 
 1308 void Assembler::addb(Address dst, int imm8) {
 1309   InstructionMark im(this);
 1310   prefix(dst);
 1311   emit_int8((unsigned char)0x80);
 1312   emit_operand(rax, dst, 1);
 1313   emit_int8(imm8);
 1314 }
 1315 
 1316 void Assembler::addw(Register dst, Register src) {
 1317   emit_int8(0x66);
 1318   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1319   emit_arith(0x03, 0xC0, dst, src);
 1320 }
 1321 
 1322 void Assembler::addw(Address dst, int imm16) {
 1323   InstructionMark im(this);
 1324   emit_int8(0x66);
 1325   prefix(dst);
 1326   emit_int8((unsigned char)0x81);
 1327   emit_operand(rax, dst, 2);
 1328   emit_int16(imm16);
 1329 }
 1330 
 1331 void Assembler::addl(Address dst, Register src) {
 1332   InstructionMark im(this);
 1333   prefix(dst, src);
 1334   emit_int8(0x01);
 1335   emit_operand(src, dst, 0);
 1336 }
 1337 
 1338 void Assembler::addl(Register dst, int32_t imm32) {
 1339   prefix(dst);
 1340   emit_arith(0x81, 0xC0, dst, imm32);
 1341 }
 1342 
 1343 void Assembler::addl(Register dst, Address src) {
 1344   InstructionMark im(this);
 1345   prefix(src, dst);
 1346   emit_int8(0x03);
 1347   emit_operand(dst, src, 0);
 1348 }
 1349 
 1350 void Assembler::addl(Register dst, Register src) {
 1351   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1352   emit_arith(0x03, 0xC0, dst, src);
 1353 }
 1354 
 1355 void Assembler::addr_nop_4() {
 1356   assert(UseAddressNop, "no CPU support");
 1357   // 4 bytes: NOP DWORD PTR [EAX+0]
 1358   emit_int32(0x0F,
 1359              0x1F,
 1360              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1361              0);   // 8-bits offset (1 byte)
 1362 }
 1363 
 1364 void Assembler::addr_nop_5() {
 1365   assert(UseAddressNop, "no CPU support");
 1366   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1367   emit_int32(0x0F,
 1368              0x1F,
 1369              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1370              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1371   emit_int8(0);     // 8-bits offset (1 byte)
 1372 }
 1373 
 1374 void Assembler::addr_nop_7() {
 1375   assert(UseAddressNop, "no CPU support");
 1376   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1377   emit_int24(0x0F,
 1378              0x1F,
 1379              (unsigned char)0x80);
 1380                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1381   emit_int32(0);   // 32-bits offset (4 bytes)
 1382 }
 1383 
 1384 void Assembler::addr_nop_8() {
 1385   assert(UseAddressNop, "no CPU support");
 1386   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1387   emit_int32(0x0F,
 1388              0x1F,
 1389              (unsigned char)0x84,
 1390                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1391              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1392   emit_int32(0);    // 32-bits offset (4 bytes)
 1393 }
 1394 
 1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1397   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1398   attributes.set_rex_vex_w_reverted();
 1399   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1400   emit_int16(0x58, (0xC0 | encode));
 1401 }
 1402 
 1403 void Assembler::addsd(XMMRegister dst, Address src) {
 1404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1405   InstructionMark im(this);
 1406   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1407   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1408   attributes.set_rex_vex_w_reverted();
 1409   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1410   emit_int8(0x58);
 1411   emit_operand(dst, src, 0);
 1412 }
 1413 
 1414 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1415   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1416   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1417   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1418   emit_int16(0x58, (0xC0 | encode));
 1419 }
 1420 
 1421 void Assembler::addss(XMMRegister dst, Address src) {
 1422   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1423   InstructionMark im(this);
 1424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1425   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1426   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1427   emit_int8(0x58);
 1428   emit_operand(dst, src, 0);
 1429 }
 1430 
 1431 void Assembler::aesdec(XMMRegister dst, Address src) {
 1432   assert(VM_Version::supports_aes(), "");
 1433   InstructionMark im(this);
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int8((unsigned char)0xDE);
 1437   emit_operand(dst, src, 0);
 1438 }
 1439 
 1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1441   assert(VM_Version::supports_aes(), "");
 1442   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1443   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1448   assert(VM_Version::supports_avx512_vaes(), "");
 1449   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1450   attributes.set_is_evex_instruction();
 1451   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1453 }
 1454 
 1455 
 1456 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionMark im(this);
 1459   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1460   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1461   emit_int8((unsigned char)0xDF);
 1462   emit_operand(dst, src, 0);
 1463 }
 1464 
 1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1466   assert(VM_Version::supports_aes(), "");
 1467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1468   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1469   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1470 }
 1471 
 1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1473   assert(VM_Version::supports_avx512_vaes(), "");
 1474   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1475   attributes.set_is_evex_instruction();
 1476   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1477   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1478 }
 1479 
 1480 void Assembler::aesenc(XMMRegister dst, Address src) {
 1481   assert(VM_Version::supports_aes(), "");
 1482   InstructionMark im(this);
 1483   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1484   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1485   emit_int8((unsigned char)0xDC);
 1486   emit_operand(dst, src, 0);
 1487 }
 1488 
 1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1490   assert(VM_Version::supports_aes(), "");
 1491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1492   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1493   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1494 }
 1495 
 1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1497   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1499   attributes.set_is_evex_instruction();
 1500   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1501   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1502 }
 1503 
 1504 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1505   assert(VM_Version::supports_aes(), "");
 1506   InstructionMark im(this);
 1507   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1508   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1509   emit_int8((unsigned char)0xDD);
 1510   emit_operand(dst, src, 0);
 1511 }
 1512 
 1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1514   assert(VM_Version::supports_aes(), "");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1521   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1522   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1523   attributes.set_is_evex_instruction();
 1524   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1526 }
 1527 
 1528 void Assembler::andb(Address dst, Register src) {
 1529   InstructionMark im(this);
 1530   prefix(dst, src, true);
 1531   emit_int8(0x20);
 1532   emit_operand(src, dst, 0);
 1533 }
 1534 
 1535 void Assembler::andw(Register dst, Register src) {
 1536   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1537   emit_arith(0x23, 0xC0, dst, src);
 1538 }
 1539 
 1540 void Assembler::andl(Address dst, int32_t imm32) {
 1541   InstructionMark im(this);
 1542   prefix(dst);
 1543   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1544 }
 1545 
 1546 void Assembler::andl(Register dst, int32_t imm32) {
 1547   prefix(dst);
 1548   emit_arith(0x81, 0xE0, dst, imm32);
 1549 }
 1550 
 1551 void Assembler::andl(Address dst, Register src) {
 1552   InstructionMark im(this);
 1553   prefix(dst, src);
 1554   emit_int8(0x21);
 1555   emit_operand(src, dst, 0);
 1556 }
 1557 
 1558 void Assembler::andl(Register dst, Address src) {
 1559   InstructionMark im(this);
 1560   prefix(src, dst);
 1561   emit_int8(0x23);
 1562   emit_operand(dst, src, 0);
 1563 }
 1564 
 1565 void Assembler::andl(Register dst, Register src) {
 1566   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1567   emit_arith(0x23, 0xC0, dst, src);
 1568 }
 1569 
 1570 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1571   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1573   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1574   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1575 }
 1576 
 1577 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1578   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1579   InstructionMark im(this);
 1580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1581   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1582   emit_int8((unsigned char)0xF2);
 1583   emit_operand(dst, src2, 0);
 1584 }
 1585 
 1586 void Assembler::bsfl(Register dst, Register src) {
 1587   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1588   emit_int24(0x0F,
 1589              (unsigned char)0xBC,
 1590              0xC0 | encode);
 1591 }
 1592 
 1593 void Assembler::bsrl(Register dst, Register src) {
 1594   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1595   emit_int24(0x0F,
 1596              (unsigned char)0xBD,
 1597              0xC0 | encode);
 1598 }
 1599 
 1600 void Assembler::bswapl(Register reg) { // bswap
 1601   int encode = prefix_and_encode(reg->encoding());
 1602   emit_int16(0x0F, (0xC8 | encode));
 1603 }
 1604 
 1605 void Assembler::blsil(Register dst, Register src) {
 1606   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1607   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1608   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1609   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1610 }
 1611 
 1612 void Assembler::blsil(Register dst, Address src) {
 1613   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1614   InstructionMark im(this);
 1615   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1616   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1617   emit_int8((unsigned char)0xF3);
 1618   emit_operand(rbx, src, 0);
 1619 }
 1620 
 1621 void Assembler::blsmskl(Register dst, Register src) {
 1622   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1623   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1624   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1625   emit_int16((unsigned char)0xF3,
 1626              0xC0 | encode);
 1627 }
 1628 
 1629 void Assembler::blsmskl(Register dst, Address src) {
 1630   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1631   InstructionMark im(this);
 1632   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1633   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1634   emit_int8((unsigned char)0xF3);
 1635   emit_operand(rdx, src, 0);
 1636 }
 1637 
 1638 void Assembler::blsrl(Register dst, Register src) {
 1639   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1640   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1641   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1642   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1643 }
 1644 
 1645 void Assembler::blsrl(Register dst, Address src) {
 1646   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1647   InstructionMark im(this);
 1648   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1649   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1650   emit_int8((unsigned char)0xF3);
 1651   emit_operand(rcx, src, 0);
 1652 }
 1653 
 1654 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1655   // suspect disp32 is always good
 1656   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1657 
 1658   if (L.is_bound()) {
 1659     const int long_size = 5;
 1660     int offs = (int)( target(L) - pc() );
 1661     assert(offs <= 0, "assembler error");
 1662     InstructionMark im(this);
 1663     // 1110 1000 #32-bit disp
 1664     emit_int8((unsigned char)0xE8);
 1665     emit_data(offs - long_size, rtype, operand);
 1666   } else {
 1667     InstructionMark im(this);
 1668     // 1110 1000 #32-bit disp
 1669     L.add_patch_at(code(), locator());
 1670 
 1671     emit_int8((unsigned char)0xE8);
 1672     emit_data(int(0), rtype, operand);
 1673   }
 1674 }
 1675 
 1676 void Assembler::call(Register dst) {
 1677   int encode = prefix_and_encode(dst->encoding());
 1678   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1679 }
 1680 
 1681 
 1682 void Assembler::call(Address adr) {
 1683   InstructionMark im(this);
 1684   prefix(adr);
 1685   emit_int8((unsigned char)0xFF);
 1686   emit_operand(rdx, adr, 0);
 1687 }
 1688 
 1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1690   InstructionMark im(this);
 1691   emit_int8((unsigned char)0xE8);
 1692   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1693   // Entry is NULL in case of a scratch emit.
 1694   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1695   // Technically, should use call32_operand, but this format is
 1696   // implied by the fact that we're emitting a call instruction.
 1697 
 1698   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1699   emit_data((int) disp, rspec, operand);
 1700 }
 1701 
 1702 void Assembler::cdql() {
 1703   emit_int8((unsigned char)0x99);
 1704 }
 1705 
 1706 void Assembler::cld() {
 1707   emit_int8((unsigned char)0xFC);
 1708 }
 1709 
 1710 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1711   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1712   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1713   emit_int24(0x0F,
 1714              0x40 | cc,
 1715              0xC0 | encode);
 1716 }
 1717 
 1718 
 1719 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1720   InstructionMark im(this);
 1721   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1722   prefix(src, dst);
 1723   emit_int16(0x0F, (0x40 | cc));
 1724   emit_operand(dst, src, 0);
 1725 }
 1726 
 1727 void Assembler::cmpb(Address dst, int imm8) {
 1728   InstructionMark im(this);
 1729   prefix(dst);
 1730   emit_int8((unsigned char)0x80);
 1731   emit_operand(rdi, dst, 1);
 1732   emit_int8(imm8);
 1733 }
 1734 
 1735 void Assembler::cmpl(Address dst, int32_t imm32) {
 1736   InstructionMark im(this);
 1737   prefix(dst);
 1738   emit_arith_operand(0x81, as_Register(7), dst, imm32);
 1739 }
 1740 
 1741 void Assembler::cmpl(Register dst, int32_t imm32) {
 1742   prefix(dst);
 1743   emit_arith(0x81, 0xF8, dst, imm32);
 1744 }
 1745 
 1746 void Assembler::cmpl(Register dst, Register src) {
 1747   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1748   emit_arith(0x3B, 0xC0, dst, src);
 1749 }
 1750 
 1751 void Assembler::cmpl(Register dst, Address src) {
 1752   InstructionMark im(this);
 1753   prefix(src, dst);
 1754   emit_int8(0x3B);
 1755   emit_operand(dst, src, 0);
 1756 }
 1757 
 1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
 1759   InstructionMark im(this);
 1760   prefix(dst);
 1761   emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
 1762 }
 1763 
 1764 void Assembler::cmpw(Address dst, int imm16) {
 1765   InstructionMark im(this);
 1766   emit_int8(0x66);
 1767   prefix(dst);
 1768   emit_int8((unsigned char)0x81);
 1769   emit_operand(rdi, dst, 2);
 1770   emit_int16(imm16);
 1771 }
 1772 
 1773 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1774 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1775 // The ZF is set if the compared values were equal, and cleared otherwise.
 1776 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1777   InstructionMark im(this);
 1778   prefix(adr, reg);
 1779   emit_int16(0x0F, (unsigned char)0xB1);
 1780   emit_operand(reg, adr, 0);
 1781 }
 1782 
 1783 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1784   InstructionMark im(this);
 1785   size_prefix();
 1786   prefix(adr, reg);
 1787   emit_int16(0x0F, (unsigned char)0xB1);
 1788   emit_operand(reg, adr, 0);
 1789 }
 1790 
 1791 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1792 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1793 // The ZF is set if the compared values were equal, and cleared otherwise.
 1794 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1795   InstructionMark im(this);
 1796   prefix(adr, reg, true);
 1797   emit_int16(0x0F, (unsigned char)0xB0);
 1798   emit_operand(reg, adr, 0);
 1799 }
 1800 
 1801 void Assembler::comisd(XMMRegister dst, Address src) {
 1802   // NOTE: dbx seems to decode this as comiss even though the
 1803   // 0x66 is there. Strangely ucomisd comes out correct
 1804   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1805   InstructionMark im(this);
 1806   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1807   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1808   attributes.set_rex_vex_w_reverted();
 1809   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1810   emit_int8(0x2F);
 1811   emit_operand(dst, src, 0);
 1812 }
 1813 
 1814 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1815   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1816   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1817   attributes.set_rex_vex_w_reverted();
 1818   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1819   emit_int16(0x2F, (0xC0 | encode));
 1820 }
 1821 
 1822 void Assembler::comiss(XMMRegister dst, Address src) {
 1823   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1824   InstructionMark im(this);
 1825   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1826   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1827   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1828   emit_int8(0x2F);
 1829   emit_operand(dst, src, 0);
 1830 }
 1831 
 1832 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1833   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1834   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1835   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1836   emit_int16(0x2F, (0xC0 | encode));
 1837 }
 1838 
 1839 void Assembler::cpuid() {
 1840   emit_int16(0x0F, (unsigned char)0xA2);
 1841 }
 1842 
 1843 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1844 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1845 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1846 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1847 //
 1848 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1849 //
 1850 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1851 //
 1852 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1853 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1854   assert(VM_Version::supports_sse4_2(), "");
 1855   int8_t w = 0x01;
 1856   Prefix p = Prefix_EMPTY;
 1857 
 1858   emit_int8((unsigned char)0xF2);
 1859   switch (sizeInBytes) {
 1860   case 1:
 1861     w = 0;
 1862     break;
 1863   case 2:
 1864   case 4:
 1865     break;
 1866   LP64_ONLY(case 8:)
 1867     // This instruction is not valid in 32 bits
 1868     // Note:
 1869     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1870     //
 1871     // Page B - 72   Vol. 2C says
 1872     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1873     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1874     //                                                                            F0!!!
 1875     // while 3 - 208 Vol. 2A
 1876     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1877     //
 1878     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1879     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1880     p = REX_W;
 1881     break;
 1882   default:
 1883     assert(0, "Unsupported value for a sizeInBytes argument");
 1884     break;
 1885   }
 1886   LP64_ONLY(prefix(crc, v, p);)
 1887   emit_int32(0x0F,
 1888              0x38,
 1889              0xF0 | w,
 1890              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1891 }
 1892 
 1893 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1894   assert(VM_Version::supports_sse4_2(), "");
 1895   InstructionMark im(this);
 1896   int8_t w = 0x01;
 1897   Prefix p = Prefix_EMPTY;
 1898 
 1899   emit_int8((int8_t)0xF2);
 1900   switch (sizeInBytes) {
 1901   case 1:
 1902     w = 0;
 1903     break;
 1904   case 2:
 1905   case 4:
 1906     break;
 1907   LP64_ONLY(case 8:)
 1908     // This instruction is not valid in 32 bits
 1909     p = REX_W;
 1910     break;
 1911   default:
 1912     assert(0, "Unsupported value for a sizeInBytes argument");
 1913     break;
 1914   }
 1915   LP64_ONLY(prefix(crc, adr, p);)
 1916   emit_int24(0x0F, 0x38, (0xF0 | w));
 1917   emit_operand(crc, adr, 0);
 1918 }
 1919 
 1920 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1921   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1922   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1923   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1924   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1925 }
 1926 
 1927 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1928   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1929   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1930   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1931   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1932 }
 1933 
 1934 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 1935   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1936   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1937   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1938   emit_int24(0x1D, (0xC0 | encode), imm8);
 1939 }
 1940 
 1941 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
 1942   assert(VM_Version::supports_evex(), "");
 1943   InstructionMark im(this);
 1944   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 1945   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
 1946   attributes.reset_is_clear_context();
 1947   attributes.set_embedded_opmask_register_specifier(mask);
 1948   attributes.set_is_evex_instruction();
 1949   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1950   emit_int8(0x1D);
 1951   emit_operand(src, dst, 1);
 1952   emit_int8(imm8);
 1953 }
 1954 
 1955 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) {
 1956   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1957   InstructionMark im(this);
 1958   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1959   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 1960   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1961   emit_int8(0x1D);
 1962   emit_operand(src, dst, 1);
 1963   emit_int8(imm8);
 1964 }
 1965 
 1966 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1967   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1968   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
 1969   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1970   emit_int16(0x13, (0xC0 | encode));
 1971 }
 1972 
 1973 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) {
 1974   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1975   InstructionMark im(this);
 1976   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 1977   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 1978   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1979   emit_int8(0x13);
 1980   emit_operand(dst, src, 0);
 1981 }
 1982 
 1983 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1984   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1985   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1986   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1987   emit_int16(0x5B, (0xC0 | encode));
 1988 }
 1989 
 1990 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1991   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1992   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1993   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1994   emit_int16(0x5B, (0xC0 | encode));
 1995 }
 1996 
 1997 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1998   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1999   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2000   attributes.set_rex_vex_w_reverted();
 2001   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2002   emit_int16(0x5A, (0xC0 | encode));
 2003 }
 2004 
 2005 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 2006   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2007   InstructionMark im(this);
 2008   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2009   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2010   attributes.set_rex_vex_w_reverted();
 2011   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2012   emit_int8(0x5A);
 2013   emit_operand(dst, src, 0);
 2014 }
 2015 
 2016 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 2017   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2018   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2019   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2020   emit_int16(0x2A, (0xC0 | encode));
 2021 }
 2022 
 2023 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 2024   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2025   InstructionMark im(this);
 2026   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2027   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2028   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2029   emit_int8(0x2A);
 2030   emit_operand(dst, src, 0);
 2031 }
 2032 
 2033 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 2034   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2035   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2036   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2037   emit_int16(0x2A, (0xC0 | encode));
 2038 }
 2039 
 2040 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 2041   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2042   InstructionMark im(this);
 2043   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2044   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2045   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2046   emit_int8(0x2A);
 2047   emit_operand(dst, src, 0);
 2048 }
 2049 
 2050 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 2051   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2052   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2053   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2054   emit_int16(0x2A, (0xC0 | encode));
 2055 }
 2056 
 2057 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 2058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2060   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2061   emit_int16(0x5A, (0xC0 | encode));
 2062 }
 2063 
 2064 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 2065   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2066   InstructionMark im(this);
 2067   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2068   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2069   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2070   emit_int8(0x5A);
 2071   emit_operand(dst, src, 0);
 2072 }
 2073 
 2074 
 2075 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 2076   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2077   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2078   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2079   emit_int16(0x2C, (0xC0 | encode));
 2080 }
 2081 
 2082 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 2083   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2084   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2085   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2086   emit_int16(0x2D, (0xC0 | encode));
 2087 }
 2088 
 2089 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2090   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2091   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2092   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2093   emit_int16(0x2C, (0xC0 | encode));
 2094 }
 2095 
 2096 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2098   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2099   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2100   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2101   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2102 }
 2103 
 2104 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2105   assert(VM_Version::supports_ssse3(), "");
 2106   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2107   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2108   emit_int16(0x1C, (0xC0 | encode));
 2109 }
 2110 
 2111 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2112   assert(VM_Version::supports_ssse3(), "");
 2113   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2114   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2115   emit_int16(0x1D, (0xC0 | encode));
 2116 }
 2117 
 2118 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2119   assert(VM_Version::supports_ssse3(), "");
 2120   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2121   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2122   emit_int16(0x1E, (0xC0 | encode));
 2123 }
 2124 
 2125 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2126   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2127          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2128          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2129   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2130   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2131   emit_int16(0x1C, (0xC0 | encode));
 2132 }
 2133 
 2134 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2135   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2136          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2137          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2138   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2139   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2140   emit_int16(0x1D, (0xC0 | encode));
 2141 }
 2142 
 2143 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2144   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2145   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2146   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2147   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2149   emit_int16(0x1E, (0xC0 | encode));
 2150 }
 2151 
 2152 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2153   assert(UseAVX > 2, "");
 2154   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2155   attributes.set_is_evex_instruction();
 2156   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2157   emit_int16(0x1F, (0xC0 | encode));
 2158 }
 2159 
 2160 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2161   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2162   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2163   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2164   emit_int16(0x5A, (0xC0 | encode));
 2165 }
 2166 
 2167 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2168   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2169   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2170   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2171   attributes.set_rex_vex_w_reverted();
 2172   emit_int16(0x5A, (0xC0 | encode));
 2173 }
 2174 
 2175 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2176   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2177   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2178   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2179   emit_int16(0x5B, (0xC0 | encode));
 2180 }
 2181 
 2182 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2183   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2184   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2185   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2186   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2187 }
 2188 
 2189 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2190   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2191   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2192   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2193   emit_int16(0x5B, (0xC0 | encode));
 2194 }
 2195 
 2196 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2197   assert(VM_Version::supports_avx512dq(), "");
 2198   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2199   attributes.set_is_evex_instruction();
 2200   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2201   emit_int16(0x7A, (0xC0 | encode));
 2202 }
 2203 
 2204 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2205   assert(VM_Version::supports_avx512dq(), "");
 2206   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2207   attributes.set_is_evex_instruction();
 2208   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2209   emit_int16(0x7B, (0xC0 | encode));
 2210 }
 2211 
 2212 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2213   assert(VM_Version::supports_avx512dq(), "");
 2214   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2215   attributes.set_is_evex_instruction();
 2216   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2217   emit_int16(0x5B, (0xC0 | encode));
 2218 }
 2219 
 2220 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2221   assert(VM_Version::supports_avx512dq(), "");
 2222   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2223   attributes.set_is_evex_instruction();
 2224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2225   emit_int16(0x7A, (0xC0 | encode));
 2226 }
 2227 
 2228 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2229   assert(VM_Version::supports_avx512dq(), "");
 2230   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2231   attributes.set_is_evex_instruction();
 2232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2233   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2234 }
 2235 
 2236 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2237   assert(VM_Version::supports_avx512bw(), "");
 2238   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2239   attributes.set_is_evex_instruction();
 2240   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2241   emit_int16(0x30, (0xC0 | encode));
 2242 }
 2243 
 2244 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2245   assert(UseAVX > 2, "");
 2246   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2247   attributes.set_is_evex_instruction();
 2248   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2249   emit_int16(0x33, (0xC0 | encode));
 2250 }
 2251 
 2252 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2253   assert(UseAVX > 2, "");
 2254   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2255   attributes.set_is_evex_instruction();
 2256   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2257   emit_int16(0x31, (0xC0 | encode));
 2258 }
 2259 
 2260 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2261   assert(UseAVX > 2, "");
 2262   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2263   attributes.set_is_evex_instruction();
 2264   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2265   emit_int16(0x35, (0xC0 | encode));
 2266 }
 2267 
 2268 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2269   assert(UseAVX > 2, "");
 2270   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2271   attributes.set_is_evex_instruction();
 2272   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2273   emit_int16(0x32, (0xC0 | encode));
 2274 }
 2275 
 2276 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2277   assert(UseAVX > 2, "");
 2278   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2279   attributes.set_is_evex_instruction();
 2280   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2281   emit_int16(0x34, (0xC0 | encode));
 2282 }
 2283 
 2284 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2285   assert(UseAVX > 2, "");
 2286   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2287   attributes.set_is_evex_instruction();
 2288   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2289   emit_int16(0x25, (0xC0 | encode));
 2290 }
 2291 
 2292 void Assembler::decl(Address dst) {
 2293   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2294   InstructionMark im(this);
 2295   prefix(dst);
 2296   emit_int8((unsigned char)0xFF);
 2297   emit_operand(rcx, dst, 0);
 2298 }
 2299 
 2300 void Assembler::divsd(XMMRegister dst, Address src) {
 2301   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2302   InstructionMark im(this);
 2303   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2304   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2305   attributes.set_rex_vex_w_reverted();
 2306   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2307   emit_int8(0x5E);
 2308   emit_operand(dst, src, 0);
 2309 }
 2310 
 2311 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2312   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2313   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2314   attributes.set_rex_vex_w_reverted();
 2315   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2316   emit_int16(0x5E, (0xC0 | encode));
 2317 }
 2318 
 2319 void Assembler::divss(XMMRegister dst, Address src) {
 2320   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2321   InstructionMark im(this);
 2322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2323   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2324   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2325   emit_int8(0x5E);
 2326   emit_operand(dst, src, 0);
 2327 }
 2328 
 2329 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2330   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2331   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2332   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2333   emit_int16(0x5E, (0xC0 | encode));
 2334 }
 2335 
 2336 void Assembler::hlt() {
 2337   emit_int8((unsigned char)0xF4);
 2338 }
 2339 
 2340 void Assembler::idivl(Register src) {
 2341   int encode = prefix_and_encode(src->encoding());
 2342   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2343 }
 2344 
 2345 void Assembler::divl(Register src) { // Unsigned
 2346   int encode = prefix_and_encode(src->encoding());
 2347   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2348 }
 2349 
 2350 void Assembler::imull(Register src) {
 2351   int encode = prefix_and_encode(src->encoding());
 2352   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2353 }
 2354 
 2355 void Assembler::imull(Register dst, Register src) {
 2356   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2357   emit_int24(0x0F,
 2358              (unsigned char)0xAF,
 2359              (0xC0 | encode));
 2360 }
 2361 
 2362 void Assembler::imull(Register dst, Address src, int32_t value) {
 2363   InstructionMark im(this);
 2364   prefix(src, dst);
 2365   if (is8bit(value)) {
 2366     emit_int8((unsigned char)0x6B);
 2367     emit_operand(dst, src, 1);
 2368     emit_int8(value);
 2369   } else {
 2370     emit_int8((unsigned char)0x69);
 2371     emit_operand(dst, src, 4);
 2372     emit_int32(value);
 2373   }
 2374 }
 2375 
 2376 void Assembler::imull(Register dst, Register src, int value) {
 2377   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2378   if (is8bit(value)) {
 2379     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2380   } else {
 2381     emit_int16(0x69, (0xC0 | encode));
 2382     emit_int32(value);
 2383   }
 2384 }
 2385 
 2386 void Assembler::imull(Register dst, Address src) {
 2387   InstructionMark im(this);
 2388   prefix(src, dst);
 2389   emit_int16(0x0F, (unsigned char)0xAF);
 2390   emit_operand(dst, src, 0);
 2391 }
 2392 
 2393 
 2394 void Assembler::incl(Address dst) {
 2395   // Don't use it directly. Use MacroAssembler::increment() instead.
 2396   InstructionMark im(this);
 2397   prefix(dst);
 2398   emit_int8((unsigned char)0xFF);
 2399   emit_operand(rax, dst, 0);
 2400 }
 2401 
 2402 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2403   InstructionMark im(this);
 2404   assert((0 <= cc) && (cc < 16), "illegal cc");
 2405   if (L.is_bound()) {
 2406     address dst = target(L);
 2407     assert(dst != NULL, "jcc most probably wrong");
 2408 
 2409     const int short_size = 2;
 2410     const int long_size = 6;
 2411     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2412     if (maybe_short && is8bit(offs - short_size)) {
 2413       // 0111 tttn #8-bit disp
 2414       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2415     } else {
 2416       // 0000 1111 1000 tttn #32-bit disp
 2417       assert(is_simm32(offs - long_size),
 2418              "must be 32bit offset (call4)");
 2419       emit_int16(0x0F, (0x80 | cc));
 2420       emit_int32(offs - long_size);
 2421     }
 2422   } else {
 2423     // Note: could eliminate cond. jumps to this jump if condition
 2424     //       is the same however, seems to be rather unlikely case.
 2425     // Note: use jccb() if label to be bound is very close to get
 2426     //       an 8-bit displacement
 2427     L.add_patch_at(code(), locator());
 2428     emit_int16(0x0F, (0x80 | cc));
 2429     emit_int32(0);
 2430   }
 2431 }
 2432 
 2433 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2434   if (L.is_bound()) {
 2435     const int short_size = 2;
 2436     address entry = target(L);
 2437 #ifdef ASSERT
 2438     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2439     intptr_t delta = short_branch_delta();
 2440     if (delta != 0) {
 2441       dist += (dist < 0 ? (-delta) :delta);
 2442     }
 2443     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2444 #endif
 2445     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2446     // 0111 tttn #8-bit disp
 2447     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2448   } else {
 2449     InstructionMark im(this);
 2450     L.add_patch_at(code(), locator(), file, line);
 2451     emit_int16(0x70 | cc, 0);
 2452   }
 2453 }
 2454 
 2455 void Assembler::jmp(Address adr) {
 2456   InstructionMark im(this);
 2457   prefix(adr);
 2458   emit_int8((unsigned char)0xFF);
 2459   emit_operand(rsp, adr, 0);
 2460 }
 2461 
 2462 void Assembler::jmp(Label& L, bool maybe_short) {
 2463   if (L.is_bound()) {
 2464     address entry = target(L);
 2465     assert(entry != NULL, "jmp most probably wrong");
 2466     InstructionMark im(this);
 2467     const int short_size = 2;
 2468     const int long_size = 5;
 2469     intptr_t offs = entry - pc();
 2470     if (maybe_short && is8bit(offs - short_size)) {
 2471       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2472     } else {
 2473       emit_int8((unsigned char)0xE9);
 2474       emit_int32(offs - long_size);
 2475     }
 2476   } else {
 2477     // By default, forward jumps are always 32-bit displacements, since
 2478     // we can't yet know where the label will be bound.  If you're sure that
 2479     // the forward jump will not run beyond 256 bytes, use jmpb to
 2480     // force an 8-bit displacement.
 2481     InstructionMark im(this);
 2482     L.add_patch_at(code(), locator());
 2483     emit_int8((unsigned char)0xE9);
 2484     emit_int32(0);
 2485   }
 2486 }
 2487 
 2488 void Assembler::jmp(Register entry) {
 2489   int encode = prefix_and_encode(entry->encoding());
 2490   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2491 }
 2492 
 2493 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2494   InstructionMark im(this);
 2495   emit_int8((unsigned char)0xE9);
 2496   assert(dest != NULL, "must have a target");
 2497   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2498   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2499   emit_data(disp, rspec, call32_operand);
 2500 }
 2501 
 2502 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2503   if (L.is_bound()) {
 2504     const int short_size = 2;
 2505     address entry = target(L);
 2506     assert(entry != NULL, "jmp most probably wrong");
 2507 #ifdef ASSERT
 2508     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2509     intptr_t delta = short_branch_delta();
 2510     if (delta != 0) {
 2511       dist += (dist < 0 ? (-delta) :delta);
 2512     }
 2513     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2514 #endif
 2515     intptr_t offs = entry - pc();
 2516     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2517   } else {
 2518     InstructionMark im(this);
 2519     L.add_patch_at(code(), locator(), file, line);
 2520     emit_int16((unsigned char)0xEB, 0);
 2521   }
 2522 }
 2523 
 2524 void Assembler::ldmxcsr( Address src) {
 2525   if (UseAVX > 0 ) {
 2526     InstructionMark im(this);
 2527     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2528     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2529     emit_int8((unsigned char)0xAE);
 2530     emit_operand(as_Register(2), src, 0);
 2531   } else {
 2532     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2533     InstructionMark im(this);
 2534     prefix(src);
 2535     emit_int16(0x0F, (unsigned char)0xAE);
 2536     emit_operand(as_Register(2), src, 0);
 2537   }
 2538 }
 2539 
 2540 void Assembler::leal(Register dst, Address src) {
 2541   InstructionMark im(this);
 2542   prefix(src, dst);
 2543   emit_int8((unsigned char)0x8D);
 2544   emit_operand(dst, src, 0);
 2545 }
 2546 
 2547 void Assembler::lfence() {
 2548   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2549 }
 2550 
 2551 void Assembler::lock() {
 2552   emit_int8((unsigned char)0xF0);
 2553 }
 2554 
 2555 void Assembler::size_prefix() {
 2556   emit_int8(0x66);
 2557 }
 2558 
 2559 void Assembler::lzcntl(Register dst, Register src) {
 2560   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2561   emit_int8((unsigned char)0xF3);
 2562   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2563   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2564 }
 2565 
 2566 void Assembler::lzcntl(Register dst, Address src) {
 2567   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2568   InstructionMark im(this);
 2569   emit_int8((unsigned char)0xF3);
 2570   prefix(src, dst);
 2571   emit_int16(0x0F, (unsigned char)0xBD);
 2572   emit_operand(dst, src, 0);
 2573 }
 2574 
 2575 // Emit mfence instruction
 2576 void Assembler::mfence() {
 2577   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2578   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2579 }
 2580 
 2581 // Emit sfence instruction
 2582 void Assembler::sfence() {
 2583   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2584   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2585 }
 2586 
 2587 void Assembler::mov(Register dst, Register src) {
 2588   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2589 }
 2590 
 2591 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2592   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2593   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2594   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2595   attributes.set_rex_vex_w_reverted();
 2596   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2597   emit_int16(0x28, (0xC0 | encode));
 2598 }
 2599 
 2600 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2601   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2602   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2603   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2604   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2605   emit_int16(0x28, (0xC0 | encode));
 2606 }
 2607 
 2608 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2609   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2611   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2612   emit_int16(0x16, (0xC0 | encode));
 2613 }
 2614 
 2615 void Assembler::movb(Register dst, Address src) {
 2616   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2617   InstructionMark im(this);
 2618   prefix(src, dst, true);
 2619   emit_int8((unsigned char)0x8A);
 2620   emit_operand(dst, src, 0);
 2621 }
 2622 
 2623 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2624   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2625   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2626   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2627   attributes.set_rex_vex_w_reverted();
 2628   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2629   emit_int16(0x12, 0xC0 | encode);
 2630 }
 2631 
 2632 void Assembler::movddup(XMMRegister dst, Address src) {
 2633   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2634   InstructionMark im(this);
 2635   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2636   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2637   attributes.set_rex_vex_w_reverted();
 2638   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2639   emit_int8(0x12);
 2640   emit_operand(dst, src, 0);
 2641 }
 2642 
 2643 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2644   assert(VM_Version::supports_avx(), "");
 2645   InstructionMark im(this);
 2646   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2647   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2648   attributes.set_rex_vex_w_reverted();
 2649   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2650   emit_int8(0x12);
 2651   emit_operand(dst, src, 0);
 2652 }
 2653 
 2654 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2655   assert(VM_Version::supports_avx512dq(), "");
 2656   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2657   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2658   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2659 }
 2660 
 2661 void Assembler::kmovbl(KRegister dst, Register src) {
 2662   assert(VM_Version::supports_avx512dq(), "");
 2663   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2664   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2665   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2666 }
 2667 
 2668 void Assembler::kmovbl(Register dst, KRegister src) {
 2669   assert(VM_Version::supports_avx512dq(), "");
 2670   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2671   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2672   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2673 }
 2674 
 2675 void Assembler::kmovwl(KRegister dst, Register src) {
 2676   assert(VM_Version::supports_evex(), "");
 2677   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2678   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2679   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2680 }
 2681 
 2682 void Assembler::kmovwl(Register dst, KRegister src) {
 2683   assert(VM_Version::supports_evex(), "");
 2684   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2685   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2686   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2687 }
 2688 
 2689 void Assembler::kmovwl(KRegister dst, Address src) {
 2690   assert(VM_Version::supports_evex(), "");
 2691   InstructionMark im(this);
 2692   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2693   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2694   emit_int8((unsigned char)0x90);
 2695   emit_operand(dst, src, 0);
 2696 }
 2697 
 2698 void Assembler::kmovwl(Address dst, KRegister src) {
 2699   assert(VM_Version::supports_evex(), "");
 2700   InstructionMark im(this);
 2701   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2702   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2703   emit_int8((unsigned char)0x91);
 2704   emit_operand(src, dst, 0);
 2705 }
 2706 
 2707 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2708   assert(VM_Version::supports_evex(), "");
 2709   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2710   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2711   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2712 }
 2713 
 2714 void Assembler::kmovdl(KRegister dst, Register src) {
 2715   assert(VM_Version::supports_avx512bw(), "");
 2716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2717   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2718   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2719 }
 2720 
 2721 void Assembler::kmovdl(Register dst, KRegister src) {
 2722   assert(VM_Version::supports_avx512bw(), "");
 2723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2724   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2725   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2726 }
 2727 
 2728 void Assembler::kmovql(KRegister dst, KRegister src) {
 2729   assert(VM_Version::supports_avx512bw(), "");
 2730   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2731   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2732   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2733 }
 2734 
 2735 void Assembler::kmovql(KRegister dst, Address src) {
 2736   assert(VM_Version::supports_avx512bw(), "");
 2737   InstructionMark im(this);
 2738   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2739   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2740   emit_int8((unsigned char)0x90);
 2741   emit_operand(dst, src, 0);
 2742 }
 2743 
 2744 void Assembler::kmovql(Address dst, KRegister src) {
 2745   assert(VM_Version::supports_avx512bw(), "");
 2746   InstructionMark im(this);
 2747   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2748   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2749   emit_int8((unsigned char)0x91);
 2750   emit_operand(src, dst, 0);
 2751 }
 2752 
 2753 void Assembler::kmovql(KRegister dst, Register src) {
 2754   assert(VM_Version::supports_avx512bw(), "");
 2755   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2756   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2757   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2758 }
 2759 
 2760 void Assembler::kmovql(Register dst, KRegister src) {
 2761   assert(VM_Version::supports_avx512bw(), "");
 2762   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2763   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2764   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2765 }
 2766 
 2767 void Assembler::knotwl(KRegister dst, KRegister src) {
 2768   assert(VM_Version::supports_evex(), "");
 2769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2770   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2771   emit_int16(0x44, (0xC0 | encode));
 2772 }
 2773 
 2774 void Assembler::knotbl(KRegister dst, KRegister src) {
 2775   assert(VM_Version::supports_evex(), "");
 2776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2777   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2778   emit_int16(0x44, (0xC0 | encode));
 2779 }
 2780 
 2781 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2782   assert(VM_Version::supports_avx512dq(), "");
 2783   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2784   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2785   emit_int16(0x45, (0xC0 | encode));
 2786 }
 2787 
 2788 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2789   assert(VM_Version::supports_evex(), "");
 2790   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2791   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2792   emit_int16(0x45, (0xC0 | encode));
 2793 }
 2794 
 2795 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2796   assert(VM_Version::supports_avx512bw(), "");
 2797   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2798   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2799   emit_int16(0x45, (0xC0 | encode));
 2800 }
 2801 
 2802 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2803   assert(VM_Version::supports_avx512bw(), "");
 2804   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2805   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2806   emit_int16(0x45, (0xC0 | encode));
 2807 }
 2808 
 2809 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2810   assert(VM_Version::supports_avx512dq(), "");
 2811   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2812   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2813   emit_int16(0x47, (0xC0 | encode));
 2814 }
 2815 
 2816 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2817   assert(VM_Version::supports_evex(), "");
 2818   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2819   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2820   emit_int16(0x47, (0xC0 | encode));
 2821 }
 2822 
 2823 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2824   assert(VM_Version::supports_avx512bw(), "");
 2825   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2826   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2827   emit_int16(0x47, (0xC0 | encode));
 2828 }
 2829 
 2830 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2831   assert(VM_Version::supports_avx512bw(), "");
 2832   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2833   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2834   emit_int16(0x47, (0xC0 | encode));
 2835 }
 2836 
 2837 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2838   assert(VM_Version::supports_avx512dq(), "");
 2839   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2840   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2841   emit_int16(0x41, (0xC0 | encode));
 2842 }
 2843 
 2844 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2845   assert(VM_Version::supports_evex(), "");
 2846   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2847   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2848   emit_int16(0x41, (0xC0 | encode));
 2849 }
 2850 
 2851 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2852   assert(VM_Version::supports_avx512bw(), "");
 2853   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2854   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2855   emit_int16(0x41, (0xC0 | encode));
 2856 }
 2857 
 2858 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2859   assert(VM_Version::supports_avx512bw(), "");
 2860   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2861   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2862   emit_int16(0x41, (0xC0 | encode));
 2863 }
 2864 
 2865 void Assembler::knotdl(KRegister dst, KRegister src) {
 2866   assert(VM_Version::supports_avx512bw(), "");
 2867   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2868   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2869   emit_int16(0x44, (0xC0 | encode));
 2870 }
 2871 
 2872 void Assembler::knotql(KRegister dst, KRegister src) {
 2873   assert(VM_Version::supports_avx512bw(), "");
 2874   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2875   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2876   emit_int16(0x44, (0xC0 | encode));
 2877 }
 2878 
 2879 // This instruction produces ZF or CF flags
 2880 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2881   assert(VM_Version::supports_avx512dq(), "");
 2882   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2883   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2884   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2885 }
 2886 
 2887 // This instruction produces ZF or CF flags
 2888 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2889   assert(VM_Version::supports_evex(), "");
 2890   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2891   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2892   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2893 }
 2894 
 2895 // This instruction produces ZF or CF flags
 2896 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2897   assert(VM_Version::supports_avx512bw(), "");
 2898   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2899   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2900   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2901 }
 2902 
 2903 // This instruction produces ZF or CF flags
 2904 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2905   assert(VM_Version::supports_avx512bw(), "");
 2906   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2907   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2908   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2909 }
 2910 
 2911 // This instruction produces ZF or CF flags
 2912 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2913   assert(VM_Version::supports_avx512bw(), "");
 2914   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2915   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2916   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2917 }
 2918 
 2919 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2920   assert(VM_Version::supports_avx512bw(), "");
 2921   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2922   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2923   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2924 }
 2925 
 2926 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2927   assert(VM_Version::supports_avx512dq(), "");
 2928   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2929   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2930   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2931 }
 2932 
 2933 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2934   assert(VM_Version::supports_avx512dq(), "");
 2935   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2936   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2937   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2938 }
 2939 
 2940 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2941   assert(VM_Version::supports_avx512bw(), "");
 2942   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2943   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2944   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2945 }
 2946 
 2947 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2948   assert(VM_Version::supports_avx512bw(), "");
 2949   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2950   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2951   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2952 }
 2953 
 2954 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2955   assert(VM_Version::supports_avx512dq(), "");
 2956   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2957   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2958   emit_int16(0x46, (0xC0 | encode));
 2959 }
 2960 
 2961 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2962   assert(VM_Version::supports_avx512dq(), "");
 2963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2964   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2965   emit_int16(0x32, (0xC0 | encode));
 2966   emit_int8(imm8);
 2967 }
 2968 
 2969 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2970   assert(VM_Version::supports_avx512bw(), "");
 2971   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2972   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2973   emit_int16(0x33, (0xC0 | encode));
 2974   emit_int8(imm8);
 2975 }
 2976 
 2977 
 2978 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2979   assert(VM_Version::supports_avx512dq(), "");
 2980   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2981   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2982   emit_int16(0x30, (0xC0 | encode));
 2983 }
 2984 
 2985 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2986   assert(VM_Version::supports_evex(), "");
 2987   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2988   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2989   emit_int16(0x30, (0xC0 | encode));
 2990   emit_int8(imm8);
 2991 }
 2992 
 2993 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2994   assert(VM_Version::supports_avx512bw(), "");
 2995   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2996   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2997   emit_int16(0x31, (0xC0 | encode));
 2998   emit_int8(imm8);
 2999 }
 3000 
 3001 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 3002   assert(VM_Version::supports_avx512bw(), "");
 3003   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3004   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3005   emit_int16(0x31, (0xC0 | encode));
 3006   emit_int8(imm8);
 3007 }
 3008 
 3009 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 3010   assert(VM_Version::supports_avx512bw(), "");
 3011   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3012   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3013   emit_int16(0x4B, (0xC0 | encode));
 3014 }
 3015 
 3016 void Assembler::movb(Address dst, int imm8) {
 3017   InstructionMark im(this);
 3018    prefix(dst);
 3019   emit_int8((unsigned char)0xC6);
 3020   emit_operand(rax, dst, 1);
 3021   emit_int8(imm8);
 3022 }
 3023 
 3024 
 3025 void Assembler::movb(Address dst, Register src) {
 3026   assert(src->has_byte_register(), "must have byte register");
 3027   InstructionMark im(this);
 3028   prefix(dst, src, true);
 3029   emit_int8((unsigned char)0x88);
 3030   emit_operand(src, dst, 0);
 3031 }
 3032 
 3033 void Assembler::movdl(XMMRegister dst, Register src) {
 3034   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3035   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3036   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3037   emit_int16(0x6E, (0xC0 | encode));
 3038 }
 3039 
 3040 void Assembler::movdl(Register dst, XMMRegister src) {
 3041   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3042   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3043   // swap src/dst to get correct prefix
 3044   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3045   emit_int16(0x7E, (0xC0 | encode));
 3046 }
 3047 
 3048 void Assembler::movdl(XMMRegister dst, Address src) {
 3049   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3050   InstructionMark im(this);
 3051   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3052   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3053   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3054   emit_int8(0x6E);
 3055   emit_operand(dst, src, 0);
 3056 }
 3057 
 3058 void Assembler::movdl(Address dst, XMMRegister src) {
 3059   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3060   InstructionMark im(this);
 3061   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3062   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3063   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3064   emit_int8(0x7E);
 3065   emit_operand(src, dst, 0);
 3066 }
 3067 
 3068 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 3069   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3070   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3071   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3072   emit_int16(0x6F, (0xC0 | encode));
 3073 }
 3074 
 3075 void Assembler::movdqa(XMMRegister dst, Address src) {
 3076   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3077   InstructionMark im(this);
 3078   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3079   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3080   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3081   emit_int8(0x6F);
 3082   emit_operand(dst, src, 0);
 3083 }
 3084 
 3085 void Assembler::movdqu(XMMRegister dst, Address src) {
 3086   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3087   InstructionMark im(this);
 3088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3089   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3090   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3091   emit_int8(0x6F);
 3092   emit_operand(dst, src, 0);
 3093 }
 3094 
 3095 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 3096   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3098   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3099   emit_int16(0x6F, (0xC0 | encode));
 3100 }
 3101 
 3102 void Assembler::movdqu(Address dst, XMMRegister src) {
 3103   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3104   InstructionMark im(this);
 3105   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3106   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3107   attributes.reset_is_clear_context();
 3108   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3109   emit_int8(0x7F);
 3110   emit_operand(src, dst, 0);
 3111 }
 3112 
 3113 // Move Unaligned 256bit Vector
 3114 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 3115   assert(UseAVX > 0, "");
 3116   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3117   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3118   emit_int16(0x6F, (0xC0 | encode));
 3119 }
 3120 
 3121 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3122   assert(UseAVX > 0, "");
 3123   InstructionMark im(this);
 3124   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3125   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3126   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3127   emit_int8(0x6F);
 3128   emit_operand(dst, src, 0);
 3129 }
 3130 
 3131 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3132   assert(UseAVX > 0, "");
 3133   InstructionMark im(this);
 3134   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3135   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3136   attributes.reset_is_clear_context();
 3137   // swap src<->dst for encoding
 3138   assert(src != xnoreg, "sanity");
 3139   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3140   emit_int8(0x7F);
 3141   emit_operand(src, dst, 0);
 3142 }
 3143 
 3144 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3145   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3146   InstructionMark im(this);
 3147   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3148   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3149   emit_int8((unsigned char)0x8C);
 3150   emit_operand(dst, src, 0);
 3151 }
 3152 
 3153 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3154   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3155   InstructionMark im(this);
 3156   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3157   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3158   emit_int8((unsigned char)0x8C);
 3159   emit_operand(dst, src, 0);
 3160 }
 3161 
 3162 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3163   assert(UseAVX > 0, "requires some form of AVX");
 3164   InstructionMark im(this);
 3165   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3166   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3167   emit_int8(0x2C);
 3168   emit_operand(dst, src, 0);
 3169 }
 3170 
 3171 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3172   assert(UseAVX > 0, "requires some form of AVX");
 3173   InstructionMark im(this);
 3174   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3175   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3176   emit_int8(0x2D);
 3177   emit_operand(dst, src, 0);
 3178 }
 3179 
 3180 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3181   assert(UseAVX > 0, "");
 3182   InstructionMark im(this);
 3183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3184   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3185   emit_int8(0x2E);
 3186   emit_operand(src, dst, 0);
 3187 }
 3188 
 3189 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3190   assert(UseAVX > 0, "");
 3191   InstructionMark im(this);
 3192   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3193   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3194   emit_int8(0x2F);
 3195   emit_operand(src, dst, 0);
 3196 }
 3197 
 3198 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3199 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3200   assert(VM_Version::supports_avx512vlbw(), "");
 3201   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3202   attributes.set_embedded_opmask_register_specifier(mask);
 3203   attributes.set_is_evex_instruction();
 3204   if (merge) {
 3205     attributes.reset_is_clear_context();
 3206   }
 3207   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3208   emit_int16(0x6F, (0xC0 | encode));
 3209 }
 3210 
 3211 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
 3212   // Unmasked instruction
 3213   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3214 }
 3215 
 3216 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3217   assert(VM_Version::supports_avx512vlbw(), "");
 3218   InstructionMark im(this);
 3219   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3220   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3221   attributes.set_embedded_opmask_register_specifier(mask);
 3222   attributes.set_is_evex_instruction();
 3223   if (merge) {
 3224     attributes.reset_is_clear_context();
 3225   }
 3226   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3227   emit_int8(0x6F);
 3228   emit_operand(dst, src, 0);
 3229 }
 3230 
 3231 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
 3232   // Unmasked instruction
 3233   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3234 }
 3235 
 3236 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3237   assert(VM_Version::supports_avx512vlbw(), "");
 3238   assert(src != xnoreg, "sanity");
 3239   InstructionMark im(this);
 3240   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3241   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3242   attributes.set_embedded_opmask_register_specifier(mask);
 3243   attributes.set_is_evex_instruction();
 3244   if (merge) {
 3245     attributes.reset_is_clear_context();
 3246   }
 3247   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3248   emit_int8(0x7F);
 3249   emit_operand(src, dst, 0);
 3250 }
 3251 
 3252 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
 3253   // Unmasked instruction
 3254   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3255 }
 3256 
 3257 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3258   assert(VM_Version::supports_avx512vlbw(), "");
 3259   InstructionMark im(this);
 3260   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3261   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3262   attributes.set_embedded_opmask_register_specifier(mask);
 3263   attributes.set_is_evex_instruction();
 3264   if (merge) {
 3265     attributes.reset_is_clear_context();
 3266   }
 3267   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3268   emit_int8(0x6F);
 3269   emit_operand(dst, src, 0);
 3270 }
 3271 
 3272 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
 3273   // Unmasked instruction
 3274   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3275 }
 3276 
 3277 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3278   assert(VM_Version::supports_avx512vlbw(), "");
 3279   assert(src != xnoreg, "sanity");
 3280   InstructionMark im(this);
 3281   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3282   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3283   attributes.set_embedded_opmask_register_specifier(mask);
 3284   attributes.set_is_evex_instruction();
 3285   if (merge) {
 3286     attributes.reset_is_clear_context();
 3287   }
 3288   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3289   emit_int8(0x7F);
 3290   emit_operand(src, dst, 0);
 3291 }
 3292 
 3293 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3294   // Unmasked instruction
 3295   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3296 }
 3297 
 3298 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3299   assert(VM_Version::supports_evex(), "");
 3300   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3301   attributes.set_embedded_opmask_register_specifier(mask);
 3302   attributes.set_is_evex_instruction();
 3303   if (merge) {
 3304     attributes.reset_is_clear_context();
 3305   }
 3306   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3307   emit_int16(0x6F, (0xC0 | encode));
 3308 }
 3309 
 3310 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3311   // Unmasked instruction
 3312   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3313 }
 3314 
 3315 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3316   assert(VM_Version::supports_evex(), "");
 3317   InstructionMark im(this);
 3318   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3319   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3320   attributes.set_embedded_opmask_register_specifier(mask);
 3321   attributes.set_is_evex_instruction();
 3322   if (merge) {
 3323     attributes.reset_is_clear_context();
 3324   }
 3325   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3326   emit_int8(0x6F);
 3327   emit_operand(dst, src, 0);
 3328 }
 3329 
 3330 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3331   // Unmasked isntruction
 3332   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3333 }
 3334 
 3335 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3336   assert(VM_Version::supports_evex(), "");
 3337   assert(src != xnoreg, "sanity");
 3338   InstructionMark im(this);
 3339   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3340   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3341   attributes.set_embedded_opmask_register_specifier(mask);
 3342   attributes.set_is_evex_instruction();
 3343   if (merge) {
 3344     attributes.reset_is_clear_context();
 3345   }
 3346   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3347   emit_int8(0x7F);
 3348   emit_operand(src, dst, 0);
 3349 }
 3350 
 3351 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3352   // Unmasked instruction
 3353   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3354 }
 3355 
 3356 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3357   assert(VM_Version::supports_evex(), "");
 3358   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3359   attributes.set_embedded_opmask_register_specifier(mask);
 3360   attributes.set_is_evex_instruction();
 3361   if (merge) {
 3362     attributes.reset_is_clear_context();
 3363   }
 3364   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3365   emit_int16(0x6F, (0xC0 | encode));
 3366 }
 3367 
 3368 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3369   // Unmasked instruction
 3370   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3371 }
 3372 
 3373 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3374   assert(VM_Version::supports_evex(), "");
 3375   InstructionMark im(this);
 3376   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3377   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3378   attributes.set_embedded_opmask_register_specifier(mask);
 3379   attributes.set_is_evex_instruction();
 3380   if (merge) {
 3381     attributes.reset_is_clear_context();
 3382   }
 3383   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3384   emit_int8(0x6F);
 3385   emit_operand(dst, src, 0);
 3386 }
 3387 
 3388 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3389   // Unmasked instruction
 3390   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3391 }
 3392 
 3393 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3394   assert(VM_Version::supports_evex(), "");
 3395   assert(src != xnoreg, "sanity");
 3396   InstructionMark im(this);
 3397   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3398   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3399   attributes.set_embedded_opmask_register_specifier(mask);
 3400   if (merge) {
 3401     attributes.reset_is_clear_context();
 3402   }
 3403   attributes.set_is_evex_instruction();
 3404   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3405   emit_int8(0x7F);
 3406   emit_operand(src, dst, 0);
 3407 }
 3408 
 3409 // Uses zero extension on 64bit
 3410 
 3411 void Assembler::movl(Register dst, int32_t imm32) {
 3412   int encode = prefix_and_encode(dst->encoding());
 3413   emit_int8(0xB8 | encode);
 3414   emit_int32(imm32);
 3415 }
 3416 
 3417 void Assembler::movl(Register dst, Register src) {
 3418   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3419   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3420 }
 3421 
 3422 void Assembler::movl(Register dst, Address src) {
 3423   InstructionMark im(this);
 3424   prefix(src, dst);
 3425   emit_int8((unsigned char)0x8B);
 3426   emit_operand(dst, src, 0);
 3427 }
 3428 
 3429 void Assembler::movl(Address dst, int32_t imm32) {
 3430   InstructionMark im(this);
 3431   prefix(dst);
 3432   emit_int8((unsigned char)0xC7);
 3433   emit_operand(rax, dst, 4);
 3434   emit_int32(imm32);
 3435 }
 3436 
 3437 void Assembler::movl(Address dst, Register src) {
 3438   InstructionMark im(this);
 3439   prefix(dst, src);
 3440   emit_int8((unsigned char)0x89);
 3441   emit_operand(src, dst, 0);
 3442 }
 3443 
 3444 // New cpus require to use movsd and movss to avoid partial register stall
 3445 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3446 // The selection is done in MacroAssembler::movdbl() and movflt().
 3447 void Assembler::movlpd(XMMRegister dst, Address src) {
 3448   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3449   InstructionMark im(this);
 3450   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3451   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3452   attributes.set_rex_vex_w_reverted();
 3453   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3454   emit_int8(0x12);
 3455   emit_operand(dst, src, 0);
 3456 }
 3457 
 3458 void Assembler::movq(XMMRegister dst, Address src) {
 3459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3460   InstructionMark im(this);
 3461   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3462   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3463   attributes.set_rex_vex_w_reverted();
 3464   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3465   emit_int8(0x7E);
 3466   emit_operand(dst, src, 0);
 3467 }
 3468 
 3469 void Assembler::movq(Address dst, XMMRegister src) {
 3470   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3471   InstructionMark im(this);
 3472   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3473   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3474   attributes.set_rex_vex_w_reverted();
 3475   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3476   emit_int8((unsigned char)0xD6);
 3477   emit_operand(src, dst, 0);
 3478 }
 3479 
 3480 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3481   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3482   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3483   attributes.set_rex_vex_w_reverted();
 3484   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3485   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3486 }
 3487 
 3488 void Assembler::movq(Register dst, XMMRegister src) {
 3489   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3490   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3491   // swap src/dst to get correct prefix
 3492   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3493   emit_int16(0x7E, (0xC0 | encode));
 3494 }
 3495 
 3496 void Assembler::movq(XMMRegister dst, Register src) {
 3497   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3498   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3499   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3500   emit_int16(0x6E, (0xC0 | encode));
 3501 }
 3502 
 3503 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3504   InstructionMark im(this);
 3505   prefix(src, dst);
 3506   emit_int16(0x0F, (unsigned char)0xBE);
 3507   emit_operand(dst, src, 0);
 3508 }
 3509 
 3510 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3511   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3512   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3513   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3514 }
 3515 
 3516 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3517   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3518   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3519   attributes.set_rex_vex_w_reverted();
 3520   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3521   emit_int16(0x10, (0xC0 | encode));
 3522 }
 3523 
 3524 void Assembler::movsd(XMMRegister dst, Address src) {
 3525   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3526   InstructionMark im(this);
 3527   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3528   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3529   attributes.set_rex_vex_w_reverted();
 3530   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3531   emit_int8(0x10);
 3532   emit_operand(dst, src, 0);
 3533 }
 3534 
 3535 void Assembler::movsd(Address dst, XMMRegister src) {
 3536   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3537   InstructionMark im(this);
 3538   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3539   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3540   attributes.reset_is_clear_context();
 3541   attributes.set_rex_vex_w_reverted();
 3542   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3543   emit_int8(0x11);
 3544   emit_operand(src, dst, 0);
 3545 }
 3546 
 3547 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3548   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3549   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3550   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3551   emit_int16(0x10, (0xC0 | encode));
 3552 }
 3553 
 3554 void Assembler::movss(XMMRegister dst, Address src) {
 3555   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3556   InstructionMark im(this);
 3557   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3558   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3559   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3560   emit_int8(0x10);
 3561   emit_operand(dst, src, 0);
 3562 }
 3563 
 3564 void Assembler::movss(Address dst, XMMRegister src) {
 3565   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3566   InstructionMark im(this);
 3567   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3568   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3569   attributes.reset_is_clear_context();
 3570   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3571   emit_int8(0x11);
 3572   emit_operand(src, dst, 0);
 3573 }
 3574 
 3575 void Assembler::movswl(Register dst, Address src) { // movsxw
 3576   InstructionMark im(this);
 3577   prefix(src, dst);
 3578   emit_int16(0x0F, (unsigned char)0xBF);
 3579   emit_operand(dst, src, 0);
 3580 }
 3581 
 3582 void Assembler::movswl(Register dst, Register src) { // movsxw
 3583   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3584   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3585 }
 3586 
 3587 void Assembler::movups(XMMRegister dst, Address src) {
 3588   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3589   InstructionMark im(this);
 3590   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3591   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3592   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3593   emit_int8(0x10);
 3594   emit_operand(dst, src, 0);
 3595 }
 3596 
 3597 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
 3598   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3599   InstructionMark im(this);
 3600   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3601   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3602   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3603   emit_int8(0x10);
 3604   emit_operand(dst, src, 0);
 3605 }
 3606 
 3607 void Assembler::movups(Address dst, XMMRegister src) {
 3608   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3609   InstructionMark im(this);
 3610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3611   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3612   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3613   emit_int8(0x11);
 3614   emit_operand(src, dst, 0);
 3615 }
 3616 
 3617 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
 3618   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3619   InstructionMark im(this);
 3620   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3621   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3622   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3623   emit_int8(0x11);
 3624   emit_operand(src, dst, 0);
 3625 }
 3626 
 3627 void Assembler::movw(Address dst, int imm16) {
 3628   InstructionMark im(this);
 3629 
 3630   emit_int8(0x66); // switch to 16-bit mode
 3631   prefix(dst);
 3632   emit_int8((unsigned char)0xC7);
 3633   emit_operand(rax, dst, 2);
 3634   emit_int16(imm16);
 3635 }
 3636 
 3637 void Assembler::movw(Register dst, Address src) {
 3638   InstructionMark im(this);
 3639   emit_int8(0x66);
 3640   prefix(src, dst);
 3641   emit_int8((unsigned char)0x8B);
 3642   emit_operand(dst, src, 0);
 3643 }
 3644 
 3645 void Assembler::movw(Address dst, Register src) {
 3646   InstructionMark im(this);
 3647   emit_int8(0x66);
 3648   prefix(dst, src);
 3649   emit_int8((unsigned char)0x89);
 3650   emit_operand(src, dst, 0);
 3651 }
 3652 
 3653 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3654   InstructionMark im(this);
 3655   prefix(src, dst);
 3656   emit_int16(0x0F, (unsigned char)0xB6);
 3657   emit_operand(dst, src, 0);
 3658 }
 3659 
 3660 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3661   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3662   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3663   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3664 }
 3665 
 3666 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3667   InstructionMark im(this);
 3668   prefix(src, dst);
 3669   emit_int16(0x0F, (unsigned char)0xB7);
 3670   emit_operand(dst, src, 0);
 3671 }
 3672 
 3673 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3674   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3675   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3676 }
 3677 
 3678 void Assembler::mull(Address src) {
 3679   InstructionMark im(this);
 3680   prefix(src);
 3681   emit_int8((unsigned char)0xF7);
 3682   emit_operand(rsp, src, 0);
 3683 }
 3684 
 3685 void Assembler::mull(Register src) {
 3686   int encode = prefix_and_encode(src->encoding());
 3687   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3688 }
 3689 
 3690 void Assembler::mulsd(XMMRegister dst, Address src) {
 3691   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3692   InstructionMark im(this);
 3693   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3694   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3695   attributes.set_rex_vex_w_reverted();
 3696   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3697   emit_int8(0x59);
 3698   emit_operand(dst, src, 0);
 3699 }
 3700 
 3701 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3702   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3703   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3704   attributes.set_rex_vex_w_reverted();
 3705   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3706   emit_int16(0x59, (0xC0 | encode));
 3707 }
 3708 
 3709 void Assembler::mulss(XMMRegister dst, Address src) {
 3710   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3711   InstructionMark im(this);
 3712   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3713   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3714   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3715   emit_int8(0x59);
 3716   emit_operand(dst, src, 0);
 3717 }
 3718 
 3719 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3720   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3721   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3722   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3723   emit_int16(0x59, (0xC0 | encode));
 3724 }
 3725 
 3726 void Assembler::negl(Register dst) {
 3727   int encode = prefix_and_encode(dst->encoding());
 3728   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3729 }
 3730 
 3731 void Assembler::negl(Address dst) {
 3732   InstructionMark im(this);
 3733   prefix(dst);
 3734   emit_int8((unsigned char)0xF7);
 3735   emit_operand(as_Register(3), dst, 0);
 3736 }
 3737 
 3738 void Assembler::nop(int i) {
 3739 #ifdef ASSERT
 3740   assert(i > 0, " ");
 3741   // The fancy nops aren't currently recognized by debuggers making it a
 3742   // pain to disassemble code while debugging. If asserts are on clearly
 3743   // speed is not an issue so simply use the single byte traditional nop
 3744   // to do alignment.
 3745 
 3746   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3747   return;
 3748 
 3749 #endif // ASSERT
 3750 
 3751   if (UseAddressNop && VM_Version::is_intel()) {
 3752     //
 3753     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3754     //  1: 0x90
 3755     //  2: 0x66 0x90
 3756     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3757     //  4: 0x0F 0x1F 0x40 0x00
 3758     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3759     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3760     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3761     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3762     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3763     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3764     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3765 
 3766     // The rest coding is Intel specific - don't use consecutive address nops
 3767 
 3768     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3769     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3770     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3771     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3772 
 3773     while(i >= 15) {
 3774       // For Intel don't generate consecutive address nops (mix with regular nops)
 3775       i -= 15;
 3776       emit_int24(0x66, 0x66, 0x66);
 3777       addr_nop_8();
 3778       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3779     }
 3780     switch (i) {
 3781       case 14:
 3782         emit_int8(0x66); // size prefix
 3783       case 13:
 3784         emit_int8(0x66); // size prefix
 3785       case 12:
 3786         addr_nop_8();
 3787         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3788         break;
 3789       case 11:
 3790         emit_int8(0x66); // size prefix
 3791       case 10:
 3792         emit_int8(0x66); // size prefix
 3793       case 9:
 3794         emit_int8(0x66); // size prefix
 3795       case 8:
 3796         addr_nop_8();
 3797         break;
 3798       case 7:
 3799         addr_nop_7();
 3800         break;
 3801       case 6:
 3802         emit_int8(0x66); // size prefix
 3803       case 5:
 3804         addr_nop_5();
 3805         break;
 3806       case 4:
 3807         addr_nop_4();
 3808         break;
 3809       case 3:
 3810         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3811         emit_int8(0x66); // size prefix
 3812       case 2:
 3813         emit_int8(0x66); // size prefix
 3814       case 1:
 3815         emit_int8((unsigned char)0x90);
 3816                          // nop
 3817         break;
 3818       default:
 3819         assert(i == 0, " ");
 3820     }
 3821     return;
 3822   }
 3823   if (UseAddressNop && VM_Version::is_amd_family()) {
 3824     //
 3825     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3826     //  1: 0x90
 3827     //  2: 0x66 0x90
 3828     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3829     //  4: 0x0F 0x1F 0x40 0x00
 3830     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3831     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3832     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3833     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3834     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3835     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3836     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3837 
 3838     // The rest coding is AMD specific - use consecutive address nops
 3839 
 3840     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3841     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3842     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3843     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3844     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3845     //     Size prefixes (0x66) are added for larger sizes
 3846 
 3847     while(i >= 22) {
 3848       i -= 11;
 3849       emit_int24(0x66, 0x66, 0x66);
 3850       addr_nop_8();
 3851     }
 3852     // Generate first nop for size between 21-12
 3853     switch (i) {
 3854       case 21:
 3855         i -= 1;
 3856         emit_int8(0x66); // size prefix
 3857       case 20:
 3858       case 19:
 3859         i -= 1;
 3860         emit_int8(0x66); // size prefix
 3861       case 18:
 3862       case 17:
 3863         i -= 1;
 3864         emit_int8(0x66); // size prefix
 3865       case 16:
 3866       case 15:
 3867         i -= 8;
 3868         addr_nop_8();
 3869         break;
 3870       case 14:
 3871       case 13:
 3872         i -= 7;
 3873         addr_nop_7();
 3874         break;
 3875       case 12:
 3876         i -= 6;
 3877         emit_int8(0x66); // size prefix
 3878         addr_nop_5();
 3879         break;
 3880       default:
 3881         assert(i < 12, " ");
 3882     }
 3883 
 3884     // Generate second nop for size between 11-1
 3885     switch (i) {
 3886       case 11:
 3887         emit_int8(0x66); // size prefix
 3888       case 10:
 3889         emit_int8(0x66); // size prefix
 3890       case 9:
 3891         emit_int8(0x66); // size prefix
 3892       case 8:
 3893         addr_nop_8();
 3894         break;
 3895       case 7:
 3896         addr_nop_7();
 3897         break;
 3898       case 6:
 3899         emit_int8(0x66); // size prefix
 3900       case 5:
 3901         addr_nop_5();
 3902         break;
 3903       case 4:
 3904         addr_nop_4();
 3905         break;
 3906       case 3:
 3907         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3908         emit_int8(0x66); // size prefix
 3909       case 2:
 3910         emit_int8(0x66); // size prefix
 3911       case 1:
 3912         emit_int8((unsigned char)0x90);
 3913                          // nop
 3914         break;
 3915       default:
 3916         assert(i == 0, " ");
 3917     }
 3918     return;
 3919   }
 3920 
 3921   if (UseAddressNop && VM_Version::is_zx()) {
 3922     //
 3923     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3924     //  1: 0x90
 3925     //  2: 0x66 0x90
 3926     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3927     //  4: 0x0F 0x1F 0x40 0x00
 3928     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3929     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3930     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3931     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3932     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3933     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3934     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3935 
 3936     // The rest coding is ZX specific - don't use consecutive address nops
 3937 
 3938     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3939     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3940     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3941     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3942 
 3943     while (i >= 15) {
 3944       // For ZX don't generate consecutive address nops (mix with regular nops)
 3945       i -= 15;
 3946       emit_int24(0x66, 0x66, 0x66);
 3947       addr_nop_8();
 3948       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3949     }
 3950     switch (i) {
 3951       case 14:
 3952         emit_int8(0x66); // size prefix
 3953       case 13:
 3954         emit_int8(0x66); // size prefix
 3955       case 12:
 3956         addr_nop_8();
 3957         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3958         break;
 3959       case 11:
 3960         emit_int8(0x66); // size prefix
 3961       case 10:
 3962         emit_int8(0x66); // size prefix
 3963       case 9:
 3964         emit_int8(0x66); // size prefix
 3965       case 8:
 3966         addr_nop_8();
 3967         break;
 3968       case 7:
 3969         addr_nop_7();
 3970         break;
 3971       case 6:
 3972         emit_int8(0x66); // size prefix
 3973       case 5:
 3974         addr_nop_5();
 3975         break;
 3976       case 4:
 3977         addr_nop_4();
 3978         break;
 3979       case 3:
 3980         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3981         emit_int8(0x66); // size prefix
 3982       case 2:
 3983         emit_int8(0x66); // size prefix
 3984       case 1:
 3985         emit_int8((unsigned char)0x90);
 3986                          // nop
 3987         break;
 3988       default:
 3989         assert(i == 0, " ");
 3990     }
 3991     return;
 3992   }
 3993 
 3994   // Using nops with size prefixes "0x66 0x90".
 3995   // From AMD Optimization Guide:
 3996   //  1: 0x90
 3997   //  2: 0x66 0x90
 3998   //  3: 0x66 0x66 0x90
 3999   //  4: 0x66 0x66 0x66 0x90
 4000   //  5: 0x66 0x66 0x90 0x66 0x90
 4001   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 4002   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 4003   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 4004   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 4005   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 4006   //
 4007   while (i > 12) {
 4008     i -= 4;
 4009     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 4010   }
 4011   // 1 - 12 nops
 4012   if (i > 8) {
 4013     if (i > 9) {
 4014       i -= 1;
 4015       emit_int8(0x66);
 4016     }
 4017     i -= 3;
 4018     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4019   }
 4020   // 1 - 8 nops
 4021   if (i > 4) {
 4022     if (i > 6) {
 4023       i -= 1;
 4024       emit_int8(0x66);
 4025     }
 4026     i -= 3;
 4027     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4028   }
 4029   switch (i) {
 4030     case 4:
 4031       emit_int8(0x66);
 4032     case 3:
 4033       emit_int8(0x66);
 4034     case 2:
 4035       emit_int8(0x66);
 4036     case 1:
 4037       emit_int8((unsigned char)0x90);
 4038       break;
 4039     default:
 4040       assert(i == 0, " ");
 4041   }
 4042 }
 4043 
 4044 void Assembler::notl(Register dst) {
 4045   int encode = prefix_and_encode(dst->encoding());
 4046   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 4047 }
 4048 
 4049 void Assembler::orw(Register dst, Register src) {
 4050   (void)prefix_and_encode(dst->encoding(), src->encoding());
 4051   emit_arith(0x0B, 0xC0, dst, src);
 4052 }
 4053 
 4054 void Assembler::orl(Address dst, int32_t imm32) {
 4055   InstructionMark im(this);
 4056   prefix(dst);
 4057   emit_arith_operand(0x81, rcx, dst, imm32);
 4058 }
 4059 
 4060 void Assembler::orl(Register dst, int32_t imm32) {
 4061   prefix(dst);
 4062   emit_arith(0x81, 0xC8, dst, imm32);
 4063 }
 4064 
 4065 void Assembler::orl(Register dst, Address src) {
 4066   InstructionMark im(this);
 4067   prefix(src, dst);
 4068   emit_int8(0x0B);
 4069   emit_operand(dst, src, 0);
 4070 }
 4071 
 4072 void Assembler::orl(Register dst, Register src) {
 4073   (void) prefix_and_encode(dst->encoding(), src->encoding());
 4074   emit_arith(0x0B, 0xC0, dst, src);
 4075 }
 4076 
 4077 void Assembler::orl(Address dst, Register src) {
 4078   InstructionMark im(this);
 4079   prefix(dst, src);
 4080   emit_int8(0x09);
 4081   emit_operand(src, dst, 0);
 4082 }
 4083 
 4084 void Assembler::orb(Address dst, int imm8) {
 4085   InstructionMark im(this);
 4086   prefix(dst);
 4087   emit_int8((unsigned char)0x80);
 4088   emit_operand(rcx, dst, 1);
 4089   emit_int8(imm8);
 4090 }
 4091 
 4092 void Assembler::orb(Address dst, Register src) {
 4093   InstructionMark im(this);
 4094   prefix(dst, src, true);
 4095   emit_int8(0x08);
 4096   emit_operand(src, dst, 0);
 4097 }
 4098 
 4099 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 4100   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4101   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4102   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4103   emit_int16(0x63, (0xC0 | encode));
 4104 }
 4105 
 4106 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4107   assert(UseAVX > 0, "some form of AVX must be enabled");
 4108   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4109   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4110   emit_int16(0x63, (0xC0 | encode));
 4111 }
 4112 
 4113 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 4114   assert(VM_Version::supports_sse2(), "");
 4115   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4116   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4117   emit_int16(0x6B, (0xC0 | encode));
 4118 }
 4119 
 4120 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4121   assert(UseAVX > 0, "some form of AVX must be enabled");
 4122   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4123   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4124   emit_int16(0x6B, (0xC0 | encode));
 4125 }
 4126 
 4127 void Assembler::packuswb(XMMRegister dst, Address src) {
 4128   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4129   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4130   InstructionMark im(this);
 4131   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4132   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4133   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4134   emit_int8(0x67);
 4135   emit_operand(dst, src, 0);
 4136 }
 4137 
 4138 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 4139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4140   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4141   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4142   emit_int16(0x67, (0xC0 | encode));
 4143 }
 4144 
 4145 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4146   assert(UseAVX > 0, "some form of AVX must be enabled");
 4147   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4148   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4149   emit_int16(0x67, (0xC0 | encode));
 4150 }
 4151 
 4152 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4153   assert(VM_Version::supports_sse4_1(), "");
 4154   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4155   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4156   emit_int16(0x2B, (0xC0 | encode));
 4157 }
 4158 
 4159 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4160   assert(UseAVX > 0, "some form of AVX must be enabled");
 4161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4162   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4163   emit_int16(0x2B, (0xC0 | encode));
 4164 }
 4165 
 4166 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4167   assert(VM_Version::supports_avx2(), "");
 4168   assert(vector_len != AVX_128bit, "");
 4169   // VEX.256.66.0F3A.W1 00 /r ib
 4170   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4171   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4172   emit_int24(0x00, (0xC0 | encode), imm8);
 4173 }
 4174 
 4175 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4176   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4177          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4178   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4179   attributes.set_is_evex_instruction();
 4180   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4181   emit_int16(0x36, (0xC0 | encode));
 4182 }
 4183 
 4184 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4185   assert(VM_Version::supports_avx512_vbmi(), "");
 4186   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4187   attributes.set_is_evex_instruction();
 4188   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4189   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4190 }
 4191 
 4192 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4193   assert(VM_Version::supports_avx512_vbmi(), "");
 4194   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4195   attributes.set_is_evex_instruction();
 4196   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4197   emit_int8((unsigned char)0x8D);
 4198   emit_operand(dst, src, 0);
 4199 }
 4200 
 4201 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4202   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4203          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4204          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4205   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4206   attributes.set_is_evex_instruction();
 4207   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4208   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4209 }
 4210 
 4211 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4212   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4213   // VEX.NDS.256.66.0F38.W0 36 /r
 4214   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4216   emit_int16(0x36, (0xC0 | encode));
 4217 }
 4218 
 4219 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4220   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4221   // VEX.NDS.256.66.0F38.W0 36 /r
 4222   InstructionMark im(this);
 4223   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4224   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4225   emit_int8(0x36);
 4226   emit_operand(dst, src, 0);
 4227 }
 4228 
 4229 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4230   assert(VM_Version::supports_avx2(), "");
 4231   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4232   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4233   emit_int24(0x46, (0xC0 | encode), imm8);
 4234 }
 4235 
 4236 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4237   assert(VM_Version::supports_avx(), "");
 4238   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4239   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4240   emit_int24(0x06, (0xC0 | encode), imm8);
 4241 }
 4242 
 4243 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4244   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4245   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4246   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4247   emit_int24(0x04, (0xC0 | encode), imm8);
 4248 }
 4249 
 4250 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4251   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4252   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4253   attributes.set_rex_vex_w_reverted();
 4254   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4255   emit_int24(0x05, (0xC0 | encode), imm8);
 4256 }
 4257 
 4258 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4259   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4260   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4261   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4262   emit_int24(0x01, (0xC0 | encode), imm8);
 4263 }
 4264 
 4265 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4266   assert(VM_Version::supports_evex(), "");
 4267   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4268   attributes.set_is_evex_instruction();
 4269   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4270   emit_int16(0x76, (0xC0 | encode));
 4271 }
 4272 
 4273 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4274   assert(VM_Version::supports_avx512_vbmi(), "");
 4275   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4276   attributes.set_is_evex_instruction();
 4277   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4278   emit_int16(0x7D, (0xC0 | encode));
 4279 }
 4280 
 4281 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4282   assert(VM_Version::supports_avx512_vbmi(), "");
 4283   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4284   attributes.set_is_evex_instruction();
 4285   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4286   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4287 }
 4288 
 4289 void Assembler::pause() {
 4290   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4291 }
 4292 
 4293 void Assembler::ud2() {
 4294   emit_int16(0x0F, 0x0B);
 4295 }
 4296 
 4297 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4298   assert(VM_Version::supports_sse4_2(), "");
 4299   InstructionMark im(this);
 4300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4301   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4302   emit_int8(0x61);
 4303   emit_operand(dst, src, 1);
 4304   emit_int8(imm8);
 4305 }
 4306 
 4307 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4308   assert(VM_Version::supports_sse4_2(), "");
 4309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4310   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4311   emit_int24(0x61, (0xC0 | encode), imm8);
 4312 }
 4313 
 4314 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4315 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4316   assert(VM_Version::supports_sse2(), "");
 4317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4318   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4319   emit_int16(0x74, (0xC0 | encode));
 4320 }
 4321 
 4322 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4323   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4324   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4325   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4326   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4327   emit_int16(cond_encoding, (0xC0 | encode));
 4328 }
 4329 
 4330 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4331 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4332   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4333   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4334   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4335   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4336   emit_int16(0x74, (0xC0 | encode));
 4337 }
 4338 
 4339 // In this context, kdst is written the mask used to process the equal components
 4340 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4341   assert(VM_Version::supports_avx512bw(), "");
 4342   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4343   attributes.set_is_evex_instruction();
 4344   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4345   emit_int16(0x74, (0xC0 | encode));
 4346 }
 4347 
 4348 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4349   assert(VM_Version::supports_avx512vlbw(), "");
 4350   InstructionMark im(this);
 4351   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4352   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4353   attributes.set_is_evex_instruction();
 4354   int dst_enc = kdst->encoding();
 4355   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4356   emit_int8(0x64);
 4357   emit_operand(as_Register(dst_enc), src, 0);
 4358 }
 4359 
 4360 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4361   assert(VM_Version::supports_avx512vlbw(), "");
 4362   InstructionMark im(this);
 4363   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4364   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4365   attributes.reset_is_clear_context();
 4366   attributes.set_embedded_opmask_register_specifier(mask);
 4367   attributes.set_is_evex_instruction();
 4368   int dst_enc = kdst->encoding();
 4369   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4370   emit_int8(0x64);
 4371   emit_operand(as_Register(dst_enc), src, 0);
 4372 }
 4373 
 4374 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4375   assert(VM_Version::supports_avx512vlbw(), "");
 4376   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4377   attributes.set_is_evex_instruction();
 4378   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4379   emit_int24(0x3E, (0xC0 | encode), vcc);
 4380 }
 4381 
 4382 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4383   assert(VM_Version::supports_avx512vlbw(), "");
 4384   InstructionMark im(this);
 4385   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4386   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4387   attributes.set_is_evex_instruction();
 4388   int dst_enc = kdst->encoding();
 4389   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4390   emit_int8(0x3E);
 4391   emit_operand(as_Register(dst_enc), src, 1);
 4392   emit_int8(vcc);
 4393 }
 4394 
 4395 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4396   assert(VM_Version::supports_avx512bw(), "");
 4397   InstructionMark im(this);
 4398   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4399   attributes.set_is_evex_instruction();
 4400   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4401   int dst_enc = kdst->encoding();
 4402   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4403   emit_int8(0x74);
 4404   emit_operand(as_Register(dst_enc), src, 0);
 4405 }
 4406 
 4407 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4408   assert(VM_Version::supports_avx512vlbw(), "");
 4409   InstructionMark im(this);
 4410   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4411   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4412   attributes.reset_is_clear_context();
 4413   attributes.set_embedded_opmask_register_specifier(mask);
 4414   attributes.set_is_evex_instruction();
 4415   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4416   emit_int8(0x74);
 4417   emit_operand(as_Register(kdst->encoding()), src, 0);
 4418 }
 4419 
 4420 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4421 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4422   assert(VM_Version::supports_sse2(), "");
 4423   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4424   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4425   emit_int16(0x75, (0xC0 | encode));
 4426 }
 4427 
 4428 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4429 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4430   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4431   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4432   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4433   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4434   emit_int16(0x75, (0xC0 | encode));
 4435 }
 4436 
 4437 // In this context, kdst is written the mask used to process the equal components
 4438 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4439   assert(VM_Version::supports_avx512bw(), "");
 4440   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4441   attributes.set_is_evex_instruction();
 4442   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4443   emit_int16(0x75, (0xC0 | encode));
 4444 }
 4445 
 4446 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4447   assert(VM_Version::supports_avx512bw(), "");
 4448   InstructionMark im(this);
 4449   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4450   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4451   attributes.set_is_evex_instruction();
 4452   int dst_enc = kdst->encoding();
 4453   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4454   emit_int8(0x75);
 4455   emit_operand(as_Register(dst_enc), src, 0);
 4456 }
 4457 
 4458 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4459 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4460   assert(VM_Version::supports_sse2(), "");
 4461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4462   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4463   emit_int16(0x76, (0xC0 | encode));
 4464 }
 4465 
 4466 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4467 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4468   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4469   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4470   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4471   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4472   emit_int16(0x76, (0xC0 | encode));
 4473 }
 4474 
 4475 // In this context, kdst is written the mask used to process the equal components
 4476 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4477   assert(VM_Version::supports_evex(), "");
 4478   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4479   attributes.set_is_evex_instruction();
 4480   attributes.reset_is_clear_context();
 4481   attributes.set_embedded_opmask_register_specifier(mask);
 4482   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4483   emit_int16(0x76, (0xC0 | encode));
 4484 }
 4485 
 4486 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4487   assert(VM_Version::supports_evex(), "");
 4488   InstructionMark im(this);
 4489   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4490   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4491   attributes.set_is_evex_instruction();
 4492   attributes.reset_is_clear_context();
 4493   attributes.set_embedded_opmask_register_specifier(mask);
 4494   int dst_enc = kdst->encoding();
 4495   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4496   emit_int8(0x76);
 4497   emit_operand(as_Register(dst_enc), src, 0);
 4498 }
 4499 
 4500 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4501 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4502   assert(VM_Version::supports_sse4_1(), "");
 4503   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4504   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4505   emit_int16(0x29, (0xC0 | encode));
 4506 }
 4507 
 4508 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4509   assert(VM_Version::supports_evex(), "");
 4510   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4511   attributes.set_is_evex_instruction();
 4512   attributes.reset_is_clear_context();
 4513   attributes.set_embedded_opmask_register_specifier(mask);
 4514   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4515   emit_int16(0x29, (0xC0 | encode));
 4516 }
 4517 
 4518 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4519   assert(VM_Version::supports_avx(), "");
 4520   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4521   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4522   emit_int16(cond_encoding, (0xC0 | encode));
 4523 }
 4524 
 4525 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4526 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4527   assert(VM_Version::supports_avx(), "");
 4528   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4529   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4530   emit_int16(0x29, (0xC0 | encode));
 4531 }
 4532 
 4533 // In this context, kdst is written the mask used to process the equal components
 4534 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4535   assert(VM_Version::supports_evex(), "");
 4536   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4537   attributes.reset_is_clear_context();
 4538   attributes.set_is_evex_instruction();
 4539   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4540   emit_int16(0x29, (0xC0 | encode));
 4541 }
 4542 
 4543 // In this context, kdst is written the mask used to process the equal components
 4544 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4545   assert(VM_Version::supports_evex(), "");
 4546   InstructionMark im(this);
 4547   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4548   attributes.reset_is_clear_context();
 4549   attributes.set_is_evex_instruction();
 4550   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4551   int dst_enc = kdst->encoding();
 4552   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4553   emit_int8(0x29);
 4554   emit_operand(as_Register(dst_enc), src, 0);
 4555 }
 4556 
 4557 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4558   assert(VM_Version::supports_sse4_1(), "");
 4559   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4560   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4561   emit_int16(0x37, (0xC0 | encode));
 4562 }
 4563 
 4564 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4565   assert(VM_Version::supports_sse2(), "");
 4566   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4567   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4568   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4569 }
 4570 
 4571 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4572   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4573          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4574   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4575   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4576   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4577 }
 4578 
 4579 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4580   assert(VM_Version::supports_avx(), "");
 4581   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4582   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4583   emit_int16(0x50, (0xC0 | encode));
 4584 }
 4585 
 4586 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4587   assert(VM_Version::supports_avx(), "");
 4588   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4589   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4590   emit_int16(0x50, (0xC0 | encode));
 4591 }
 4592 
 4593 
 4594 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4595   assert(VM_Version::supports_sse4_1(), "");
 4596   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4597   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4598   emit_int24(0x16, (0xC0 | encode), imm8);
 4599 }
 4600 
 4601 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4602   assert(VM_Version::supports_sse4_1(), "");
 4603   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4604   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4605   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4606   emit_int8(0x16);
 4607   emit_operand(src, dst, 1);
 4608   emit_int8(imm8);
 4609 }
 4610 
 4611 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4612   assert(VM_Version::supports_sse4_1(), "");
 4613   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4614   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4615   emit_int24(0x16, (0xC0 | encode), imm8);
 4616 }
 4617 
 4618 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4619   assert(VM_Version::supports_sse4_1(), "");
 4620   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4621   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4622   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4623   emit_int8(0x16);
 4624   emit_operand(src, dst, 1);
 4625   emit_int8(imm8);
 4626 }
 4627 
 4628 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4629   assert(VM_Version::supports_sse2(), "");
 4630   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4631   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4632   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4633 }
 4634 
 4635 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4636   assert(VM_Version::supports_sse4_1(), "");
 4637   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4638   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4639   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4640   emit_int8(0x15);
 4641   emit_operand(src, dst, 1);
 4642   emit_int8(imm8);
 4643 }
 4644 
 4645 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4646   assert(VM_Version::supports_sse4_1(), "");
 4647   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4648   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4649   emit_int24(0x14, (0xC0 | encode), imm8);
 4650 }
 4651 
 4652 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4653   assert(VM_Version::supports_sse4_1(), "");
 4654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4655   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4656   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4657   emit_int8(0x14);
 4658   emit_operand(src, dst, 1);
 4659   emit_int8(imm8);
 4660 }
 4661 
 4662 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4663   assert(VM_Version::supports_sse4_1(), "");
 4664   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4665   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4666   emit_int24(0x22, (0xC0 | encode), imm8);
 4667 }
 4668 
 4669 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4670   assert(VM_Version::supports_sse4_1(), "");
 4671   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4672   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4673   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4674   emit_int8(0x22);
 4675   emit_operand(dst, src, 1);
 4676   emit_int8(imm8);
 4677 }
 4678 
 4679 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4680   assert(VM_Version::supports_avx(), "");
 4681   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4682   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4683   emit_int24(0x22, (0xC0 | encode), imm8);
 4684 }
 4685 
 4686 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4687   assert(VM_Version::supports_sse4_1(), "");
 4688   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4689   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4690   emit_int24(0x22, (0xC0 | encode), imm8);
 4691 }
 4692 
 4693 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4694   assert(VM_Version::supports_sse4_1(), "");
 4695   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4696   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4697   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4698   emit_int8(0x22);
 4699   emit_operand(dst, src, 1);
 4700   emit_int8(imm8);
 4701 }
 4702 
 4703 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4704   assert(VM_Version::supports_avx(), "");
 4705   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4706   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4707   emit_int24(0x22, (0xC0 | encode), imm8);
 4708 }
 4709 
 4710 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4711   assert(VM_Version::supports_sse2(), "");
 4712   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4713   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4714   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4715 }
 4716 
 4717 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4718   assert(VM_Version::supports_sse2(), "");
 4719   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4720   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4721   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4722   emit_int8((unsigned char)0xC4);
 4723   emit_operand(dst, src, 1);
 4724   emit_int8(imm8);
 4725 }
 4726 
 4727 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4728   assert(VM_Version::supports_avx(), "");
 4729   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4730   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4731   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4732 }
 4733 
 4734 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4735   assert(VM_Version::supports_sse4_1(), "");
 4736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4737   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4738   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4739   emit_int8(0x20);
 4740   emit_operand(dst, src, 1);
 4741   emit_int8(imm8);
 4742 }
 4743 
 4744 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4745   assert(VM_Version::supports_sse4_1(), "");
 4746   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4747   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4748   emit_int24(0x20, (0xC0 | encode), imm8);
 4749 }
 4750 
 4751 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4752   assert(VM_Version::supports_avx(), "");
 4753   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4754   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4755   emit_int24(0x20, (0xC0 | encode), imm8);
 4756 }
 4757 
 4758 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4759   assert(VM_Version::supports_sse4_1(), "");
 4760   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4761   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4762   emit_int24(0x21, (0xC0 | encode), imm8);
 4763 }
 4764 
 4765 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4766   assert(VM_Version::supports_avx(), "");
 4767   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4768   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4769   emit_int24(0x21, (0xC0 | encode), imm8);
 4770 }
 4771 
 4772 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4773   assert(VM_Version::supports_sse4_1(), "");
 4774   InstructionMark im(this);
 4775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4776   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4777   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4778   emit_int8(0x30);
 4779   emit_operand(dst, src, 0);
 4780 }
 4781 
 4782 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4783   assert(VM_Version::supports_sse4_1(), "");
 4784   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4785   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4786   emit_int16(0x30, (0xC0 | encode));
 4787 }
 4788 
 4789 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4790   assert(VM_Version::supports_sse4_1(), "");
 4791   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4792   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4793   emit_int16(0x20, (0xC0 | encode));
 4794 }
 4795 
 4796 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4797   assert(VM_Version::supports_sse4_1(), "");
 4798   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4799   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4800   emit_int16(0x35, (0xC0 | encode));
 4801 }
 4802 
 4803 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4804   assert(VM_Version::supports_sse4_1(), "");
 4805   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4806   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4807   emit_int16(0x21, (0xC0 | encode));
 4808 }
 4809 
 4810 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4811   assert(VM_Version::supports_sse4_1(), "");
 4812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4813   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4814   emit_int16(0x31, (0xC0 | encode));
 4815 }
 4816 
 4817 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4818   assert(VM_Version::supports_sse4_1(), "");
 4819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4820   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4821   emit_int16(0x22, (0xC0 | encode));
 4822 }
 4823 
 4824 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4825   assert(VM_Version::supports_sse4_1(), "");
 4826   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4827   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4828   emit_int16(0x23, (0xC0 | encode));
 4829 }
 4830 
 4831 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4832   assert(VM_Version::supports_avx(), "");
 4833   InstructionMark im(this);
 4834   assert(dst != xnoreg, "sanity");
 4835   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4836   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4837   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4838   emit_int8(0x30);
 4839   emit_operand(dst, src, 0);
 4840 }
 4841 
 4842 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4843   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4844   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4845   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4846   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4847   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4848   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4849 }
 4850 
 4851 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4852   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4853   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4854   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4855   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4856   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4857   emit_int16(0x20, (0xC0 | encode));
 4858 }
 4859 
 4860 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4861   assert(VM_Version::supports_avx512vlbw(), "");
 4862   assert(dst != xnoreg, "sanity");
 4863   InstructionMark im(this);
 4864   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4865   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4866   attributes.set_embedded_opmask_register_specifier(mask);
 4867   attributes.set_is_evex_instruction();
 4868   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4869   emit_int8(0x30);
 4870   emit_operand(dst, src, 0);
 4871 }
 4872 
 4873 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4874   assert(VM_Version::supports_avx512vl(), "");
 4875   assert(dst != xnoreg, "sanity");
 4876   InstructionMark im(this);
 4877   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4878   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4879   attributes.set_embedded_opmask_register_specifier(mask);
 4880   attributes.set_is_evex_instruction();
 4881   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4882   emit_int8(0x31);
 4883   emit_operand(dst, src, 0);
 4884 }
 4885 
 4886 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) {
 4887   evpmovzxbd(dst, k0, src, vector_len);
 4888 }
 4889 
 4890 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4891   assert(VM_Version::supports_evex(), "");
 4892   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4893   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4894   attributes.set_is_evex_instruction();
 4895   attributes.set_embedded_opmask_register_specifier(mask);
 4896   if (merge) {
 4897     attributes.reset_is_clear_context();
 4898   }
 4899   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4900   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4901 }
 4902 
 4903 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4904   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4905   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4906   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4907   emit_int16(0x35, (0xC0 | encode));
 4908 }
 4909 
 4910 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4911   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4912   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4913   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4914   emit_int16(0x31, (0xC0 | encode));
 4915 }
 4916 
 4917 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4918   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4919   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4920   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4921   emit_int16(0x32, (0xC0 | encode));
 4922 }
 4923 
 4924 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4925   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4926          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4927              VM_Version::supports_evex(), "");
 4928   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4929   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4930   emit_int16(0x21, (0xC0 | encode));
 4931 }
 4932 
 4933 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4934   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4935          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4936              VM_Version::supports_evex(), "");
 4937   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4938   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4939   emit_int16(0x22, (0xC0 | encode));
 4940 }
 4941 
 4942 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4943   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4944          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4945              VM_Version::supports_evex(), "");
 4946   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4947   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4948   emit_int16(0x23, (0xC0 | encode));
 4949 }
 4950 
 4951 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4952   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4953          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4954              VM_Version::supports_evex(), "");
 4955   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4956   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4957   emit_int16(0x24, (0xC0 | encode));
 4958 }
 4959 
 4960 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4961   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4962          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4963              VM_Version::supports_evex(), "");
 4964   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4965   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4966   emit_int16(0x25, (0xC0 | encode));
 4967 }
 4968 
 4969 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4970   assert(VM_Version::supports_avx512vlbw(), "");
 4971   assert(src != xnoreg, "sanity");
 4972   InstructionMark im(this);
 4973   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4974   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4975   attributes.set_is_evex_instruction();
 4976   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4977   emit_int8(0x30);
 4978   emit_operand(src, dst, 0);
 4979 }
 4980 
 4981 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4982   assert(VM_Version::supports_avx512vlbw(), "");
 4983   assert(src != xnoreg, "sanity");
 4984   InstructionMark im(this);
 4985   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4986   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4987   attributes.reset_is_clear_context();
 4988   attributes.set_embedded_opmask_register_specifier(mask);
 4989   attributes.set_is_evex_instruction();
 4990   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4991   emit_int8(0x30);
 4992   emit_operand(src, dst, 0);
 4993 }
 4994 
 4995 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4996   assert(VM_Version::supports_evex(), "");
 4997   assert(src != xnoreg, "sanity");
 4998   InstructionMark im(this);
 4999   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5000   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 5001   attributes.set_is_evex_instruction();
 5002   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5003   emit_int8(0x31);
 5004   emit_operand(src, dst, 0);
 5005 }
 5006 
 5007 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 5008   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5009   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5010   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 5011   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5012   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5013   emit_int16(0x33, (0xC0 | encode));
 5014 }
 5015 
 5016 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 5017   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5018   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5019   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 5020   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5021   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5022   emit_int16(0x34, (0xC0 | encode));
 5023 }
 5024 
 5025 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 5026   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5027   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5028   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5029   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5030 }
 5031 
 5032 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5033   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5034     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5035     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 5036   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5037   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5038   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5039 }
 5040 
 5041 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5042 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5043        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5044        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5045   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5046   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5047   emit_int16(0x04, (0xC0 | encode));
 5048 }
 5049 
 5050 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5051   evpmadd52luq(dst, k0, src1, src2, false, vector_len);
 5052 }
 5053 
 5054 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5055   assert(VM_Version::supports_avx512ifma(), "");
 5056   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5057   attributes.set_is_evex_instruction();
 5058   attributes.set_embedded_opmask_register_specifier(mask);
 5059   if (merge) {
 5060     attributes.reset_is_clear_context();
 5061   }
 5062 
 5063   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5064   emit_int16((unsigned char)0xB4, (0xC0 | encode));
 5065 }
 5066 
 5067 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5068   evpmadd52huq(dst, k0, src1, src2, false, vector_len);
 5069 }
 5070 
 5071 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5072   assert(VM_Version::supports_avx512ifma(), "");
 5073   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5074   attributes.set_is_evex_instruction();
 5075   attributes.set_embedded_opmask_register_specifier(mask);
 5076   if (merge) {
 5077     attributes.reset_is_clear_context();
 5078   }
 5079 
 5080   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5081   emit_int16((unsigned char)0xB5, (0xC0 | encode));
 5082 }
 5083 
 5084 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5085   assert(VM_Version::supports_evex(), "");
 5086   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 5087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5088   attributes.set_is_evex_instruction();
 5089   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5090   emit_int16(0x52, (0xC0 | encode));
 5091 }
 5092 
 5093 // generic
 5094 void Assembler::pop(Register dst) {
 5095   int encode = prefix_and_encode(dst->encoding());
 5096   emit_int8(0x58 | encode);
 5097 }
 5098 
 5099 void Assembler::popcntl(Register dst, Address src) {
 5100   assert(VM_Version::supports_popcnt(), "must support");
 5101   InstructionMark im(this);
 5102   emit_int8((unsigned char)0xF3);
 5103   prefix(src, dst);
 5104   emit_int16(0x0F, (unsigned char)0xB8);
 5105   emit_operand(dst, src, 0);
 5106 }
 5107 
 5108 void Assembler::popcntl(Register dst, Register src) {
 5109   assert(VM_Version::supports_popcnt(), "must support");
 5110   emit_int8((unsigned char)0xF3);
 5111   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5112   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 5113 }
 5114 
 5115 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5116   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5117   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5118   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5119   attributes.set_embedded_opmask_register_specifier(mask);
 5120   attributes.set_is_evex_instruction();
 5121   if (merge) {
 5122     attributes.reset_is_clear_context();
 5123   }
 5124   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5125   emit_int16(0x54, (0xC0 | encode));
 5126 }
 5127 
 5128 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5129   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5130   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5131   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5132   attributes.set_is_evex_instruction();
 5133   attributes.set_embedded_opmask_register_specifier(mask);
 5134   if (merge) {
 5135     attributes.reset_is_clear_context();
 5136   }
 5137   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5138   emit_int16(0x54, (0xC0 | encode));
 5139 }
 5140 
 5141 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5142   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5143   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5144   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5145   attributes.set_is_evex_instruction();
 5146   attributes.set_embedded_opmask_register_specifier(mask);
 5147   if (merge) {
 5148     attributes.reset_is_clear_context();
 5149   }
 5150   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5151   emit_int16(0x55, (0xC0 | encode));
 5152 }
 5153 
 5154 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5155   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5156   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5157   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5158   attributes.set_is_evex_instruction();
 5159   attributes.set_embedded_opmask_register_specifier(mask);
 5160   if (merge) {
 5161     attributes.reset_is_clear_context();
 5162   }
 5163   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5164   emit_int16(0x55, (0xC0 | encode));
 5165 }
 5166 
 5167 void Assembler::popf() {
 5168   emit_int8((unsigned char)0x9D);
 5169 }
 5170 
 5171 #ifndef _LP64 // no 32bit push/pop on amd64
 5172 void Assembler::popl(Address dst) {
 5173   // NOTE: this will adjust stack by 8byte on 64bits
 5174   InstructionMark im(this);
 5175   prefix(dst);
 5176   emit_int8((unsigned char)0x8F);
 5177   emit_operand(rax, dst, 0);
 5178 }
 5179 #endif
 5180 
 5181 void Assembler::prefetchnta(Address src) {
 5182   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5183   InstructionMark im(this);
 5184   prefix(src);
 5185   emit_int16(0x0F, 0x18);
 5186   emit_operand(rax, src, 0); // 0, src
 5187 }
 5188 
 5189 void Assembler::prefetchr(Address src) {
 5190   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5191   InstructionMark im(this);
 5192   prefix(src);
 5193   emit_int16(0x0F, 0x0D);
 5194   emit_operand(rax, src, 0); // 0, src
 5195 }
 5196 
 5197 void Assembler::prefetcht0(Address src) {
 5198   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5199   InstructionMark im(this);
 5200   prefix(src);
 5201   emit_int16(0x0F, 0x18);
 5202   emit_operand(rcx, src, 0); // 1, src
 5203 }
 5204 
 5205 void Assembler::prefetcht1(Address src) {
 5206   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5207   InstructionMark im(this);
 5208   prefix(src);
 5209   emit_int16(0x0F, 0x18);
 5210   emit_operand(rdx, src, 0); // 2, src
 5211 }
 5212 
 5213 void Assembler::prefetcht2(Address src) {
 5214   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5215   InstructionMark im(this);
 5216   prefix(src);
 5217   emit_int16(0x0F, 0x18);
 5218   emit_operand(rbx, src, 0); // 3, src
 5219 }
 5220 
 5221 void Assembler::prefetchw(Address src) {
 5222   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5223   InstructionMark im(this);
 5224   prefix(src);
 5225   emit_int16(0x0F, 0x0D);
 5226   emit_operand(rcx, src, 0); // 1, src
 5227 }
 5228 
 5229 void Assembler::prefix(Prefix p) {
 5230   emit_int8(p);
 5231 }
 5232 
 5233 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5234   assert(VM_Version::supports_ssse3(), "");
 5235   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5236   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5237   emit_int16(0x00, (0xC0 | encode));
 5238 }
 5239 
 5240 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 5241   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 5242   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5243   attributes.set_is_evex_instruction();
 5244   attributes.set_embedded_opmask_register_specifier(mask);
 5245   if (merge) {
 5246     attributes.reset_is_clear_context();
 5247   }
 5248   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5249   emit_int16(0x00, (0xC0 | encode));
 5250 }
 5251 
 5252 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5253   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5254          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5255          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5256   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5257   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5258   emit_int16(0x00, (0xC0 | encode));
 5259 }
 5260 
 5261 void Assembler::pshufb(XMMRegister dst, Address src) {
 5262   assert(VM_Version::supports_ssse3(), "");
 5263   InstructionMark im(this);
 5264   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5265   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5266   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5267   emit_int8(0x00);
 5268   emit_operand(dst, src, 0);
 5269 }
 5270 
 5271 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5272   assert(isByte(mode), "invalid value");
 5273   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5274   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5275   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5276   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5277   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5278 }
 5279 
 5280 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5281   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5282          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5283          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5285   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5286   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5287   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5288 }
 5289 
 5290 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5291   assert(isByte(mode), "invalid value");
 5292   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5293   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5294   InstructionMark im(this);
 5295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5296   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5297   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5298   emit_int8(0x70);
 5299   emit_operand(dst, src, 1);
 5300   emit_int8(mode & 0xFF);
 5301 }
 5302 
 5303 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5304   assert(isByte(mode), "invalid value");
 5305   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5307   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5308   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5309 }
 5310 
 5311 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5312     assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5313             (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5314             (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
 5315     NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5316     InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5317     int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5318     emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5319 }
 5320 
 5321 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5322   assert(isByte(mode), "invalid value");
 5323   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5324   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5325   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5326   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5327 }
 5328 
 5329 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5330   assert(isByte(mode), "invalid value");
 5331   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5332   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5333   InstructionMark im(this);
 5334   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5335   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5336   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5337   emit_int8(0x70);
 5338   emit_operand(dst, src, 1);
 5339   emit_int8(mode & 0xFF);
 5340 }
 5341 
 5342 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5343     assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5344             (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5345             (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
 5346     NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5347     InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5348     int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5349     emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5350 }
 5351 
 5352 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5353   assert(VM_Version::supports_evex(), "requires EVEX support");
 5354   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5355   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5356   attributes.set_is_evex_instruction();
 5357   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5358   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5359 }
 5360 
 5361 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5362   assert(isByte(imm8), "invalid value");
 5363   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5364   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5365   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5366   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5367 }
 5368 
 5369 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5370   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5371   attributes.set_rex_vex_w_reverted();
 5372   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5373   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5374 }
 5375 
 5376 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
 5377   assert(isByte(imm8), "invalid value");
 5378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5379   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5380   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5381   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5382 }
 5383 
 5384 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5385   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5386   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5387   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5388 }
 5389 
 5390 void Assembler::psrldq(XMMRegister dst, int shift) {
 5391   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5392   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5393   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5394   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5395   emit_int24(0x73, (0xC0 | encode), shift);
 5396 }
 5397 
 5398 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5399   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5400          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5401          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5402   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5403   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5404   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5405 }
 5406 
 5407 void Assembler::pslldq(XMMRegister dst, int shift) {
 5408   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5409   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5411   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5412   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5413   emit_int24(0x73, (0xC0 | encode), shift);
 5414 }
 5415 
 5416 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5417   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5418          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5419          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5420   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5421   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5422   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5423 }
 5424 
 5425 void Assembler::ptest(XMMRegister dst, Address src) {
 5426   assert(VM_Version::supports_sse4_1(), "");
 5427   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5428   InstructionMark im(this);
 5429   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5430   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5431   emit_int8(0x17);
 5432   emit_operand(dst, src, 0);
 5433 }
 5434 
 5435 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5436   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5437   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5438   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5439   emit_int8(0x17);
 5440   emit_int8((0xC0 | encode));
 5441 }
 5442 
 5443 void Assembler::vptest(XMMRegister dst, Address src) {
 5444   assert(VM_Version::supports_avx(), "");
 5445   InstructionMark im(this);
 5446   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5447   assert(dst != xnoreg, "sanity");
 5448   // swap src<->dst for encoding
 5449   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5450   emit_int8(0x17);
 5451   emit_operand(dst, src, 0);
 5452 }
 5453 
 5454 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5455   assert(VM_Version::supports_avx(), "");
 5456   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5457   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5458   emit_int16(0x17, (0xC0 | encode));
 5459 }
 5460 
 5461 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5462   assert(VM_Version::supports_avx(), "");
 5463   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5465   emit_int16(0x17, (0xC0 | encode));
 5466 }
 5467 
 5468 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) {
 5469   assert(VM_Version::supports_avx(), "");
 5470   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5472   emit_int16(0x0E, (0xC0 | encode));
 5473 }
 5474 
 5475 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5476   assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), "");
 5477   // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
 5478   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5479   attributes.set_is_evex_instruction();
 5480   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5481   emit_int16(0x26, (0xC0 | encode));
 5482 }
 5483 
 5484 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5485   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
 5486   // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
 5487   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5488   attributes.set_is_evex_instruction();
 5489   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5490   emit_int16(0x27, (0xC0 | encode));
 5491 }
 5492 
 5493 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5494   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
 5495   // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r
 5496   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5497   attributes.set_is_evex_instruction();
 5498   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5499   emit_int16(0x27, (0xC0 | encode));
 5500 }
 5501 
 5502 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5503   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5504   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5505   InstructionMark im(this);
 5506   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5507   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5508   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5509   emit_int8(0x60);
 5510   emit_operand(dst, src, 0);
 5511 }
 5512 
 5513 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5517   emit_int16(0x60, (0xC0 | encode));
 5518 }
 5519 
 5520 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5522   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5523   InstructionMark im(this);
 5524   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5525   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5526   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5527   emit_int8(0x62);
 5528   emit_operand(dst, src, 0);
 5529 }
 5530 
 5531 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5532   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5535   emit_int16(0x62, (0xC0 | encode));
 5536 }
 5537 
 5538 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5539   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5540   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5541   attributes.set_rex_vex_w_reverted();
 5542   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5543   emit_int16(0x6C, (0xC0 | encode));
 5544 }
 5545 
 5546 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5547   evpunpcklqdq(dst, k0, src1, src2, false, vector_len);
 5548 }
 5549 
 5550 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5551   assert(VM_Version::supports_evex(), "requires AVX512F");
 5552   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 5553   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5554   attributes.set_is_evex_instruction();
 5555   attributes.set_embedded_opmask_register_specifier(mask);
 5556   if (merge) {
 5557     attributes.reset_is_clear_context();
 5558   }
 5559 
 5560   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5561   emit_int16(0x6C, (0xC0 | encode));
 5562 }
 5563 
 5564 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5565   evpunpckhqdq(dst, k0, src1, src2, false, vector_len);
 5566 }
 5567 
 5568 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5569   assert(VM_Version::supports_evex(), "requires AVX512F");
 5570   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 5571   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5572   attributes.set_is_evex_instruction();
 5573   attributes.set_embedded_opmask_register_specifier(mask);
 5574   if (merge) {
 5575     attributes.reset_is_clear_context();
 5576   }
 5577 
 5578   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5579   emit_int16(0x6D, (0xC0 | encode));
 5580 }
 5581 
 5582 void Assembler::push(int32_t imm32) {
 5583   // in 64bits we push 64bits onto the stack but only
 5584   // take a 32bit immediate
 5585   emit_int8(0x68);
 5586   emit_int32(imm32);
 5587 }
 5588 
 5589 void Assembler::push(Register src) {
 5590   int encode = prefix_and_encode(src->encoding());
 5591   emit_int8(0x50 | encode);
 5592 }
 5593 
 5594 void Assembler::pushf() {
 5595   emit_int8((unsigned char)0x9C);
 5596 }
 5597 
 5598 #ifndef _LP64 // no 32bit push/pop on amd64
 5599 void Assembler::pushl(Address src) {
 5600   // Note this will push 64bit on 64bit
 5601   InstructionMark im(this);
 5602   prefix(src);
 5603   emit_int8((unsigned char)0xFF);
 5604   emit_operand(rsi, src, 0);
 5605 }
 5606 #endif
 5607 
 5608 void Assembler::rcll(Register dst, int imm8) {
 5609   assert(isShiftCount(imm8), "illegal shift count");
 5610   int encode = prefix_and_encode(dst->encoding());
 5611   if (imm8 == 1) {
 5612     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5613   } else {
 5614     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5615   }
 5616 }
 5617 
 5618 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5619   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5620   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5621   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5622   emit_int16(0x53, (0xC0 | encode));
 5623 }
 5624 
 5625 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5626   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5628   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5629   emit_int16(0x53, (0xC0 | encode));
 5630 }
 5631 
 5632 void Assembler::rdtsc() {
 5633   emit_int16(0x0F, 0x31);
 5634 }
 5635 
 5636 void Assembler::rdtscp() {
 5637   emit_int24(0x0F, 0x01, (unsigned char)0xF9);
 5638 }
 5639 
 5640 // copies data from [esi] to [edi] using rcx pointer sized words
 5641 // generic
 5642 void Assembler::rep_mov() {
 5643   // REP
 5644   // MOVSQ
 5645   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5646   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5647 }
 5648 
 5649 // sets rcx bytes with rax, value at [edi]
 5650 void Assembler::rep_stosb() {
 5651   // REP
 5652   // STOSB
 5653   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5654   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5655 }
 5656 
 5657 // sets rcx pointer sized words with rax, value at [edi]
 5658 // generic
 5659 void Assembler::rep_stos() {
 5660   // REP
 5661   // LP64:STOSQ, LP32:STOSD
 5662   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5663   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5664 }
 5665 
 5666 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5667 // generic
 5668 void Assembler::repne_scan() { // repne_scan
 5669   // SCASQ
 5670   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5671   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5672 }
 5673 
 5674 #ifdef _LP64
 5675 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5676 // generic
 5677 void Assembler::repne_scanl() { // repne_scan
 5678   // SCASL
 5679   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5680 }
 5681 #endif
 5682 
 5683 void Assembler::ret(int imm16) {
 5684   if (imm16 == 0) {
 5685     emit_int8((unsigned char)0xC3);
 5686   } else {
 5687     emit_int8((unsigned char)0xC2);
 5688     emit_int16(imm16);
 5689   }
 5690 }
 5691 
 5692 void Assembler::roll(Register dst, int imm8) {
 5693   assert(isShiftCount(imm8), "illegal shift count");
 5694   int encode = prefix_and_encode(dst->encoding());
 5695   if (imm8 == 1) {
 5696     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5697   } else {
 5698     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5699   }
 5700 }
 5701 
 5702 void Assembler::roll(Register dst) {
 5703   int encode = prefix_and_encode(dst->encoding());
 5704   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5705 }
 5706 
 5707 void Assembler::rorl(Register dst, int imm8) {
 5708   assert(isShiftCount(imm8), "illegal shift count");
 5709   int encode = prefix_and_encode(dst->encoding());
 5710   if (imm8 == 1) {
 5711     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5712   } else {
 5713     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5714   }
 5715 }
 5716 
 5717 void Assembler::rorl(Register dst) {
 5718   int encode = prefix_and_encode(dst->encoding());
 5719   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5720 }
 5721 
 5722 #ifdef _LP64
 5723 void Assembler::rorq(Register dst) {
 5724   int encode = prefixq_and_encode(dst->encoding());
 5725   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5726 }
 5727 
 5728 void Assembler::rorq(Register dst, int imm8) {
 5729   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5730   int encode = prefixq_and_encode(dst->encoding());
 5731   if (imm8 == 1) {
 5732     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5733   } else {
 5734     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5735   }
 5736 }
 5737 
 5738 void Assembler::rolq(Register dst) {
 5739   int encode = prefixq_and_encode(dst->encoding());
 5740   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5741 }
 5742 
 5743 void Assembler::rolq(Register dst, int imm8) {
 5744   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5745   int encode = prefixq_and_encode(dst->encoding());
 5746   if (imm8 == 1) {
 5747     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5748   } else {
 5749     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5750   }
 5751 }
 5752 #endif
 5753 
 5754 void Assembler::sahf() {
 5755 #ifdef _LP64
 5756   // Not supported in 64bit mode
 5757   ShouldNotReachHere();
 5758 #endif
 5759   emit_int8((unsigned char)0x9E);
 5760 }
 5761 
 5762 void Assembler::sall(Address dst, int imm8) {
 5763   InstructionMark im(this);
 5764   assert(isShiftCount(imm8), "illegal shift count");
 5765   prefix(dst);
 5766   if (imm8 == 1) {
 5767     emit_int8((unsigned char)0xD1);
 5768     emit_operand(as_Register(4), dst, 0);
 5769   }
 5770   else {
 5771     emit_int8((unsigned char)0xC1);
 5772     emit_operand(as_Register(4), dst, 1);
 5773     emit_int8(imm8);
 5774   }
 5775 }
 5776 
 5777 void Assembler::sall(Address dst) {
 5778   InstructionMark im(this);
 5779   prefix(dst);
 5780   emit_int8((unsigned char)0xD3);
 5781   emit_operand(as_Register(4), dst, 0);
 5782 }
 5783 
 5784 void Assembler::sall(Register dst, int imm8) {
 5785   assert(isShiftCount(imm8), "illegal shift count");
 5786   int encode = prefix_and_encode(dst->encoding());
 5787   if (imm8 == 1) {
 5788     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5789   } else {
 5790     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5791   }
 5792 }
 5793 
 5794 void Assembler::sall(Register dst) {
 5795   int encode = prefix_and_encode(dst->encoding());
 5796   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5797 }
 5798 
 5799 void Assembler::sarl(Address dst, int imm8) {
 5800   assert(isShiftCount(imm8), "illegal shift count");
 5801   InstructionMark im(this);
 5802   prefix(dst);
 5803   if (imm8 == 1) {
 5804     emit_int8((unsigned char)0xD1);
 5805     emit_operand(as_Register(7), dst, 0);
 5806   }
 5807   else {
 5808     emit_int8((unsigned char)0xC1);
 5809     emit_operand(as_Register(7), dst, 1);
 5810     emit_int8(imm8);
 5811   }
 5812 }
 5813 
 5814 void Assembler::sarl(Address dst) {
 5815   InstructionMark im(this);
 5816   prefix(dst);
 5817   emit_int8((unsigned char)0xD3);
 5818   emit_operand(as_Register(7), dst, 0);
 5819 }
 5820 
 5821 void Assembler::sarl(Register dst, int imm8) {
 5822   int encode = prefix_and_encode(dst->encoding());
 5823   assert(isShiftCount(imm8), "illegal shift count");
 5824   if (imm8 == 1) {
 5825     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5826   } else {
 5827     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5828   }
 5829 }
 5830 
 5831 void Assembler::sarl(Register dst) {
 5832   int encode = prefix_and_encode(dst->encoding());
 5833   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5834 }
 5835 
 5836 void Assembler::sbbl(Address dst, int32_t imm32) {
 5837   InstructionMark im(this);
 5838   prefix(dst);
 5839   emit_arith_operand(0x81, rbx, dst, imm32);
 5840 }
 5841 
 5842 void Assembler::sbbl(Register dst, int32_t imm32) {
 5843   prefix(dst);
 5844   emit_arith(0x81, 0xD8, dst, imm32);
 5845 }
 5846 
 5847 
 5848 void Assembler::sbbl(Register dst, Address src) {
 5849   InstructionMark im(this);
 5850   prefix(src, dst);
 5851   emit_int8(0x1B);
 5852   emit_operand(dst, src, 0);
 5853 }
 5854 
 5855 void Assembler::sbbl(Register dst, Register src) {
 5856   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5857   emit_arith(0x1B, 0xC0, dst, src);
 5858 }
 5859 
 5860 void Assembler::setb(Condition cc, Register dst) {
 5861   assert(0 <= cc && cc < 16, "illegal cc");
 5862   int encode = prefix_and_encode(dst->encoding(), true);
 5863   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5864 }
 5865 
 5866 void Assembler::sete(Register dst) {
 5867   int encode = prefix_and_encode(dst->encoding(), true);
 5868   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5869 }
 5870 
 5871 void Assembler::setl(Register dst) {
 5872   int encode = prefix_and_encode(dst->encoding(), true);
 5873   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5874 }
 5875 
 5876 void Assembler::setne(Register dst) {
 5877   int encode = prefix_and_encode(dst->encoding(), true);
 5878   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5879 }
 5880 
 5881 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5882   assert(VM_Version::supports_ssse3(), "");
 5883   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5884   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5885   emit_int24(0x0F, (0xC0 | encode), imm8);
 5886 }
 5887 
 5888 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5889   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5890          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5891          0, "");
 5892   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5893   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5894   emit_int24(0x0F, (0xC0 | encode), imm8);
 5895 }
 5896 
 5897 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5898   assert(VM_Version::supports_evex(), "");
 5899   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5900   attributes.set_is_evex_instruction();
 5901   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5902   emit_int24(0x3, (0xC0 | encode), imm8);
 5903 }
 5904 
 5905 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5906   assert(VM_Version::supports_sse4_1(), "");
 5907   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5908   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5909   emit_int24(0x0E, (0xC0 | encode), imm8);
 5910 }
 5911 
 5912 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5913   assert(VM_Version::supports_sha(), "");
 5914   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5915   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5916 }
 5917 
 5918 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5919   assert(VM_Version::supports_sha(), "");
 5920   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5921   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5922 }
 5923 
 5924 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5925   assert(VM_Version::supports_sha(), "");
 5926   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5927   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5928 }
 5929 
 5930 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5931   assert(VM_Version::supports_sha(), "");
 5932   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5933   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5934 }
 5935 
 5936 // xmm0 is implicit additional source to this instruction.
 5937 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5938   assert(VM_Version::supports_sha(), "");
 5939   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5940   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5941 }
 5942 
 5943 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5944   assert(VM_Version::supports_sha(), "");
 5945   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5946   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5947 }
 5948 
 5949 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5950   assert(VM_Version::supports_sha(), "");
 5951   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5952   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5953 }
 5954 
 5955 
 5956 void Assembler::shll(Register dst, int imm8) {
 5957   assert(isShiftCount(imm8), "illegal shift count");
 5958   int encode = prefix_and_encode(dst->encoding());
 5959   if (imm8 == 1 ) {
 5960     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5961   } else {
 5962     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5963   }
 5964 }
 5965 
 5966 void Assembler::shll(Register dst) {
 5967   int encode = prefix_and_encode(dst->encoding());
 5968   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5969 }
 5970 
 5971 void Assembler::shrl(Register dst, int imm8) {
 5972   assert(isShiftCount(imm8), "illegal shift count");
 5973   int encode = prefix_and_encode(dst->encoding());
 5974   if (imm8 == 1) {
 5975     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5976   }
 5977   else {
 5978     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5979   }
 5980 }
 5981 
 5982 void Assembler::shrl(Register dst) {
 5983   int encode = prefix_and_encode(dst->encoding());
 5984   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5985 }
 5986 
 5987 void Assembler::shrl(Address dst) {
 5988   InstructionMark im(this);
 5989   prefix(dst);
 5990   emit_int8((unsigned char)0xD3);
 5991   emit_operand(as_Register(5), dst, 0);
 5992 }
 5993 
 5994 void Assembler::shrl(Address dst, int imm8) {
 5995   InstructionMark im(this);
 5996   assert(isShiftCount(imm8), "illegal shift count");
 5997   prefix(dst);
 5998   if (imm8 == 1) {
 5999     emit_int8((unsigned char)0xD1);
 6000     emit_operand(as_Register(5), dst, 0);
 6001   }
 6002   else {
 6003     emit_int8((unsigned char)0xC1);
 6004     emit_operand(as_Register(5), dst, 1);
 6005     emit_int8(imm8);
 6006   }
 6007 }
 6008 
 6009 
 6010 void Assembler::shldl(Register dst, Register src) {
 6011   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6012   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 6013 }
 6014 
 6015 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 6016   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6017   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 6018 }
 6019 
 6020 void Assembler::shrdl(Register dst, Register src) {
 6021   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6022   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 6023 }
 6024 
 6025 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 6026   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6027   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 6028 }
 6029 
 6030 #ifdef _LP64
 6031 void Assembler::shldq(Register dst, Register src, int8_t imm8) {
 6032   int encode = prefixq_and_encode(src->encoding(), dst->encoding());
 6033   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 6034 }
 6035 
 6036 void Assembler::shrdq(Register dst, Register src, int8_t imm8) {
 6037   int encode = prefixq_and_encode(src->encoding(), dst->encoding());
 6038   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 6039 }
 6040 #endif
 6041 
 6042 // copies a single word from [esi] to [edi]
 6043 void Assembler::smovl() {
 6044   emit_int8((unsigned char)0xA5);
 6045 }
 6046 
 6047 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 6048   assert(VM_Version::supports_sse4_1(), "");
 6049   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6050   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6051   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 6052 }
 6053 
 6054 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 6055   assert(VM_Version::supports_sse4_1(), "");
 6056   InstructionMark im(this);
 6057   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6058   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6059   emit_int8(0x0B);
 6060   emit_operand(dst, src, 1);
 6061   emit_int8((unsigned char)rmode);
 6062 }
 6063 
 6064 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 6065   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6066   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6067   attributes.set_rex_vex_w_reverted();
 6068   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6069   emit_int16(0x51, (0xC0 | encode));
 6070 }
 6071 
 6072 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 6073   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6074   InstructionMark im(this);
 6075   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6076   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6077   attributes.set_rex_vex_w_reverted();
 6078   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6079   emit_int8(0x51);
 6080   emit_operand(dst, src, 0);
 6081 }
 6082 
 6083 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 6084   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6085   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6086   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6087   emit_int16(0x51, (0xC0 | encode));
 6088 }
 6089 
 6090 void Assembler::std() {
 6091   emit_int8((unsigned char)0xFD);
 6092 }
 6093 
 6094 void Assembler::sqrtss(XMMRegister dst, Address src) {
 6095   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6096   InstructionMark im(this);
 6097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6098   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6099   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6100   emit_int8(0x51);
 6101   emit_operand(dst, src, 0);
 6102 }
 6103 
 6104 void Assembler::stmxcsr( Address dst) {
 6105   if (UseAVX > 0 ) {
 6106     assert(VM_Version::supports_avx(), "");
 6107     InstructionMark im(this);
 6108     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6109     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6110     emit_int8((unsigned char)0xAE);
 6111     emit_operand(as_Register(3), dst, 0);
 6112   } else {
 6113     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6114     InstructionMark im(this);
 6115     prefix(dst);
 6116     emit_int16(0x0F, (unsigned char)0xAE);
 6117     emit_operand(as_Register(3), dst, 0);
 6118   }
 6119 }
 6120 
 6121 void Assembler::subl(Address dst, int32_t imm32) {
 6122   InstructionMark im(this);
 6123   prefix(dst);
 6124   emit_arith_operand(0x81, rbp, dst, imm32);
 6125 }
 6126 
 6127 void Assembler::subl(Address dst, Register src) {
 6128   InstructionMark im(this);
 6129   prefix(dst, src);
 6130   emit_int8(0x29);
 6131   emit_operand(src, dst, 0);
 6132 }
 6133 
 6134 void Assembler::subl(Register dst, int32_t imm32) {
 6135   prefix(dst);
 6136   emit_arith(0x81, 0xE8, dst, imm32);
 6137 }
 6138 
 6139 // Force generation of a 4 byte immediate value even if it fits into 8bit
 6140 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 6141   prefix(dst);
 6142   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 6143 }
 6144 
 6145 void Assembler::subl(Register dst, Address src) {
 6146   InstructionMark im(this);
 6147   prefix(src, dst);
 6148   emit_int8(0x2B);
 6149   emit_operand(dst, src, 0);
 6150 }
 6151 
 6152 void Assembler::subl(Register dst, Register src) {
 6153   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6154   emit_arith(0x2B, 0xC0, dst, src);
 6155 }
 6156 
 6157 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 6158   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6159   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6160   attributes.set_rex_vex_w_reverted();
 6161   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6162   emit_int16(0x5C, (0xC0 | encode));
 6163 }
 6164 
 6165 void Assembler::subsd(XMMRegister dst, Address src) {
 6166   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6167   InstructionMark im(this);
 6168   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6169   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6170   attributes.set_rex_vex_w_reverted();
 6171   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6172   emit_int8(0x5C);
 6173   emit_operand(dst, src, 0);
 6174 }
 6175 
 6176 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 6177   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6178   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 6179   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6180   emit_int16(0x5C, (0xC0 | encode));
 6181 }
 6182 
 6183 void Assembler::subss(XMMRegister dst, Address src) {
 6184   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6185   InstructionMark im(this);
 6186   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6187   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6188   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6189   emit_int8(0x5C);
 6190   emit_operand(dst, src, 0);
 6191 }
 6192 
 6193 void Assembler::testb(Register dst, int imm8) {
 6194   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 6195   if (dst == rax) {
 6196     emit_int8((unsigned char)0xA8);
 6197     emit_int8(imm8);
 6198   } else {
 6199     (void) prefix_and_encode(dst->encoding(), true);
 6200     emit_arith_b(0xF6, 0xC0, dst, imm8);
 6201   }
 6202 }
 6203 
 6204 void Assembler::testb(Address dst, int imm8) {
 6205   InstructionMark im(this);
 6206   prefix(dst);
 6207   emit_int8((unsigned char)0xF6);
 6208   emit_operand(rax, dst, 1);
 6209   emit_int8(imm8);
 6210 }
 6211 
 6212 void Assembler::testl(Address dst, int32_t imm32) {
 6213   InstructionMark im(this);
 6214   prefix(dst);
 6215   emit_int8((unsigned char)0xF7);
 6216   emit_operand(as_Register(0), dst, 4);
 6217   emit_int32(imm32);
 6218 }
 6219 
 6220 void Assembler::testl(Register dst, int32_t imm32) {
 6221   // not using emit_arith because test
 6222   // doesn't support sign-extension of
 6223   // 8bit operands
 6224   if (dst == rax) {
 6225     emit_int8((unsigned char)0xA9);
 6226     emit_int32(imm32);
 6227   } else {
 6228     int encode = dst->encoding();
 6229     encode = prefix_and_encode(encode);
 6230     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 6231     emit_int32(imm32);
 6232   }
 6233 }
 6234 
 6235 void Assembler::testl(Register dst, Register src) {
 6236   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6237   emit_arith(0x85, 0xC0, dst, src);
 6238 }
 6239 
 6240 void Assembler::testl(Register dst, Address src) {
 6241   InstructionMark im(this);
 6242   prefix(src, dst);
 6243   emit_int8((unsigned char)0x85);
 6244   emit_operand(dst, src, 0);
 6245 }
 6246 
 6247 void Assembler::tzcntl(Register dst, Register src) {
 6248   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6249   emit_int8((unsigned char)0xF3);
 6250   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6251   emit_int24(0x0F,
 6252              (unsigned char)0xBC,
 6253              0xC0 | encode);
 6254 }
 6255 
 6256 void Assembler::tzcntl(Register dst, Address src) {
 6257   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6258   InstructionMark im(this);
 6259   emit_int8((unsigned char)0xF3);
 6260   prefix(src, dst);
 6261   emit_int16(0x0F, (unsigned char)0xBC);
 6262   emit_operand(dst, src, 0);
 6263 }
 6264 
 6265 void Assembler::tzcntq(Register dst, Register src) {
 6266   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6267   emit_int8((unsigned char)0xF3);
 6268   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 6269   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 6270 }
 6271 
 6272 void Assembler::tzcntq(Register dst, Address src) {
 6273   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6274   InstructionMark im(this);
 6275   emit_int8((unsigned char)0xF3);
 6276   prefixq(src, dst);
 6277   emit_int16(0x0F, (unsigned char)0xBC);
 6278   emit_operand(dst, src, 0);
 6279 }
 6280 
 6281 void Assembler::ucomisd(XMMRegister dst, Address src) {
 6282   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6283   InstructionMark im(this);
 6284   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6285   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6286   attributes.set_rex_vex_w_reverted();
 6287   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6288   emit_int8(0x2E);
 6289   emit_operand(dst, src, 0);
 6290 }
 6291 
 6292 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6294   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6295   attributes.set_rex_vex_w_reverted();
 6296   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6297   emit_int16(0x2E, (0xC0 | encode));
 6298 }
 6299 
 6300 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6301   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6302   InstructionMark im(this);
 6303   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6304   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6305   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6306   emit_int8(0x2E);
 6307   emit_operand(dst, src, 0);
 6308 }
 6309 
 6310 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6311   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6312   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6313   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6314   emit_int16(0x2E, (0xC0 | encode));
 6315 }
 6316 
 6317 void Assembler::xabort(int8_t imm8) {
 6318   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6319 }
 6320 
 6321 void Assembler::xaddb(Address dst, Register src) {
 6322   InstructionMark im(this);
 6323   prefix(dst, src, true);
 6324   emit_int16(0x0F, (unsigned char)0xC0);
 6325   emit_operand(src, dst, 0);
 6326 }
 6327 
 6328 void Assembler::xaddw(Address dst, Register src) {
 6329   InstructionMark im(this);
 6330   emit_int8(0x66);
 6331   prefix(dst, src);
 6332   emit_int16(0x0F, (unsigned char)0xC1);
 6333   emit_operand(src, dst, 0);
 6334 }
 6335 
 6336 void Assembler::xaddl(Address dst, Register src) {
 6337   InstructionMark im(this);
 6338   prefix(dst, src);
 6339   emit_int16(0x0F, (unsigned char)0xC1);
 6340   emit_operand(src, dst, 0);
 6341 }
 6342 
 6343 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6344   InstructionMark im(this);
 6345   relocate(rtype);
 6346   if (abort.is_bound()) {
 6347     address entry = target(abort);
 6348     assert(entry != NULL, "abort entry NULL");
 6349     intptr_t offset = entry - pc();
 6350     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6351     emit_int32(offset - 6); // 2 opcode + 4 address
 6352   } else {
 6353     abort.add_patch_at(code(), locator());
 6354     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6355     emit_int32(0);
 6356   }
 6357 }
 6358 
 6359 void Assembler::xchgb(Register dst, Address src) { // xchg
 6360   InstructionMark im(this);
 6361   prefix(src, dst, true);
 6362   emit_int8((unsigned char)0x86);
 6363   emit_operand(dst, src, 0);
 6364 }
 6365 
 6366 void Assembler::xchgw(Register dst, Address src) { // xchg
 6367   InstructionMark im(this);
 6368   emit_int8(0x66);
 6369   prefix(src, dst);
 6370   emit_int8((unsigned char)0x87);
 6371   emit_operand(dst, src, 0);
 6372 }
 6373 
 6374 void Assembler::xchgl(Register dst, Address src) { // xchg
 6375   InstructionMark im(this);
 6376   prefix(src, dst);
 6377   emit_int8((unsigned char)0x87);
 6378   emit_operand(dst, src, 0);
 6379 }
 6380 
 6381 void Assembler::xchgl(Register dst, Register src) {
 6382   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6383   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6384 }
 6385 
 6386 void Assembler::xend() {
 6387   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6388 }
 6389 
 6390 void Assembler::xgetbv() {
 6391   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6392 }
 6393 
 6394 void Assembler::xorl(Address dst, int32_t imm32) {
 6395   InstructionMark im(this);
 6396   prefix(dst);
 6397   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6398 }
 6399 
 6400 void Assembler::xorl(Register dst, int32_t imm32) {
 6401   prefix(dst);
 6402   emit_arith(0x81, 0xF0, dst, imm32);
 6403 }
 6404 
 6405 void Assembler::xorl(Register dst, Address src) {
 6406   InstructionMark im(this);
 6407   prefix(src, dst);
 6408   emit_int8(0x33);
 6409   emit_operand(dst, src, 0);
 6410 }
 6411 
 6412 void Assembler::xorl(Register dst, Register src) {
 6413   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6414   emit_arith(0x33, 0xC0, dst, src);
 6415 }
 6416 
 6417 void Assembler::xorl(Address dst, Register src) {
 6418   InstructionMark im(this);
 6419   prefix(dst, src);
 6420   emit_int8(0x31);
 6421   emit_operand(src, dst, 0);
 6422 }
 6423 
 6424 void Assembler::xorb(Register dst, Address src) {
 6425   InstructionMark im(this);
 6426   prefix(src, dst);
 6427   emit_int8(0x32);
 6428   emit_operand(dst, src, 0);
 6429 }
 6430 
 6431 void Assembler::xorb(Address dst, Register src) {
 6432   InstructionMark im(this);
 6433   prefix(dst, src, true);
 6434   emit_int8(0x30);
 6435   emit_operand(src, dst, 0);
 6436 }
 6437 
 6438 void Assembler::xorw(Register dst, Register src) {
 6439   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6440   emit_arith(0x33, 0xC0, dst, src);
 6441 }
 6442 
 6443 // AVX 3-operands scalar float-point arithmetic instructions
 6444 
 6445 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6446   assert(VM_Version::supports_avx(), "");
 6447   InstructionMark im(this);
 6448   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6449   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6450   attributes.set_rex_vex_w_reverted();
 6451   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6452   emit_int8(0x58);
 6453   emit_operand(dst, src, 0);
 6454 }
 6455 
 6456 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6457   assert(VM_Version::supports_avx(), "");
 6458   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6459   attributes.set_rex_vex_w_reverted();
 6460   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6461   emit_int16(0x58, (0xC0 | encode));
 6462 }
 6463 
 6464 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6465   assert(VM_Version::supports_avx(), "");
 6466   InstructionMark im(this);
 6467   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6468   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6469   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6470   emit_int8(0x58);
 6471   emit_operand(dst, src, 0);
 6472 }
 6473 
 6474 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6475   assert(VM_Version::supports_avx(), "");
 6476   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6477   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6478   emit_int16(0x58, (0xC0 | encode));
 6479 }
 6480 
 6481 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6482   assert(VM_Version::supports_avx(), "");
 6483   InstructionMark im(this);
 6484   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6485   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6486   attributes.set_rex_vex_w_reverted();
 6487   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6488   emit_int8(0x5E);
 6489   emit_operand(dst, src, 0);
 6490 }
 6491 
 6492 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6493   assert(VM_Version::supports_avx(), "");
 6494   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6495   attributes.set_rex_vex_w_reverted();
 6496   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6497   emit_int16(0x5E, (0xC0 | encode));
 6498 }
 6499 
 6500 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6501   assert(VM_Version::supports_avx(), "");
 6502   InstructionMark im(this);
 6503   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6504   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6505   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6506   emit_int8(0x5E);
 6507   emit_operand(dst, src, 0);
 6508 }
 6509 
 6510 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6511   assert(VM_Version::supports_avx(), "");
 6512   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6513   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6514   emit_int16(0x5E, (0xC0 | encode));
 6515 }
 6516 
 6517 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6518   assert(VM_Version::supports_fma(), "");
 6519   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6520   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6521   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6522 }
 6523 
 6524 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6525   assert(VM_Version::supports_fma(), "");
 6526   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6527   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6528   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6529 }
 6530 
 6531 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6532   assert(VM_Version::supports_avx(), "");
 6533   InstructionMark im(this);
 6534   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6535   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6536   attributes.set_rex_vex_w_reverted();
 6537   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6538   emit_int8(0x59);
 6539   emit_operand(dst, src, 0);
 6540 }
 6541 
 6542 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6543   assert(VM_Version::supports_avx(), "");
 6544   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6545   attributes.set_rex_vex_w_reverted();
 6546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6547   emit_int16(0x59, (0xC0 | encode));
 6548 }
 6549 
 6550 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6551   assert(VM_Version::supports_avx(), "");
 6552   InstructionMark im(this);
 6553   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6554   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6555   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6556   emit_int8(0x59);
 6557   emit_operand(dst, src, 0);
 6558 }
 6559 
 6560 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6561   assert(VM_Version::supports_avx(), "");
 6562   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6564   emit_int16(0x59, (0xC0 | encode));
 6565 }
 6566 
 6567 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6568   assert(VM_Version::supports_avx(), "");
 6569   InstructionMark im(this);
 6570   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6571   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6572   attributes.set_rex_vex_w_reverted();
 6573   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6574   emit_int8(0x5C);
 6575   emit_operand(dst, src, 0);
 6576 }
 6577 
 6578 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6579   assert(VM_Version::supports_avx(), "");
 6580   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6581   attributes.set_rex_vex_w_reverted();
 6582   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6583   emit_int16(0x5C, (0xC0 | encode));
 6584 }
 6585 
 6586 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6587   assert(VM_Version::supports_avx(), "");
 6588   InstructionMark im(this);
 6589   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6590   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6591   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6592   emit_int8(0x5C);
 6593   emit_operand(dst, src, 0);
 6594 }
 6595 
 6596 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6597   assert(VM_Version::supports_avx(), "");
 6598   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6599   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6600   emit_int16(0x5C, (0xC0 | encode));
 6601 }
 6602 
 6603 //====================VECTOR ARITHMETIC=====================================
 6604 
 6605 // Float-point vector arithmetic
 6606 
 6607 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6608   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6609   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6610   attributes.set_rex_vex_w_reverted();
 6611   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6612   emit_int16(0x58, (0xC0 | encode));
 6613 }
 6614 
 6615 void Assembler::addpd(XMMRegister dst, Address src) {
 6616   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6617   InstructionMark im(this);
 6618   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6619   attributes.set_rex_vex_w_reverted();
 6620   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6621   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6622   emit_int8(0x58);
 6623   emit_operand(dst, src, 0);
 6624 }
 6625 
 6626 
 6627 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6628   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6629   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6630   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6631   emit_int16(0x58, (0xC0 | encode));
 6632 }
 6633 
 6634 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6635   assert(VM_Version::supports_avx(), "");
 6636   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6637   attributes.set_rex_vex_w_reverted();
 6638   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6639   emit_int16(0x58, (0xC0 | encode));
 6640 }
 6641 
 6642 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6643   assert(VM_Version::supports_avx(), "");
 6644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6645   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6646   emit_int16(0x58, (0xC0 | encode));
 6647 }
 6648 
 6649 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6650   assert(VM_Version::supports_avx(), "");
 6651   InstructionMark im(this);
 6652   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6653   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6654   attributes.set_rex_vex_w_reverted();
 6655   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6656   emit_int8(0x58);
 6657   emit_operand(dst, src, 0);
 6658 }
 6659 
 6660 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6661   assert(VM_Version::supports_avx(), "");
 6662   InstructionMark im(this);
 6663   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6664   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6665   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6666   emit_int8(0x58);
 6667   emit_operand(dst, src, 0);
 6668 }
 6669 
 6670 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6671   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6672   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6673   attributes.set_rex_vex_w_reverted();
 6674   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6675   emit_int16(0x5C, (0xC0 | encode));
 6676 }
 6677 
 6678 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6679   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6680   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6681   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6682   emit_int16(0x5C, (0xC0 | encode));
 6683 }
 6684 
 6685 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6686   assert(VM_Version::supports_avx(), "");
 6687   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6688   attributes.set_rex_vex_w_reverted();
 6689   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6690   emit_int16(0x5C, (0xC0 | encode));
 6691 }
 6692 
 6693 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6694   assert(VM_Version::supports_avx(), "");
 6695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6696   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6697   emit_int16(0x5C, (0xC0 | encode));
 6698 }
 6699 
 6700 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6701   assert(VM_Version::supports_avx(), "");
 6702   InstructionMark im(this);
 6703   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6704   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6705   attributes.set_rex_vex_w_reverted();
 6706   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6707   emit_int8(0x5C);
 6708   emit_operand(dst, src, 0);
 6709 }
 6710 
 6711 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6712   assert(VM_Version::supports_avx(), "");
 6713   InstructionMark im(this);
 6714   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6715   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6716   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6717   emit_int8(0x5C);
 6718   emit_operand(dst, src, 0);
 6719 }
 6720 
 6721 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6723   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6724   attributes.set_rex_vex_w_reverted();
 6725   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6726   emit_int16(0x59, (0xC0 | encode));
 6727 }
 6728 
 6729 void Assembler::mulpd(XMMRegister dst, Address src) {
 6730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6731   InstructionMark im(this);
 6732   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6733   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6734   attributes.set_rex_vex_w_reverted();
 6735   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6736   emit_int8(0x59);
 6737   emit_operand(dst, src, 0);
 6738 }
 6739 
 6740 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6742   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6743   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6744   emit_int16(0x59, (0xC0 | encode));
 6745 }
 6746 
 6747 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6748   assert(VM_Version::supports_avx(), "");
 6749   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6750   attributes.set_rex_vex_w_reverted();
 6751   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6752   emit_int16(0x59, (0xC0 | encode));
 6753 }
 6754 
 6755 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6756   assert(VM_Version::supports_avx(), "");
 6757   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6758   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6759   emit_int16(0x59, (0xC0 | encode));
 6760 }
 6761 
 6762 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6763   assert(VM_Version::supports_avx(), "");
 6764   InstructionMark im(this);
 6765   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6766   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6767   attributes.set_rex_vex_w_reverted();
 6768   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6769   emit_int8(0x59);
 6770   emit_operand(dst, src, 0);
 6771 }
 6772 
 6773 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6774   assert(VM_Version::supports_avx(), "");
 6775   InstructionMark im(this);
 6776   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6777   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6778   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6779   emit_int8(0x59);
 6780   emit_operand(dst, src, 0);
 6781 }
 6782 
 6783 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6784   assert(VM_Version::supports_fma(), "");
 6785   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6786   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6787   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6788 }
 6789 
 6790 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6791   assert(VM_Version::supports_fma(), "");
 6792   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6793   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6794   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6795 }
 6796 
 6797 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6798   assert(VM_Version::supports_fma(), "");
 6799   InstructionMark im(this);
 6800   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6801   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6802   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6803   emit_int8((unsigned char)0xB8);
 6804   emit_operand(dst, src2, 0);
 6805 }
 6806 
 6807 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6808   assert(VM_Version::supports_fma(), "");
 6809   InstructionMark im(this);
 6810   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6811   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6812   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6813   emit_int8((unsigned char)0xB8);
 6814   emit_operand(dst, src2, 0);
 6815 }
 6816 
 6817 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6819   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6820   attributes.set_rex_vex_w_reverted();
 6821   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6822   emit_int16(0x5E, (0xC0 | encode));
 6823 }
 6824 
 6825 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6826   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6827   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6828   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6829   emit_int16(0x5E, (0xC0 | encode));
 6830 }
 6831 
 6832 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6833   assert(VM_Version::supports_avx(), "");
 6834   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6835   attributes.set_rex_vex_w_reverted();
 6836   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6837   emit_int16(0x5E, (0xC0 | encode));
 6838 }
 6839 
 6840 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6841   assert(VM_Version::supports_avx(), "");
 6842   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6843   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6844   emit_int16(0x5E, (0xC0 | encode));
 6845 }
 6846 
 6847 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6848   assert(VM_Version::supports_avx(), "");
 6849   InstructionMark im(this);
 6850   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6851   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6852   attributes.set_rex_vex_w_reverted();
 6853   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6854   emit_int8(0x5E);
 6855   emit_operand(dst, src, 0);
 6856 }
 6857 
 6858 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6859   assert(VM_Version::supports_avx(), "");
 6860   InstructionMark im(this);
 6861   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6862   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6863   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6864   emit_int8(0x5E);
 6865   emit_operand(dst, src, 0);
 6866 }
 6867 
 6868 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6869   assert(VM_Version::supports_avx(), "");
 6870   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6871   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6872   emit_int24(0x09, (0xC0 | encode), (rmode));
 6873 }
 6874 
 6875 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6876   assert(VM_Version::supports_avx(), "");
 6877   InstructionMark im(this);
 6878   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6879   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6880   emit_int8(0x09);
 6881   emit_operand(dst, src, 1);
 6882   emit_int8((rmode));
 6883 }
 6884 
 6885 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6886   assert(VM_Version::supports_evex(), "requires EVEX support");
 6887   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6888   attributes.set_is_evex_instruction();
 6889   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6890   emit_int24(0x09, (0xC0 | encode), (rmode));
 6891 }
 6892 
 6893 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6894   assert(VM_Version::supports_evex(), "requires EVEX support");
 6895   assert(dst != xnoreg, "sanity");
 6896   InstructionMark im(this);
 6897   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6898   attributes.set_is_evex_instruction();
 6899   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6900   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6901   emit_int8(0x09);
 6902   emit_operand(dst, src, 1);
 6903   emit_int8((rmode));
 6904 }
 6905 
 6906 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6907   assert(VM_Version::supports_avx(), "");
 6908   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6909   attributes.set_rex_vex_w_reverted();
 6910   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6911   emit_int16(0x51, (0xC0 | encode));
 6912 }
 6913 
 6914 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6915   assert(VM_Version::supports_avx(), "");
 6916   InstructionMark im(this);
 6917   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6918   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6919   attributes.set_rex_vex_w_reverted();
 6920   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6921   emit_int8(0x51);
 6922   emit_operand(dst, src, 0);
 6923 }
 6924 
 6925 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6926   assert(VM_Version::supports_avx(), "");
 6927   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6928   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6929   emit_int16(0x51, (0xC0 | encode));
 6930 }
 6931 
 6932 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6933   assert(VM_Version::supports_avx(), "");
 6934   InstructionMark im(this);
 6935   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6936   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6937   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6938   emit_int8(0x51);
 6939   emit_operand(dst, src, 0);
 6940 }
 6941 
 6942 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6944   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6945   attributes.set_rex_vex_w_reverted();
 6946   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6947   emit_int16(0x54, (0xC0 | encode));
 6948 }
 6949 
 6950 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6951   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6952   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6953   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6954   emit_int16(0x54, (0xC0 | encode));
 6955 }
 6956 
 6957 void Assembler::andps(XMMRegister dst, Address src) {
 6958   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6959   InstructionMark im(this);
 6960   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6961   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6962   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6963   emit_int8(0x54);
 6964   emit_operand(dst, src, 0);
 6965 }
 6966 
 6967 void Assembler::andpd(XMMRegister dst, Address src) {
 6968   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6969   InstructionMark im(this);
 6970   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6971   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6972   attributes.set_rex_vex_w_reverted();
 6973   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6974   emit_int8(0x54);
 6975   emit_operand(dst, src, 0);
 6976 }
 6977 
 6978 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6979   assert(VM_Version::supports_avx(), "");
 6980   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6981   attributes.set_rex_vex_w_reverted();
 6982   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6983   emit_int16(0x54, (0xC0 | encode));
 6984 }
 6985 
 6986 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6987   assert(VM_Version::supports_avx(), "");
 6988   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6989   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6990   emit_int16(0x54, (0xC0 | encode));
 6991 }
 6992 
 6993 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6994   assert(VM_Version::supports_avx(), "");
 6995   InstructionMark im(this);
 6996   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6997   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6998   attributes.set_rex_vex_w_reverted();
 6999   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7000   emit_int8(0x54);
 7001   emit_operand(dst, src, 0);
 7002 }
 7003 
 7004 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7005   assert(VM_Version::supports_avx(), "");
 7006   InstructionMark im(this);
 7007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7008   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7009   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7010   emit_int8(0x54);
 7011   emit_operand(dst, src, 0);
 7012 }
 7013 
 7014 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 7015   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7016   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7017   attributes.set_rex_vex_w_reverted();
 7018   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7019   emit_int8(0x15);
 7020   emit_int8((0xC0 | encode));
 7021 }
 7022 
 7023 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 7024   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7025   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7026   attributes.set_rex_vex_w_reverted();
 7027   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7028   emit_int16(0x14, (0xC0 | encode));
 7029 }
 7030 
 7031 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 7032   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7033   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7034   attributes.set_rex_vex_w_reverted();
 7035   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7036   emit_int16(0x57, (0xC0 | encode));
 7037 }
 7038 
 7039 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 7040   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7041   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7042   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7043   emit_int16(0x57, (0xC0 | encode));
 7044 }
 7045 
 7046 void Assembler::xorpd(XMMRegister dst, Address src) {
 7047   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7048   InstructionMark im(this);
 7049   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7050   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7051   attributes.set_rex_vex_w_reverted();
 7052   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7053   emit_int8(0x57);
 7054   emit_operand(dst, src, 0);
 7055 }
 7056 
 7057 void Assembler::xorps(XMMRegister dst, Address src) {
 7058   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7059   InstructionMark im(this);
 7060   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7061   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7062   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7063   emit_int8(0x57);
 7064   emit_operand(dst, src, 0);
 7065 }
 7066 
 7067 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7068   assert(VM_Version::supports_avx(), "");
 7069   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7070   attributes.set_rex_vex_w_reverted();
 7071   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7072   emit_int16(0x57, (0xC0 | encode));
 7073 }
 7074 
 7075 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7076   assert(VM_Version::supports_avx(), "");
 7077   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7078   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7079   emit_int16(0x57, (0xC0 | encode));
 7080 }
 7081 
 7082 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7083   assert(VM_Version::supports_avx(), "");
 7084   InstructionMark im(this);
 7085   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7086   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7087   attributes.set_rex_vex_w_reverted();
 7088   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7089   emit_int8(0x57);
 7090   emit_operand(dst, src, 0);
 7091 }
 7092 
 7093 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7094   assert(VM_Version::supports_avx(), "");
 7095   InstructionMark im(this);
 7096   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7097   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7098   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7099   emit_int8(0x57);
 7100   emit_operand(dst, src, 0);
 7101 }
 7102 
 7103 // Integer vector arithmetic
 7104 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7105   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 7106          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 7107   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7108   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7109   emit_int16(0x01, (0xC0 | encode));
 7110 }
 7111 
 7112 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7113   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 7114          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 7115   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7116   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7117   emit_int16(0x02, (0xC0 | encode));
 7118 }
 7119 
 7120 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 7121   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7122   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7123   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7124   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7125 }
 7126 
 7127 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 7128   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7129   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7130   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7131   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7132 }
 7133 
 7134 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 7135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7137   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7138   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7139 }
 7140 
 7141 void Assembler::paddd(XMMRegister dst, Address src) {
 7142   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7143   InstructionMark im(this);
 7144   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7145   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7146   emit_int8((unsigned char)0xFE);
 7147   emit_operand(dst, src, 0);
 7148 }
 7149 
 7150 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 7151   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7152   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7153   attributes.set_rex_vex_w_reverted();
 7154   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7155   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7156 }
 7157 
 7158 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 7159   assert(VM_Version::supports_sse3(), "");
 7160   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7161   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7162   emit_int16(0x01, (0xC0 | encode));
 7163 }
 7164 
 7165 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 7166   assert(VM_Version::supports_sse3(), "");
 7167   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7168   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7169   emit_int16(0x02, (0xC0 | encode));
 7170 }
 7171 
 7172 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7173   assert(UseAVX > 0, "requires some form of AVX");
 7174   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7175   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7176   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7177 }
 7178 
 7179 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7180   assert(UseAVX > 0, "requires some form of AVX");
 7181   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7182   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7183   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7184 }
 7185 
 7186 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7187   assert(UseAVX > 0, "requires some form of AVX");
 7188   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7189   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7190   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7191 }
 7192 
 7193 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7194   assert(UseAVX > 0, "requires some form of AVX");
 7195   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7196   attributes.set_rex_vex_w_reverted();
 7197   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7198   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7199 }
 7200 
 7201 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7202   assert(UseAVX > 0, "requires some form of AVX");
 7203   InstructionMark im(this);
 7204   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7205   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7206   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7207   emit_int8((unsigned char)0xFC);
 7208   emit_operand(dst, src, 0);
 7209 }
 7210 
 7211 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7212   assert(UseAVX > 0, "requires some form of AVX");
 7213   InstructionMark im(this);
 7214   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7215   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7216   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7217   emit_int8((unsigned char)0xFD);
 7218   emit_operand(dst, src, 0);
 7219 }
 7220 
 7221 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7222   assert(UseAVX > 0, "requires some form of AVX");
 7223   InstructionMark im(this);
 7224   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7225   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7226   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7227   emit_int8((unsigned char)0xFE);
 7228   emit_operand(dst, src, 0);
 7229 }
 7230 
 7231 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7232   assert(UseAVX > 0, "requires some form of AVX");
 7233   InstructionMark im(this);
 7234   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7235   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7236   attributes.set_rex_vex_w_reverted();
 7237   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7238   emit_int8((unsigned char)0xD4);
 7239   emit_operand(dst, src, 0);
 7240 }
 7241 
 7242 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 7243   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7244   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7245   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7246   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7247 }
 7248 
 7249 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 7250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7252   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7253   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7254 }
 7255 
 7256 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 7257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7258   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7259   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7260 }
 7261 
 7262 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 7263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7264   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7265   attributes.set_rex_vex_w_reverted();
 7266   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7267   emit_int8((unsigned char)0xFB);
 7268   emit_int8((0xC0 | encode));
 7269 }
 7270 
 7271 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7272   assert(UseAVX > 0, "requires some form of AVX");
 7273   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7274   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7275   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 7276 }
 7277 
 7278 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7279   assert(UseAVX > 0, "requires some form of AVX");
 7280   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7281   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7282   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7283 }
 7284 
 7285 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7286   assert(UseAVX > 0, "requires some form of AVX");
 7287   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7288   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7289   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7290 }
 7291 
 7292 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7293   assert(UseAVX > 0, "requires some form of AVX");
 7294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7295   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7296   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7297 }
 7298 
 7299 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7300   assert(UseAVX > 0, "requires some form of AVX");
 7301   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7302   attributes.set_rex_vex_w_reverted();
 7303   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7304   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7305 }
 7306 
 7307 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7308   assert(UseAVX > 0, "requires some form of AVX");
 7309   InstructionMark im(this);
 7310   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7311   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7312   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7313   emit_int8((unsigned char)0xF8);
 7314   emit_operand(dst, src, 0);
 7315 }
 7316 
 7317 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7318   assert(UseAVX > 0, "requires some form of AVX");
 7319   InstructionMark im(this);
 7320   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7321   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7322   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7323   emit_int8((unsigned char)0xF9);
 7324   emit_operand(dst, src, 0);
 7325 }
 7326 
 7327 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7328   assert(UseAVX > 0, "requires some form of AVX");
 7329   InstructionMark im(this);
 7330   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7331   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7332   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7333   emit_int8((unsigned char)0xFA);
 7334   emit_operand(dst, src, 0);
 7335 }
 7336 
 7337 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7338   assert(UseAVX > 0, "requires some form of AVX");
 7339   InstructionMark im(this);
 7340   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7341   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7342   attributes.set_rex_vex_w_reverted();
 7343   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7344   emit_int8((unsigned char)0xFB);
 7345   emit_operand(dst, src, 0);
 7346 }
 7347 
 7348 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7349   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7351   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7352   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7353 }
 7354 
 7355 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7356   assert(VM_Version::supports_sse4_1(), "");
 7357   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7358   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7359   emit_int16(0x40, (0xC0 | encode));
 7360 }
 7361 
 7362 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7363   assert(VM_Version::supports_sse2(), "");
 7364   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7365   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7366   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7367 }
 7368 
 7369 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7370   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7371          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7372          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7373   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7374   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7375   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7376 }
 7377 
 7378 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7379   assert(UseAVX > 0, "requires some form of AVX");
 7380   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7381   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7382   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7383 }
 7384 
 7385 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7386   assert(UseAVX > 0, "requires some form of AVX");
 7387   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7388   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7389   emit_int16(0x40, (0xC0 | encode));
 7390 }
 7391 
 7392 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7393   assert(UseAVX > 2, "requires some form of EVEX");
 7394   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7395   attributes.set_is_evex_instruction();
 7396   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7397   emit_int16(0x40, (0xC0 | encode));
 7398 }
 7399 
 7400 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7401   assert(UseAVX > 0, "requires some form of AVX");
 7402   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7403   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7404   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7405 }
 7406 
 7407 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7408   assert(UseAVX > 0, "requires some form of AVX");
 7409   InstructionMark im(this);
 7410   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7411   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7412   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7413   emit_int8((unsigned char)0xD5);
 7414   emit_operand(dst, src, 0);
 7415 }
 7416 
 7417 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7418   assert(UseAVX > 0, "requires some form of AVX");
 7419   InstructionMark im(this);
 7420   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7421   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7422   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7423   emit_int8(0x40);
 7424   emit_operand(dst, src, 0);
 7425 }
 7426 
 7427 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7428   assert(UseAVX > 2, "requires some form of EVEX");
 7429   InstructionMark im(this);
 7430   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7431   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7432   attributes.set_is_evex_instruction();
 7433   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7434   emit_int8(0x40);
 7435   emit_operand(dst, src, 0);
 7436 }
 7437 
 7438 // Min, max
 7439 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7440   assert(VM_Version::supports_sse4_1(), "");
 7441   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7442   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7443   emit_int16(0x38, (0xC0 | encode));
 7444 }
 7445 
 7446 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7447   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7448         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7449   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7450   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7451   emit_int16(0x38, (0xC0 | encode));
 7452 }
 7453 
 7454 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7455   assert(VM_Version::supports_sse2(), "");
 7456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7457   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7458   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7459 }
 7460 
 7461 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7462   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7463         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7464   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7465   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7466   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7467 }
 7468 
 7469 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7470   assert(VM_Version::supports_sse4_1(), "");
 7471   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7472   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7473   emit_int16(0x39, (0xC0 | encode));
 7474 }
 7475 
 7476 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7477   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7478         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7479   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7480   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7481   emit_int16(0x39, (0xC0 | encode));
 7482 }
 7483 
 7484 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7485   assert(UseAVX > 2, "requires AVX512F");
 7486   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7487   attributes.set_is_evex_instruction();
 7488   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7489   emit_int16(0x39, (0xC0 | encode));
 7490 }
 7491 
 7492 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7493   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7494   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7495   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7496   emit_int16(0x5D, (0xC0 | encode));
 7497 }
 7498 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7499   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7500   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7501   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7502   emit_int16(0x5D, (0xC0 | encode));
 7503 }
 7504 
 7505 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7506   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7507   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7508   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7509   emit_int16(0x5D, (0xC0 | encode));
 7510 }
 7511 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7512   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7513   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7514   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7515   emit_int16(0x5D, (0xC0 | encode));
 7516 }
 7517 
 7518 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7519   assert(VM_Version::supports_sse4_1(), "");
 7520   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7521   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7522   emit_int16(0x3C, (0xC0 | encode));
 7523 }
 7524 
 7525 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7526   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7527         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7528   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7529   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7530   emit_int16(0x3C, (0xC0 | encode));
 7531 }
 7532 
 7533 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7534   assert(VM_Version::supports_sse2(), "");
 7535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7536   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7537   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7538 }
 7539 
 7540 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7541   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7542         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7543   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7544   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7545   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7546 }
 7547 
 7548 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7549   assert(VM_Version::supports_sse4_1(), "");
 7550   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7551   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7552   emit_int16(0x3D, (0xC0 | encode));
 7553 }
 7554 
 7555 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7556   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7557         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7558   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7559   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7560   emit_int16(0x3D, (0xC0 | encode));
 7561 }
 7562 
 7563 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7564   assert(UseAVX > 2, "requires AVX512F");
 7565   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7566   attributes.set_is_evex_instruction();
 7567   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7568   emit_int16(0x3D, (0xC0 | encode));
 7569 }
 7570 
 7571 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7572   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7573   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7574   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7575   emit_int16(0x5F, (0xC0 | encode));
 7576 }
 7577 
 7578 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7579   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7581   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7582   emit_int16(0x5F, (0xC0 | encode));
 7583 }
 7584 
 7585 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7586   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7587   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7588   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7589   emit_int16(0x5F, (0xC0 | encode));
 7590 }
 7591 
 7592 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7593   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7594   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7595   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7596   emit_int16(0x5F, (0xC0 | encode));
 7597 }
 7598 
 7599 // Shift packed integers left by specified number of bits.
 7600 void Assembler::psllw(XMMRegister dst, int shift) {
 7601   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7602   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7603   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7604   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7605   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7606 }
 7607 
 7608 void Assembler::pslld(XMMRegister dst, int shift) {
 7609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7611   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7612   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7613   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7614 }
 7615 
 7616 void Assembler::psllq(XMMRegister dst, int shift) {
 7617   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7618   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7619   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7620   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7621   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7622 }
 7623 
 7624 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7625   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7626   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7627   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7628   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7629 }
 7630 
 7631 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7632   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7633   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7634   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7635   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7636 }
 7637 
 7638 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7639   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7640   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7641   attributes.set_rex_vex_w_reverted();
 7642   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7643   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7644 }
 7645 
 7646 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7647   assert(UseAVX > 0, "requires some form of AVX");
 7648   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7649   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7650   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7651   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7652 }
 7653 
 7654 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7655   assert(UseAVX > 0, "requires some form of AVX");
 7656   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7657   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7658   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7659   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7660   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7661 }
 7662 
 7663 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7664   assert(UseAVX > 0, "requires some form of AVX");
 7665   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7666   attributes.set_rex_vex_w_reverted();
 7667   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7668   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7669   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7670 }
 7671 
 7672 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7673   assert(UseAVX > 0, "requires some form of AVX");
 7674   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7675   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7676   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7677 }
 7678 
 7679 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7680   assert(UseAVX > 0, "requires some form of AVX");
 7681   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7682   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7683   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7684 }
 7685 
 7686 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7687   assert(UseAVX > 0, "requires some form of AVX");
 7688   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7689   attributes.set_rex_vex_w_reverted();
 7690   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7691   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7692 }
 7693 
 7694 // Shift packed integers logically right by specified number of bits.
 7695 void Assembler::psrlw(XMMRegister dst, int shift) {
 7696   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7698   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7699   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7700   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7701 }
 7702 
 7703 void Assembler::psrld(XMMRegister dst, int shift) {
 7704   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7705   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7706   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7707   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7708   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7709 }
 7710 
 7711 void Assembler::psrlq(XMMRegister dst, int shift) {
 7712   // Do not confuse it with psrldq SSE2 instruction which
 7713   // shifts 128 bit value in xmm register by number of bytes.
 7714   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7715   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7716   attributes.set_rex_vex_w_reverted();
 7717   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7718   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7719   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7720 }
 7721 
 7722 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7725   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7726   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7727 }
 7728 
 7729 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7731   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7732   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7733   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7734 }
 7735 
 7736 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7737   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7738   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7739   attributes.set_rex_vex_w_reverted();
 7740   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7741   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7742 }
 7743 
 7744 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7745   assert(UseAVX > 0, "requires some form of AVX");
 7746   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7747   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7748   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7749   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7750 }
 7751 
 7752 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7753   assert(UseAVX > 0, "requires some form of AVX");
 7754   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7755   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7756   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7757   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7758 }
 7759 
 7760 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7761   assert(UseAVX > 0, "requires some form of AVX");
 7762   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7763   attributes.set_rex_vex_w_reverted();
 7764   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7765   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7766   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7767 }
 7768 
 7769 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7770   assert(UseAVX > 0, "requires some form of AVX");
 7771   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7772   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7773   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7774 }
 7775 
 7776 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7777   assert(UseAVX > 0, "requires some form of AVX");
 7778   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7779   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7780   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7781 }
 7782 
 7783 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7784   assert(UseAVX > 0, "requires some form of AVX");
 7785   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7786   attributes.set_rex_vex_w_reverted();
 7787   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7788   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7789 }
 7790 
 7791 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7792   assert(VM_Version::supports_avx512bw(), "");
 7793   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7794   attributes.set_is_evex_instruction();
 7795   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7796   emit_int16(0x10, (0xC0 | encode));
 7797 }
 7798 
 7799 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7800   assert(VM_Version::supports_avx512bw(), "");
 7801   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7802   attributes.set_is_evex_instruction();
 7803   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7804   emit_int16(0x12, (0xC0 | encode));
 7805 }
 7806 
 7807 // Shift packed integers arithmetically right by specified number of bits.
 7808 void Assembler::psraw(XMMRegister dst, int shift) {
 7809   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7810   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7811   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7812   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7813   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7814 }
 7815 
 7816 void Assembler::psrad(XMMRegister dst, int shift) {
 7817   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7818   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7819   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7820   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7821   emit_int8(0x72);
 7822   emit_int8((0xC0 | encode));
 7823   emit_int8(shift & 0xFF);
 7824 }
 7825 
 7826 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7827   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7828   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7829   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7830   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7831 }
 7832 
 7833 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7834   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7835   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7836   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7837   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7838 }
 7839 
 7840 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7841   assert(UseAVX > 0, "requires some form of AVX");
 7842   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7843   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7844   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7845   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7846 }
 7847 
 7848 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7849   assert(UseAVX > 0, "requires some form of AVX");
 7850   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7851   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7852   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7853   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7854 }
 7855 
 7856 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7857   assert(UseAVX > 0, "requires some form of AVX");
 7858   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7859   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7860   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7861 }
 7862 
 7863 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7864   assert(UseAVX > 0, "requires some form of AVX");
 7865   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7866   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7867   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7868 }
 7869 
 7870 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7871   assert(UseAVX > 2, "requires AVX512");
 7872   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7873   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7874   attributes.set_is_evex_instruction();
 7875   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7876   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7877 }
 7878 
 7879 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7880   assert(UseAVX > 2, "requires AVX512");
 7881   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7882   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7883   attributes.set_is_evex_instruction();
 7884   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7885   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7886 }
 7887 
 7888 // logical operations packed integers
 7889 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7890   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7891   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7892   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7893   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7894 }
 7895 
 7896 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7897   assert(UseAVX > 0, "requires some form of AVX");
 7898   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7899   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7900   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7901 }
 7902 
 7903 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7904   assert(UseAVX > 0, "requires some form of AVX");
 7905   InstructionMark im(this);
 7906   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7907   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7908   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7909   emit_int8((unsigned char)0xDB);
 7910   emit_operand(dst, src, 0);
 7911 }
 7912 
 7913 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7914   evpandq(dst, k0, nds, src, false, vector_len);
 7915 }
 7916 
 7917 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7918   evpandq(dst, k0, nds, src, false, vector_len);
 7919 }
 7920 
 7921 //Variable Shift packed integers logically left.
 7922 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7923   assert(UseAVX > 1, "requires AVX2");
 7924   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7925   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7926   emit_int16(0x47, (0xC0 | encode));
 7927 }
 7928 
 7929 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7930   assert(UseAVX > 1, "requires AVX2");
 7931   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7932   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7933   emit_int16(0x47, (0xC0 | encode));
 7934 }
 7935 
 7936 //Variable Shift packed integers logically right.
 7937 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7938   assert(UseAVX > 1, "requires AVX2");
 7939   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7940   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7941   emit_int16(0x45, (0xC0 | encode));
 7942 }
 7943 
 7944 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7945   assert(UseAVX > 1, "requires AVX2");
 7946   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7947   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7948   emit_int16(0x45, (0xC0 | encode));
 7949 }
 7950 
 7951 //Variable right Shift arithmetic packed integers .
 7952 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7953   assert(UseAVX > 1, "requires AVX2");
 7954   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7955   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7956   emit_int16(0x46, (0xC0 | encode));
 7957 }
 7958 
 7959 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7960   assert(VM_Version::supports_avx512bw(), "");
 7961   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7962   attributes.set_is_evex_instruction();
 7963   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7964   emit_int16(0x11, (0xC0 | encode));
 7965 }
 7966 
 7967 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7968   assert(UseAVX > 2, "requires AVX512");
 7969   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7970   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7971   attributes.set_is_evex_instruction();
 7972   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7973   emit_int16(0x46, (0xC0 | encode));
 7974 }
 7975 
 7976 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7977   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7978   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7979   attributes.set_is_evex_instruction();
 7980   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7981   emit_int16(0x71, (0xC0 | encode));
 7982 }
 7983 
 7984 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7985   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7986   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7987   attributes.set_is_evex_instruction();
 7988   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7989   emit_int16(0x73, (0xC0 | encode));
 7990 }
 7991 
 7992 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7993   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7994   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7995   attributes.set_rex_vex_w_reverted();
 7996   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7997   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7998 }
 7999 
 8000 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8001   assert(UseAVX > 0, "requires some form of AVX");
 8002   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8003   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8004   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 8005 }
 8006 
 8007 void Assembler::por(XMMRegister dst, XMMRegister src) {
 8008   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 8009   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8010   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8011   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8012 }
 8013 
 8014 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8015   assert(UseAVX > 0, "requires some form of AVX");
 8016   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8017   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8018   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8019 }
 8020 
 8021 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8022   assert(UseAVX > 0, "requires some form of AVX");
 8023   InstructionMark im(this);
 8024   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8025   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 8026   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8027   emit_int8((unsigned char)0xEB);
 8028   emit_operand(dst, src, 0);
 8029 }
 8030 
 8031 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8032   evporq(dst, k0, nds, src, false, vector_len);
 8033 }
 8034 
 8035 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8036   evporq(dst, k0, nds, src, false, vector_len);
 8037 }
 8038 
 8039 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8040   assert(VM_Version::supports_evex(), "");
 8041   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 8042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8043   attributes.set_is_evex_instruction();
 8044   attributes.set_embedded_opmask_register_specifier(mask);
 8045   if (merge) {
 8046     attributes.reset_is_clear_context();
 8047   }
 8048   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8049   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8050 }
 8051 
 8052 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8053   assert(VM_Version::supports_evex(), "");
 8054   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 8055   InstructionMark im(this);
 8056   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8057   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 8058   attributes.set_is_evex_instruction();
 8059   attributes.set_embedded_opmask_register_specifier(mask);
 8060   if (merge) {
 8061     attributes.reset_is_clear_context();
 8062   }
 8063   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8064   emit_int8((unsigned char)0xEB);
 8065   emit_operand(dst, src, 0);
 8066 }
 8067 
 8068 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 8069   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 8070   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8071   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8072   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8073 }
 8074 
 8075 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8076   assert(UseAVX > 0, "requires some form of AVX");
 8077   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 8078          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 8079          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 8080   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8081   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8082   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8083 }
 8084 
 8085 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8086   assert(UseAVX > 0, "requires some form of AVX");
 8087   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 8088          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 8089          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 8090   InstructionMark im(this);
 8091   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8092   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 8093   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8094   emit_int8((unsigned char)0xEF);
 8095   emit_operand(dst, src, 0);
 8096 }
 8097 
 8098 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8099   assert(UseAVX > 2, "requires some form of EVEX");
 8100   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8101   attributes.set_rex_vex_w_reverted();
 8102   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8103   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8104 }
 8105 
 8106 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8107   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 8108   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8109   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8110   attributes.set_is_evex_instruction();
 8111   attributes.set_embedded_opmask_register_specifier(mask);
 8112   if (merge) {
 8113     attributes.reset_is_clear_context();
 8114   }
 8115   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8116   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8117 }
 8118 
 8119 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8120   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8121   InstructionMark im(this);
 8122   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8123   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8124   attributes.set_is_evex_instruction();
 8125   attributes.set_embedded_opmask_register_specifier(mask);
 8126   if (merge) {
 8127     attributes.reset_is_clear_context();
 8128   }
 8129   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8130   emit_int8((unsigned char)0xEF);
 8131   emit_operand(dst, src, 0);
 8132 }
 8133 
 8134 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8135   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 8136   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8137   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8138   attributes.set_is_evex_instruction();
 8139   attributes.set_embedded_opmask_register_specifier(mask);
 8140   if (merge) {
 8141     attributes.reset_is_clear_context();
 8142   }
 8143   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8144   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8145 }
 8146 
 8147 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8148   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8149   InstructionMark im(this);
 8150   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8151   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8152   attributes.set_is_evex_instruction();
 8153   attributes.set_embedded_opmask_register_specifier(mask);
 8154   if (merge) {
 8155     attributes.reset_is_clear_context();
 8156   }
 8157   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8158   emit_int8((unsigned char)0xEF);
 8159   emit_operand(dst, src, 0);
 8160 }
 8161 
 8162 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8163   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8164   InstructionMark im(this);
 8165   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8166   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8167   attributes.set_is_evex_instruction();
 8168   attributes.set_embedded_opmask_register_specifier(mask);
 8169   if (merge) {
 8170     attributes.reset_is_clear_context();
 8171   }
 8172   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8173   emit_int8((unsigned char)0xDB);
 8174   emit_operand(dst, src, 0);
 8175 }
 8176 
 8177 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8178   assert(VM_Version::supports_evex(), "requires AVX512F");
 8179   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8180   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8181   attributes.set_is_evex_instruction();
 8182   attributes.set_embedded_opmask_register_specifier(mask);
 8183   if (merge) {
 8184     attributes.reset_is_clear_context();
 8185   }
 8186   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8187   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 8188 }
 8189 
 8190 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8191   assert(VM_Version::supports_evex(), "requires AVX512F");
 8192   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8193   InstructionMark im(this);
 8194   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8195   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8196   attributes.set_is_evex_instruction();
 8197   attributes.set_embedded_opmask_register_specifier(mask);
 8198   if (merge) {
 8199     attributes.reset_is_clear_context();
 8200   }
 8201   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8202   emit_int8((unsigned char)0xDB);
 8203   emit_operand(dst, src, 0);
 8204 }
 8205 
 8206 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8207   assert(VM_Version::supports_evex(), "requires AVX512F");
 8208   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8209   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8210   attributes.set_is_evex_instruction();
 8211   attributes.set_embedded_opmask_register_specifier(mask);
 8212   if (merge) {
 8213     attributes.reset_is_clear_context();
 8214   }
 8215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8216   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8217 }
 8218 
 8219 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8220   assert(VM_Version::supports_evex(), "requires AVX512F");
 8221   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8222   InstructionMark im(this);
 8223   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8224   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8225   attributes.set_is_evex_instruction();
 8226   attributes.set_embedded_opmask_register_specifier(mask);
 8227   if (merge) {
 8228     attributes.reset_is_clear_context();
 8229   }
 8230   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8231   emit_int8((unsigned char)0xEB);
 8232   emit_operand(dst, src, 0);
 8233 }
 8234 
 8235 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8236   assert(VM_Version::supports_evex(), "requires EVEX support");
 8237   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8238   attributes.set_is_evex_instruction();
 8239   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8240   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8241 }
 8242 
 8243 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8244   assert(VM_Version::supports_evex(), "requires EVEX support");
 8245   assert(dst != xnoreg, "sanity");
 8246   InstructionMark im(this);
 8247   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8248   attributes.set_is_evex_instruction();
 8249   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8250   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8251   emit_int8((unsigned char)0xEF);
 8252   emit_operand(dst, src, 0);
 8253 }
 8254 
 8255 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8256   assert(VM_Version::supports_evex(), "requires EVEX support");
 8257   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8258   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8259   attributes.set_is_evex_instruction();
 8260   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8261   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8262 }
 8263 
 8264 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8265   assert(VM_Version::supports_evex(), "requires EVEX support");
 8266   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8267   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8268   attributes.set_is_evex_instruction();
 8269   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8270   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8271 }
 8272 
 8273 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8274   assert(VM_Version::supports_evex(), "requires EVEX support");
 8275   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8276   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8277   attributes.set_is_evex_instruction();
 8278   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8279   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8280 }
 8281 
 8282 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8283   assert(VM_Version::supports_evex(), "requires EVEX support");
 8284   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8285   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8286   attributes.set_is_evex_instruction();
 8287   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8288   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8289 }
 8290 
 8291 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8292   assert(VM_Version::supports_evex(), "requires EVEX support");
 8293   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8295   attributes.set_is_evex_instruction();
 8296   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8297   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8298 }
 8299 
 8300 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8301   assert(VM_Version::supports_evex(), "requires EVEX support");
 8302   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8303   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8304   attributes.set_is_evex_instruction();
 8305   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8306   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8307 }
 8308 
 8309 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8310   assert(VM_Version::supports_evex(), "requires EVEX support");
 8311   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8312   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8313   attributes.set_is_evex_instruction();
 8314   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8315   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8316 }
 8317 
 8318 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8319   assert(VM_Version::supports_evex(), "requires EVEX support");
 8320   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8321   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8322   attributes.set_is_evex_instruction();
 8323   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8324   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8325 }
 8326 
 8327 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8328   assert(VM_Version::supports_avx512cd(), "");
 8329   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8330   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8331   attributes.set_is_evex_instruction();
 8332   attributes.set_embedded_opmask_register_specifier(mask);
 8333   if (merge) {
 8334     attributes.reset_is_clear_context();
 8335   }
 8336   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8337   emit_int16(0x44, (0xC0 | encode));
 8338 }
 8339 
 8340 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8341   assert(VM_Version::supports_avx512cd(), "");
 8342   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8343   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8344   attributes.set_is_evex_instruction();
 8345   attributes.set_embedded_opmask_register_specifier(mask);
 8346   if (merge) {
 8347     attributes.reset_is_clear_context();
 8348   }
 8349   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8350   emit_int16(0x44, (0xC0 | encode));
 8351 }
 8352 
 8353 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8354   assert(VM_Version::supports_evex(), "requires EVEX support");
 8355   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8356   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8357   attributes.set_is_evex_instruction();
 8358   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8359   emit_int8(0x25);
 8360   emit_int8((unsigned char)(0xC0 | encode));
 8361   emit_int8(imm8);
 8362 }
 8363 
 8364 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8365   assert(VM_Version::supports_evex(), "requires EVEX support");
 8366   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8367   assert(dst != xnoreg, "sanity");
 8368   InstructionMark im(this);
 8369   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8370   attributes.set_is_evex_instruction();
 8371   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8372   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8373   emit_int8(0x25);
 8374   emit_operand(dst, src3, 1);
 8375   emit_int8(imm8);
 8376 }
 8377 
 8378 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8379   assert(VM_Version::supports_evex(), "requires AVX512F");
 8380   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8381   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8382   attributes.set_is_evex_instruction();
 8383   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8384   emit_int8(0x25);
 8385   emit_int8((unsigned char)(0xC0 | encode));
 8386   emit_int8(imm8);
 8387 }
 8388 
 8389 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8390   assert(VM_Version::supports_evex(), "requires EVEX support");
 8391   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8392   assert(dst != xnoreg, "sanity");
 8393   InstructionMark im(this);
 8394   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8395   attributes.set_is_evex_instruction();
 8396   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8397   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8398   emit_int8(0x25);
 8399   emit_operand(dst, src3, 1);
 8400   emit_int8(imm8);
 8401 }
 8402 
 8403 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8404   assert(VM_Version::supports_evex(), "");
 8405   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8406   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8407   attributes.set_is_evex_instruction();
 8408   attributes.set_embedded_opmask_register_specifier(mask);
 8409   if (merge) {
 8410     attributes.reset_is_clear_context();
 8411   }
 8412   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8413   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8414 }
 8415 
 8416 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8417   assert(VM_Version::supports_evex(), "");
 8418   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8419   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8420   attributes.set_is_evex_instruction();
 8421   attributes.set_embedded_opmask_register_specifier(mask);
 8422   if (merge) {
 8423     attributes.reset_is_clear_context();
 8424   }
 8425   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8426   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8427 }
 8428 
 8429 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8430   assert(VM_Version::supports_avx512_vbmi2(), "");
 8431   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8432   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8433   attributes.set_is_evex_instruction();
 8434   attributes.set_embedded_opmask_register_specifier(mask);
 8435   if (merge) {
 8436     attributes.reset_is_clear_context();
 8437   }
 8438   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8439   emit_int16(0x62, (0xC0 | encode));
 8440 }
 8441 
 8442 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8443   assert(VM_Version::supports_avx512_vbmi2(), "");
 8444   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8445   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8446   attributes.set_is_evex_instruction();
 8447   attributes.set_embedded_opmask_register_specifier(mask);
 8448   if (merge) {
 8449     attributes.reset_is_clear_context();
 8450   }
 8451   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8452   emit_int16(0x62, (0xC0 | encode));
 8453 }
 8454 
 8455 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8456   assert(VM_Version::supports_evex(), "");
 8457   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8458   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8459   attributes.set_is_evex_instruction();
 8460   attributes.set_embedded_opmask_register_specifier(mask);
 8461   if (merge) {
 8462     attributes.reset_is_clear_context();
 8463   }
 8464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8465   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8466 }
 8467 
 8468 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8469   assert(VM_Version::supports_evex(), "");
 8470   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8471   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8472   attributes.set_is_evex_instruction();
 8473   attributes.set_embedded_opmask_register_specifier(mask);
 8474   if (merge) {
 8475     attributes.reset_is_clear_context();
 8476   }
 8477   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8478   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8479 }
 8480 
 8481 // vinserti forms
 8482 
 8483 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8484   assert(VM_Version::supports_avx2(), "");
 8485   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8486   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8487   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8488   // last byte:
 8489   // 0x00 - insert into lower 128 bits
 8490   // 0x01 - insert into upper 128 bits
 8491   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8492 }
 8493 
 8494 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8495   assert(VM_Version::supports_avx2(), "");
 8496   assert(dst != xnoreg, "sanity");
 8497   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8498   InstructionMark im(this);
 8499   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8500   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8501   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8502   emit_int8(0x38);
 8503   emit_operand(dst, src, 1);
 8504   // 0x00 - insert into lower 128 bits
 8505   // 0x01 - insert into upper 128 bits
 8506   emit_int8(imm8 & 0x01);
 8507 }
 8508 
 8509 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8510   assert(VM_Version::supports_evex(), "");
 8511   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8512   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8513   attributes.set_is_evex_instruction();
 8514   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8515   // imm8:
 8516   // 0x00 - insert into q0 128 bits (0..127)
 8517   // 0x01 - insert into q1 128 bits (128..255)
 8518   // 0x02 - insert into q2 128 bits (256..383)
 8519   // 0x03 - insert into q3 128 bits (384..511)
 8520   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8521 }
 8522 
 8523 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8524   assert(VM_Version::supports_evex(), "");
 8525   assert(dst != xnoreg, "sanity");
 8526   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8527   InstructionMark im(this);
 8528   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8529   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8530   attributes.set_is_evex_instruction();
 8531   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8532   emit_int8(0x18);
 8533   emit_operand(dst, src, 1);
 8534   // 0x00 - insert into q0 128 bits (0..127)
 8535   // 0x01 - insert into q1 128 bits (128..255)
 8536   // 0x02 - insert into q2 128 bits (256..383)
 8537   // 0x03 - insert into q3 128 bits (384..511)
 8538   emit_int8(imm8 & 0x03);
 8539 }
 8540 
 8541 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8542   assert(VM_Version::supports_evex(), "");
 8543   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8544   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8545   attributes.set_is_evex_instruction();
 8546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8547   //imm8:
 8548   // 0x00 - insert into lower 256 bits
 8549   // 0x01 - insert into upper 256 bits
 8550   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8551 }
 8552 
 8553 
 8554 // vinsertf forms
 8555 
 8556 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8557   assert(VM_Version::supports_avx(), "");
 8558   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8559   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8560   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8561   // imm8:
 8562   // 0x00 - insert into lower 128 bits
 8563   // 0x01 - insert into upper 128 bits
 8564   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8565 }
 8566 
 8567 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8568   assert(VM_Version::supports_avx(), "");
 8569   assert(dst != xnoreg, "sanity");
 8570   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8571   InstructionMark im(this);
 8572   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8573   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8574   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8575   emit_int8(0x18);
 8576   emit_operand(dst, src, 1);
 8577   // 0x00 - insert into lower 128 bits
 8578   // 0x01 - insert into upper 128 bits
 8579   emit_int8(imm8 & 0x01);
 8580 }
 8581 
 8582 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8583   assert(VM_Version::supports_evex(), "");
 8584   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8585   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8586   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8587   // imm8:
 8588   // 0x00 - insert into q0 128 bits (0..127)
 8589   // 0x01 - insert into q1 128 bits (128..255)
 8590   // 0x02 - insert into q0 128 bits (256..383)
 8591   // 0x03 - insert into q1 128 bits (384..512)
 8592   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8593 }
 8594 
 8595 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8596   assert(VM_Version::supports_evex(), "");
 8597   assert(dst != xnoreg, "sanity");
 8598   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8599   InstructionMark im(this);
 8600   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8601   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8602   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8603   emit_int8(0x18);
 8604   emit_operand(dst, src, 1);
 8605   // 0x00 - insert into q0 128 bits (0..127)
 8606   // 0x01 - insert into q1 128 bits (128..255)
 8607   // 0x02 - insert into q0 128 bits (256..383)
 8608   // 0x03 - insert into q1 128 bits (384..512)
 8609   emit_int8(imm8 & 0x03);
 8610 }
 8611 
 8612 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8613   assert(VM_Version::supports_evex(), "");
 8614   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8615   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8616   attributes.set_is_evex_instruction();
 8617   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8618   // imm8:
 8619   // 0x00 - insert into lower 256 bits
 8620   // 0x01 - insert into upper 256 bits
 8621   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8622 }
 8623 
 8624 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8625   assert(VM_Version::supports_evex(), "");
 8626   assert(dst != xnoreg, "sanity");
 8627   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8628   InstructionMark im(this);
 8629   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8630   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8631   attributes.set_is_evex_instruction();
 8632   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8633   emit_int8(0x1A);
 8634   emit_operand(dst, src, 1);
 8635   // 0x00 - insert into lower 256 bits
 8636   // 0x01 - insert into upper 256 bits
 8637   emit_int8(imm8 & 0x01);
 8638 }
 8639 
 8640 
 8641 // vextracti forms
 8642 
 8643 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8644   assert(VM_Version::supports_avx2(), "");
 8645   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8646   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8647   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8648   // imm8:
 8649   // 0x00 - extract from lower 128 bits
 8650   // 0x01 - extract from upper 128 bits
 8651   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8652 }
 8653 
 8654 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8655   assert(VM_Version::supports_avx2(), "");
 8656   assert(src != xnoreg, "sanity");
 8657   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8658   InstructionMark im(this);
 8659   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8660   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8661   attributes.reset_is_clear_context();
 8662   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8663   emit_int8(0x39);
 8664   emit_operand(src, dst, 1);
 8665   // 0x00 - extract from lower 128 bits
 8666   // 0x01 - extract from upper 128 bits
 8667   emit_int8(imm8 & 0x01);
 8668 }
 8669 
 8670 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8671   assert(VM_Version::supports_evex(), "");
 8672   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8673   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8674   attributes.set_is_evex_instruction();
 8675   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8676   // imm8:
 8677   // 0x00 - extract from bits 127:0
 8678   // 0x01 - extract from bits 255:128
 8679   // 0x02 - extract from bits 383:256
 8680   // 0x03 - extract from bits 511:384
 8681   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8682 }
 8683 
 8684 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8685   assert(VM_Version::supports_evex(), "");
 8686   assert(src != xnoreg, "sanity");
 8687   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8688   InstructionMark im(this);
 8689   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8690   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8691   attributes.reset_is_clear_context();
 8692   attributes.set_is_evex_instruction();
 8693   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8694   emit_int8(0x39);
 8695   emit_operand(src, dst, 1);
 8696   // 0x00 - extract from bits 127:0
 8697   // 0x01 - extract from bits 255:128
 8698   // 0x02 - extract from bits 383:256
 8699   // 0x03 - extract from bits 511:384
 8700   emit_int8(imm8 & 0x03);
 8701 }
 8702 
 8703 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8704   assert(VM_Version::supports_avx512dq(), "");
 8705   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8706   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8707   attributes.set_is_evex_instruction();
 8708   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8709   // imm8:
 8710   // 0x00 - extract from bits 127:0
 8711   // 0x01 - extract from bits 255:128
 8712   // 0x02 - extract from bits 383:256
 8713   // 0x03 - extract from bits 511:384
 8714   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8715 }
 8716 
 8717 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8718   assert(VM_Version::supports_evex(), "");
 8719   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8720   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8721   attributes.set_is_evex_instruction();
 8722   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8723   // imm8:
 8724   // 0x00 - extract from lower 256 bits
 8725   // 0x01 - extract from upper 256 bits
 8726   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8727 }
 8728 
 8729 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8730   assert(VM_Version::supports_evex(), "");
 8731   assert(src != xnoreg, "sanity");
 8732   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8733   InstructionMark im(this);
 8734   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8735   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8736   attributes.reset_is_clear_context();
 8737   attributes.set_is_evex_instruction();
 8738   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8739   emit_int8(0x38);
 8740   emit_operand(src, dst, 1);
 8741   // 0x00 - extract from lower 256 bits
 8742   // 0x01 - extract from upper 256 bits
 8743   emit_int8(imm8 & 0x01);
 8744 }
 8745 // vextractf forms
 8746 
 8747 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8748   assert(VM_Version::supports_avx(), "");
 8749   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8750   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8751   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8752   // imm8:
 8753   // 0x00 - extract from lower 128 bits
 8754   // 0x01 - extract from upper 128 bits
 8755   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8756 }
 8757 
 8758 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8759   assert(VM_Version::supports_avx(), "");
 8760   assert(src != xnoreg, "sanity");
 8761   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8762   InstructionMark im(this);
 8763   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8764   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8765   attributes.reset_is_clear_context();
 8766   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8767   emit_int8(0x19);
 8768   emit_operand(src, dst, 1);
 8769   // 0x00 - extract from lower 128 bits
 8770   // 0x01 - extract from upper 128 bits
 8771   emit_int8(imm8 & 0x01);
 8772 }
 8773 
 8774 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8775   assert(VM_Version::supports_evex(), "");
 8776   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8777   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8778   attributes.set_is_evex_instruction();
 8779   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8780   // imm8:
 8781   // 0x00 - extract from bits 127:0
 8782   // 0x01 - extract from bits 255:128
 8783   // 0x02 - extract from bits 383:256
 8784   // 0x03 - extract from bits 511:384
 8785   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8786 }
 8787 
 8788 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8789   assert(VM_Version::supports_evex(), "");
 8790   assert(src != xnoreg, "sanity");
 8791   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8792   InstructionMark im(this);
 8793   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8794   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8795   attributes.reset_is_clear_context();
 8796   attributes.set_is_evex_instruction();
 8797   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8798   emit_int8(0x19);
 8799   emit_operand(src, dst, 1);
 8800   // 0x00 - extract from bits 127:0
 8801   // 0x01 - extract from bits 255:128
 8802   // 0x02 - extract from bits 383:256
 8803   // 0x03 - extract from bits 511:384
 8804   emit_int8(imm8 & 0x03);
 8805 }
 8806 
 8807 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8808   assert(VM_Version::supports_avx512dq(), "");
 8809   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8810   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8811   attributes.set_is_evex_instruction();
 8812   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8813   // imm8:
 8814   // 0x00 - extract from bits 127:0
 8815   // 0x01 - extract from bits 255:128
 8816   // 0x02 - extract from bits 383:256
 8817   // 0x03 - extract from bits 511:384
 8818   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8819 }
 8820 
 8821 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8822   assert(VM_Version::supports_evex(), "");
 8823   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8824   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8825   attributes.set_is_evex_instruction();
 8826   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8827   // imm8:
 8828   // 0x00 - extract from lower 256 bits
 8829   // 0x01 - extract from upper 256 bits
 8830   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8831 }
 8832 
 8833 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8834   assert(VM_Version::supports_evex(), "");
 8835   assert(src != xnoreg, "sanity");
 8836   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8837   InstructionMark im(this);
 8838   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8839   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8840   attributes.reset_is_clear_context();
 8841   attributes.set_is_evex_instruction();
 8842   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8843   emit_int8(0x1B);
 8844   emit_operand(src, dst, 1);
 8845   // 0x00 - extract from lower 256 bits
 8846   // 0x01 - extract from upper 256 bits
 8847   emit_int8(imm8 & 0x01);
 8848 }
 8849 
 8850 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8851 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8852   assert(VM_Version::supports_avx2(), "");
 8853   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8854   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8855   emit_int16(0x78, (0xC0 | encode));
 8856 }
 8857 
 8858 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8859   assert(VM_Version::supports_avx2(), "");
 8860   assert(dst != xnoreg, "sanity");
 8861   InstructionMark im(this);
 8862   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8863   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8864   // swap src<->dst for encoding
 8865   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8866   emit_int8(0x78);
 8867   emit_operand(dst, src, 0);
 8868 }
 8869 
 8870 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8871 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8872   assert(VM_Version::supports_avx2(), "");
 8873   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8874   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8875   emit_int16(0x79, (0xC0 | encode));
 8876 }
 8877 
 8878 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8879   assert(VM_Version::supports_avx2(), "");
 8880   assert(dst != xnoreg, "sanity");
 8881   InstructionMark im(this);
 8882   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8883   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8884   // swap src<->dst for encoding
 8885   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8886   emit_int8(0x79);
 8887   emit_operand(dst, src, 0);
 8888 }
 8889 
 8890 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8891   assert(UseAVX > 0, "requires some form of AVX");
 8892   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8893   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8894   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8895 }
 8896 
 8897 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8898   assert(UseAVX > 0, "requires some form of AVX");
 8899   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8900   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8901   emit_int16(0x69, (0xC0 | encode));
 8902 }
 8903 
 8904 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8905   assert(UseAVX > 0, "requires some form of AVX");
 8906   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8907   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8908   emit_int16(0x61, (0xC0 | encode));
 8909 }
 8910 
 8911 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8912   assert(UseAVX > 0, "requires some form of AVX");
 8913   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8914   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8915   emit_int16(0x6A, (0xC0 | encode));
 8916 }
 8917 
 8918 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8919   assert(UseAVX > 0, "requires some form of AVX");
 8920   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8921   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8922   emit_int16(0x62, (0xC0 | encode));
 8923 }
 8924 
 8925 // xmm/mem sourced byte/word/dword/qword replicate
 8926 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8927   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8928   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8929   attributes.set_is_evex_instruction();
 8930   attributes.set_embedded_opmask_register_specifier(mask);
 8931   if (merge) {
 8932     attributes.reset_is_clear_context();
 8933   }
 8934   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8935   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8936 }
 8937 
 8938 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8939   InstructionMark im(this);
 8940   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8941   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8942   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8943   attributes.set_is_evex_instruction();
 8944   attributes.set_embedded_opmask_register_specifier(mask);
 8945   if (merge) {
 8946     attributes.reset_is_clear_context();
 8947   }
 8948   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8949   emit_int8((unsigned char)0xFC);
 8950   emit_operand(dst, src, 0);
 8951 }
 8952 
 8953 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8954   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8955   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8956   attributes.set_is_evex_instruction();
 8957   attributes.set_embedded_opmask_register_specifier(mask);
 8958   if (merge) {
 8959     attributes.reset_is_clear_context();
 8960   }
 8961   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8962   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8963 }
 8964 
 8965 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8966   InstructionMark im(this);
 8967   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8968   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8969   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8970   attributes.set_is_evex_instruction();
 8971   attributes.set_embedded_opmask_register_specifier(mask);
 8972   if (merge) {
 8973     attributes.reset_is_clear_context();
 8974   }
 8975   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8976   emit_int8((unsigned char)0xFD);
 8977   emit_operand(dst, src, 0);
 8978 }
 8979 
 8980 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8981   assert(VM_Version::supports_evex(), "");
 8982   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8983   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8984   attributes.set_is_evex_instruction();
 8985   attributes.set_embedded_opmask_register_specifier(mask);
 8986   if (merge) {
 8987     attributes.reset_is_clear_context();
 8988   }
 8989   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8990   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8991 }
 8992 
 8993 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8994   InstructionMark im(this);
 8995   assert(VM_Version::supports_evex(), "");
 8996   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8997   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8998   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8999   attributes.set_is_evex_instruction();
 9000   attributes.set_embedded_opmask_register_specifier(mask);
 9001   if (merge) {
 9002     attributes.reset_is_clear_context();
 9003   }
 9004   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9005   emit_int8((unsigned char)0xFE);
 9006   emit_operand(dst, src, 0);
 9007 }
 9008 
 9009 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9010   assert(VM_Version::supports_evex(), "");
 9011   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9013   attributes.set_is_evex_instruction();
 9014   attributes.set_embedded_opmask_register_specifier(mask);
 9015   if (merge) {
 9016     attributes.reset_is_clear_context();
 9017   }
 9018   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9019   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 9020 }
 9021 
 9022 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9023   InstructionMark im(this);
 9024   assert(VM_Version::supports_evex(), "");
 9025   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9026   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9027   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9028   attributes.set_is_evex_instruction();
 9029   attributes.set_embedded_opmask_register_specifier(mask);
 9030   if (merge) {
 9031     attributes.reset_is_clear_context();
 9032   }
 9033   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9034   emit_int8((unsigned char)0xD4);
 9035   emit_operand(dst, src, 0);
 9036 }
 9037 
 9038 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9039   assert(VM_Version::supports_evex(), "");
 9040   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9041   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9042   attributes.set_is_evex_instruction();
 9043   attributes.set_embedded_opmask_register_specifier(mask);
 9044   if (merge) {
 9045     attributes.reset_is_clear_context();
 9046   }
 9047   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9048   emit_int16(0x58, (0xC0 | encode));
 9049 }
 9050 
 9051 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9052   InstructionMark im(this);
 9053   assert(VM_Version::supports_evex(), "");
 9054   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9055   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9056   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9057   attributes.set_is_evex_instruction();
 9058   attributes.set_embedded_opmask_register_specifier(mask);
 9059   if (merge) {
 9060     attributes.reset_is_clear_context();
 9061   }
 9062   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9063   emit_int8(0x58);
 9064   emit_operand(dst, src, 0);
 9065 }
 9066 
 9067 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9068   assert(VM_Version::supports_evex(), "");
 9069   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9070   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9071   attributes.set_is_evex_instruction();
 9072   attributes.set_embedded_opmask_register_specifier(mask);
 9073   if (merge) {
 9074     attributes.reset_is_clear_context();
 9075   }
 9076   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9077   emit_int16(0x58, (0xC0 | encode));
 9078 }
 9079 
 9080 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9081   InstructionMark im(this);
 9082   assert(VM_Version::supports_evex(), "");
 9083   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9084   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9085   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9086   attributes.set_is_evex_instruction();
 9087   attributes.set_embedded_opmask_register_specifier(mask);
 9088   if (merge) {
 9089     attributes.reset_is_clear_context();
 9090   }
 9091   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9092   emit_int8(0x58);
 9093   emit_operand(dst, src, 0);
 9094 }
 9095 
 9096 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9097   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9098   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9099   attributes.set_is_evex_instruction();
 9100   attributes.set_embedded_opmask_register_specifier(mask);
 9101   if (merge) {
 9102     attributes.reset_is_clear_context();
 9103   }
 9104   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9105   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 9106 }
 9107 
 9108 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9109   InstructionMark im(this);
 9110   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9111   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9112   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9113   attributes.set_is_evex_instruction();
 9114   attributes.set_embedded_opmask_register_specifier(mask);
 9115   if (merge) {
 9116     attributes.reset_is_clear_context();
 9117   }
 9118   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9119   emit_int8((unsigned char)0xF8);
 9120   emit_operand(dst, src, 0);
 9121 }
 9122 
 9123 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9124   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9126   attributes.set_is_evex_instruction();
 9127   attributes.set_embedded_opmask_register_specifier(mask);
 9128   if (merge) {
 9129     attributes.reset_is_clear_context();
 9130   }
 9131   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9132   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 9133 }
 9134 
 9135 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9136   InstructionMark im(this);
 9137   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9138   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9139   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9140   attributes.set_is_evex_instruction();
 9141   attributes.set_embedded_opmask_register_specifier(mask);
 9142   if (merge) {
 9143     attributes.reset_is_clear_context();
 9144   }
 9145   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9146   emit_int8((unsigned char)0xF9);
 9147   emit_operand(dst, src, 0);
 9148 }
 9149 
 9150 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9151   assert(VM_Version::supports_evex(), "");
 9152   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9153   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9154   attributes.set_is_evex_instruction();
 9155   attributes.set_embedded_opmask_register_specifier(mask);
 9156   if (merge) {
 9157     attributes.reset_is_clear_context();
 9158   }
 9159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9160   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 9161 }
 9162 
 9163 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9164   InstructionMark im(this);
 9165   assert(VM_Version::supports_evex(), "");
 9166   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9167   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9168   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9169   attributes.set_is_evex_instruction();
 9170   attributes.set_embedded_opmask_register_specifier(mask);
 9171   if (merge) {
 9172     attributes.reset_is_clear_context();
 9173   }
 9174   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9175   emit_int8((unsigned char)0xFA);
 9176   emit_operand(dst, src, 0);
 9177 }
 9178 
 9179 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9180   assert(VM_Version::supports_evex(), "");
 9181   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9182   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9183   attributes.set_is_evex_instruction();
 9184   attributes.set_embedded_opmask_register_specifier(mask);
 9185   if (merge) {
 9186     attributes.reset_is_clear_context();
 9187   }
 9188   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9189   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 9190 }
 9191 
 9192 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9193   InstructionMark im(this);
 9194   assert(VM_Version::supports_evex(), "");
 9195   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9196   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9197   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9198   attributes.set_is_evex_instruction();
 9199   attributes.set_embedded_opmask_register_specifier(mask);
 9200   if (merge) {
 9201     attributes.reset_is_clear_context();
 9202   }
 9203   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9204   emit_int8((unsigned char)0xFB);
 9205   emit_operand(dst, src, 0);
 9206 }
 9207 
 9208 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9209   assert(VM_Version::supports_evex(), "");
 9210   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9211   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9212   attributes.set_is_evex_instruction();
 9213   attributes.set_embedded_opmask_register_specifier(mask);
 9214   if (merge) {
 9215     attributes.reset_is_clear_context();
 9216   }
 9217   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9218   emit_int16(0x5C, (0xC0 | encode));
 9219 }
 9220 
 9221 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9222   InstructionMark im(this);
 9223   assert(VM_Version::supports_evex(), "");
 9224   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9225   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9226   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9227   attributes.set_is_evex_instruction();
 9228   attributes.set_embedded_opmask_register_specifier(mask);
 9229   if (merge) {
 9230     attributes.reset_is_clear_context();
 9231   }
 9232   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9233   emit_int8(0x5C);
 9234   emit_operand(dst, src, 0);
 9235 }
 9236 
 9237 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9238   assert(VM_Version::supports_evex(), "");
 9239   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9240   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9241   attributes.set_is_evex_instruction();
 9242   attributes.set_embedded_opmask_register_specifier(mask);
 9243   if (merge) {
 9244     attributes.reset_is_clear_context();
 9245   }
 9246   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9247   emit_int16(0x5C, (0xC0 | encode));
 9248 }
 9249 
 9250 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9251   InstructionMark im(this);
 9252   assert(VM_Version::supports_evex(), "");
 9253   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9254   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9255   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9256   attributes.set_is_evex_instruction();
 9257   attributes.set_embedded_opmask_register_specifier(mask);
 9258   if (merge) {
 9259     attributes.reset_is_clear_context();
 9260   }
 9261   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9262   emit_int8(0x5C);
 9263   emit_operand(dst, src, 0);
 9264 }
 9265 
 9266 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9267   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9268   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9269   attributes.set_is_evex_instruction();
 9270   attributes.set_embedded_opmask_register_specifier(mask);
 9271   if (merge) {
 9272     attributes.reset_is_clear_context();
 9273   }
 9274   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9275   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 9276 }
 9277 
 9278 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9279   InstructionMark im(this);
 9280   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9281   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9282   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9283   attributes.set_is_evex_instruction();
 9284   attributes.set_embedded_opmask_register_specifier(mask);
 9285   if (merge) {
 9286     attributes.reset_is_clear_context();
 9287   }
 9288   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9289   emit_int8((unsigned char)0xD5);
 9290   emit_operand(dst, src, 0);
 9291 }
 9292 
 9293 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9294   assert(VM_Version::supports_evex(), "");
 9295   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9296   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9297   attributes.set_is_evex_instruction();
 9298   attributes.set_embedded_opmask_register_specifier(mask);
 9299   if (merge) {
 9300     attributes.reset_is_clear_context();
 9301   }
 9302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9303   emit_int16(0x40, (0xC0 | encode));
 9304 }
 9305 
 9306 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9307   InstructionMark im(this);
 9308   assert(VM_Version::supports_evex(), "");
 9309   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9310   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9311   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9312   attributes.set_is_evex_instruction();
 9313   attributes.set_embedded_opmask_register_specifier(mask);
 9314   if (merge) {
 9315     attributes.reset_is_clear_context();
 9316   }
 9317   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9318   emit_int8(0x40);
 9319   emit_operand(dst, src, 0);
 9320 }
 9321 
 9322 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9323   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9324   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9325   attributes.set_is_evex_instruction();
 9326   attributes.set_embedded_opmask_register_specifier(mask);
 9327   if (merge) {
 9328     attributes.reset_is_clear_context();
 9329   }
 9330   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9331   emit_int16(0x40, (0xC0 | encode));
 9332 }
 9333 
 9334 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9335   InstructionMark im(this);
 9336   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9337   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9338   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9339   attributes.set_is_evex_instruction();
 9340   attributes.set_embedded_opmask_register_specifier(mask);
 9341   if (merge) {
 9342     attributes.reset_is_clear_context();
 9343   }
 9344   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9345   emit_int8(0x40);
 9346   emit_operand(dst, src, 0);
 9347 }
 9348 
 9349 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9350   assert(VM_Version::supports_evex(), "");
 9351   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9352   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9353   attributes.set_is_evex_instruction();
 9354   attributes.set_embedded_opmask_register_specifier(mask);
 9355   if (merge) {
 9356     attributes.reset_is_clear_context();
 9357   }
 9358   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9359   emit_int16(0x59, (0xC0 | encode));
 9360 }
 9361 
 9362 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9363   InstructionMark im(this);
 9364   assert(VM_Version::supports_evex(), "");
 9365   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9366   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9367   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9368   attributes.set_is_evex_instruction();
 9369   attributes.set_embedded_opmask_register_specifier(mask);
 9370   if (merge) {
 9371     attributes.reset_is_clear_context();
 9372   }
 9373   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9374   emit_int8(0x59);
 9375   emit_operand(dst, src, 0);
 9376 }
 9377 
 9378 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9379   assert(VM_Version::supports_evex(), "");
 9380   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9381   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9382   attributes.set_is_evex_instruction();
 9383   attributes.set_embedded_opmask_register_specifier(mask);
 9384   if (merge) {
 9385     attributes.reset_is_clear_context();
 9386   }
 9387   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9388   emit_int16(0x59, (0xC0 | encode));
 9389 }
 9390 
 9391 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9392   InstructionMark im(this);
 9393   assert(VM_Version::supports_evex(), "");
 9394   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9395   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9396   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9397   attributes.set_is_evex_instruction();
 9398   attributes.set_embedded_opmask_register_specifier(mask);
 9399   if (merge) {
 9400     attributes.reset_is_clear_context();
 9401   }
 9402   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9403   emit_int8(0x59);
 9404   emit_operand(dst, src, 0);
 9405 }
 9406 
 9407 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9408   assert(VM_Version::supports_evex(), "");
 9409   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9410   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9411   attributes.set_is_evex_instruction();
 9412   attributes.set_embedded_opmask_register_specifier(mask);
 9413   if (merge) {
 9414     attributes.reset_is_clear_context();
 9415   }
 9416   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9417   emit_int16(0x51, (0xC0 | encode));
 9418 }
 9419 
 9420 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9421   InstructionMark im(this);
 9422   assert(VM_Version::supports_evex(), "");
 9423   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9424   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9425   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9426   attributes.set_is_evex_instruction();
 9427   attributes.set_embedded_opmask_register_specifier(mask);
 9428   if (merge) {
 9429     attributes.reset_is_clear_context();
 9430   }
 9431   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9432   emit_int8(0x51);
 9433   emit_operand(dst, src, 0);
 9434 }
 9435 
 9436 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9437   assert(VM_Version::supports_evex(), "");
 9438   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9439   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9440   attributes.set_is_evex_instruction();
 9441   attributes.set_embedded_opmask_register_specifier(mask);
 9442   if (merge) {
 9443     attributes.reset_is_clear_context();
 9444   }
 9445   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9446   emit_int16(0x51, (0xC0 | encode));
 9447 }
 9448 
 9449 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9450   InstructionMark im(this);
 9451   assert(VM_Version::supports_evex(), "");
 9452   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9453   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9454   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9455   attributes.set_is_evex_instruction();
 9456   attributes.set_embedded_opmask_register_specifier(mask);
 9457   if (merge) {
 9458     attributes.reset_is_clear_context();
 9459   }
 9460   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9461   emit_int8(0x51);
 9462   emit_operand(dst, src, 0);
 9463 }
 9464 
 9465 
 9466 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9467   assert(VM_Version::supports_evex(), "");
 9468   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9469   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9470   attributes.set_is_evex_instruction();
 9471   attributes.set_embedded_opmask_register_specifier(mask);
 9472   if (merge) {
 9473     attributes.reset_is_clear_context();
 9474   }
 9475   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9476   emit_int16(0x5E, (0xC0 | encode));
 9477 }
 9478 
 9479 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9480   InstructionMark im(this);
 9481   assert(VM_Version::supports_evex(), "");
 9482   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9483   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9484   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9485   attributes.set_is_evex_instruction();
 9486   attributes.set_embedded_opmask_register_specifier(mask);
 9487   if (merge) {
 9488     attributes.reset_is_clear_context();
 9489   }
 9490   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9491   emit_int8(0x5E);
 9492   emit_operand(dst, src, 0);
 9493 }
 9494 
 9495 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9496   assert(VM_Version::supports_evex(), "");
 9497   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9498   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9499   attributes.set_is_evex_instruction();
 9500   attributes.set_embedded_opmask_register_specifier(mask);
 9501   if (merge) {
 9502     attributes.reset_is_clear_context();
 9503   }
 9504   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9505   emit_int16(0x5E, (0xC0 | encode));
 9506 }
 9507 
 9508 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9509   InstructionMark im(this);
 9510   assert(VM_Version::supports_evex(), "");
 9511   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9512   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9513   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9514   attributes.set_is_evex_instruction();
 9515   attributes.set_embedded_opmask_register_specifier(mask);
 9516   if (merge) {
 9517     attributes.reset_is_clear_context();
 9518   }
 9519   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9520   emit_int8(0x5E);
 9521   emit_operand(dst, src, 0);
 9522 }
 9523 
 9524 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9525   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9526   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9527   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9528   attributes.set_is_evex_instruction();
 9529   attributes.set_embedded_opmask_register_specifier(mask);
 9530   if (merge) {
 9531     attributes.reset_is_clear_context();
 9532   }
 9533   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9534   emit_int16(0x1C, (0xC0 | encode));
 9535 }
 9536 
 9537 
 9538 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9539   InstructionMark im(this);
 9540   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9541   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9542   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9543   attributes.set_is_evex_instruction();
 9544   attributes.set_embedded_opmask_register_specifier(mask);
 9545   if (merge) {
 9546     attributes.reset_is_clear_context();
 9547   }
 9548   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9549   emit_int8(0x1C);
 9550   emit_operand(dst, src, 0);
 9551 }
 9552 
 9553 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9554   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9555   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9556   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9557   attributes.set_is_evex_instruction();
 9558   attributes.set_embedded_opmask_register_specifier(mask);
 9559   if (merge) {
 9560     attributes.reset_is_clear_context();
 9561   }
 9562   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9563   emit_int16(0x1D, (0xC0 | encode));
 9564 }
 9565 
 9566 
 9567 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9568   InstructionMark im(this);
 9569   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9570   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9571   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9572   attributes.set_is_evex_instruction();
 9573   attributes.set_embedded_opmask_register_specifier(mask);
 9574   if (merge) {
 9575     attributes.reset_is_clear_context();
 9576   }
 9577   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9578   emit_int8(0x1D);
 9579   emit_operand(dst, src, 0);
 9580 }
 9581 
 9582 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9583   assert(VM_Version::supports_evex(), "");
 9584   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9585   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9586   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9587   attributes.set_is_evex_instruction();
 9588   attributes.set_embedded_opmask_register_specifier(mask);
 9589   if (merge) {
 9590     attributes.reset_is_clear_context();
 9591   }
 9592   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9593   emit_int16(0x1E, (0xC0 | encode));
 9594 }
 9595 
 9596 
 9597 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9598   InstructionMark im(this);
 9599   assert(VM_Version::supports_evex(), "");
 9600   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9601   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9602   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9603   attributes.set_is_evex_instruction();
 9604   attributes.set_embedded_opmask_register_specifier(mask);
 9605   if (merge) {
 9606     attributes.reset_is_clear_context();
 9607   }
 9608   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9609   emit_int8(0x1E);
 9610   emit_operand(dst, src, 0);
 9611 }
 9612 
 9613 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9614   assert(VM_Version::supports_evex(), "");
 9615   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9616   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9617   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9618   attributes.set_is_evex_instruction();
 9619   attributes.set_embedded_opmask_register_specifier(mask);
 9620   if (merge) {
 9621     attributes.reset_is_clear_context();
 9622   }
 9623   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9624   emit_int16(0x1F, (0xC0 | encode));
 9625 }
 9626 
 9627 
 9628 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9629   InstructionMark im(this);
 9630   assert(VM_Version::supports_evex(), "");
 9631   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9632   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9633   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9634   attributes.set_is_evex_instruction();
 9635   attributes.set_embedded_opmask_register_specifier(mask);
 9636   if (merge) {
 9637     attributes.reset_is_clear_context();
 9638   }
 9639   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9640   emit_int8(0x1F);
 9641   emit_operand(dst, src, 0);
 9642 }
 9643 
 9644 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9645   assert(VM_Version::supports_evex(), "");
 9646   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9647   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9648   attributes.set_is_evex_instruction();
 9649   attributes.set_embedded_opmask_register_specifier(mask);
 9650   if (merge) {
 9651     attributes.reset_is_clear_context();
 9652   }
 9653   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9654   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9655 }
 9656 
 9657 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9658   InstructionMark im(this);
 9659   assert(VM_Version::supports_evex(), "");
 9660   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9661   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9662   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9663   attributes.set_is_evex_instruction();
 9664   attributes.set_embedded_opmask_register_specifier(mask);
 9665   if (merge) {
 9666     attributes.reset_is_clear_context();
 9667   }
 9668   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9669   emit_int8((unsigned char)0xA8);
 9670   emit_operand(dst, src, 0);
 9671 }
 9672 
 9673 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9674   assert(VM_Version::supports_evex(), "");
 9675   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9676   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9677   attributes.set_is_evex_instruction();
 9678   attributes.set_embedded_opmask_register_specifier(mask);
 9679   if (merge) {
 9680     attributes.reset_is_clear_context();
 9681   }
 9682   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9683   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9684 }
 9685 
 9686 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9687   InstructionMark im(this);
 9688   assert(VM_Version::supports_evex(), "");
 9689   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9690   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9691   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9692   attributes.set_is_evex_instruction();
 9693   attributes.set_embedded_opmask_register_specifier(mask);
 9694   if (merge) {
 9695     attributes.reset_is_clear_context();
 9696   }
 9697   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9698   emit_int8((unsigned char)0xA8);
 9699   emit_operand(dst, src, 0);
 9700 }
 9701 
 9702 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9703   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9704   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9705   attributes.set_is_evex_instruction();
 9706   attributes.set_embedded_opmask_register_specifier(mask);
 9707   if (merge) {
 9708     attributes.reset_is_clear_context();
 9709   }
 9710   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9711   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9712 }
 9713 
 9714 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9715   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9716   InstructionMark im(this);
 9717   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9718   attributes.set_is_evex_instruction();
 9719   attributes.set_embedded_opmask_register_specifier(mask);
 9720   if (merge) {
 9721     attributes.reset_is_clear_context();
 9722   }
 9723   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9724   emit_int8((unsigned char)0x8D);
 9725   emit_operand(dst, src, 0);
 9726 }
 9727 
 9728 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9729   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9730   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9731   attributes.set_is_evex_instruction();
 9732   attributes.set_embedded_opmask_register_specifier(mask);
 9733   if (merge) {
 9734     attributes.reset_is_clear_context();
 9735   }
 9736   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9737   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9738 }
 9739 
 9740 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9741   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9742   InstructionMark im(this);
 9743   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9744   attributes.set_is_evex_instruction();
 9745   attributes.set_embedded_opmask_register_specifier(mask);
 9746   if (merge) {
 9747     attributes.reset_is_clear_context();
 9748   }
 9749   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9750   emit_int8((unsigned char)0x8D);
 9751   emit_operand(dst, src, 0);
 9752 }
 9753 
 9754 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9755   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9756   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9757   attributes.set_is_evex_instruction();
 9758   attributes.set_embedded_opmask_register_specifier(mask);
 9759   if (merge) {
 9760     attributes.reset_is_clear_context();
 9761   }
 9762   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9763   emit_int16(0x36, (0xC0 | encode));
 9764 }
 9765 
 9766 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9767   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9768   InstructionMark im(this);
 9769   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9770   attributes.set_is_evex_instruction();
 9771   attributes.set_embedded_opmask_register_specifier(mask);
 9772   if (merge) {
 9773     attributes.reset_is_clear_context();
 9774   }
 9775   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9776   emit_int8(0x36);
 9777   emit_operand(dst, src, 0);
 9778 }
 9779 
 9780 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9781   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9782   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9783   attributes.set_is_evex_instruction();
 9784   attributes.set_embedded_opmask_register_specifier(mask);
 9785   if (merge) {
 9786     attributes.reset_is_clear_context();
 9787   }
 9788   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9789   emit_int16(0x36, (0xC0 | encode));
 9790 }
 9791 
 9792 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9793   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9794   InstructionMark im(this);
 9795   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9796   attributes.set_is_evex_instruction();
 9797   attributes.set_embedded_opmask_register_specifier(mask);
 9798   if (merge) {
 9799     attributes.reset_is_clear_context();
 9800   }
 9801   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9802   emit_int8(0x36);
 9803   emit_operand(dst, src, 0);
 9804 }
 9805 
 9806 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9807   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9808   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9809   attributes.set_is_evex_instruction();
 9810   attributes.set_embedded_opmask_register_specifier(mask);
 9811   if (merge) {
 9812     attributes.reset_is_clear_context();
 9813   }
 9814   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9815   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9816 }
 9817 
 9818 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9819   assert(VM_Version::supports_evex(), "");
 9820   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9821   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9822   attributes.set_is_evex_instruction();
 9823   attributes.set_embedded_opmask_register_specifier(mask);
 9824   if (merge) {
 9825     attributes.reset_is_clear_context();
 9826   }
 9827   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9828   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9829 }
 9830 
 9831 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9832   assert(VM_Version::supports_evex(), "");
 9833   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9834   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9835   attributes.set_is_evex_instruction();
 9836   attributes.set_embedded_opmask_register_specifier(mask);
 9837   if (merge) {
 9838     attributes.reset_is_clear_context();
 9839   }
 9840   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9841   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9842 }
 9843 
 9844 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9845   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9846   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9847   attributes.set_is_evex_instruction();
 9848   attributes.set_embedded_opmask_register_specifier(mask);
 9849   if (merge) {
 9850     attributes.reset_is_clear_context();
 9851   }
 9852   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9853   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9854 }
 9855 
 9856 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9857   assert(VM_Version::supports_evex(), "");
 9858   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9859   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9860   attributes.set_is_evex_instruction();
 9861   attributes.set_embedded_opmask_register_specifier(mask);
 9862   if (merge) {
 9863     attributes.reset_is_clear_context();
 9864   }
 9865   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9866   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9867 }
 9868 
 9869 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9870   assert(VM_Version::supports_evex(), "");
 9871   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9872   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9873   attributes.set_is_evex_instruction();
 9874   attributes.set_embedded_opmask_register_specifier(mask);
 9875   if (merge) {
 9876     attributes.reset_is_clear_context();
 9877   }
 9878   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9879   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9880 }
 9881 
 9882 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9883   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9884   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9885   attributes.set_is_evex_instruction();
 9886   attributes.set_embedded_opmask_register_specifier(mask);
 9887   if (merge) {
 9888     attributes.reset_is_clear_context();
 9889   }
 9890   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9891   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9892 }
 9893 
 9894 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9895   assert(VM_Version::supports_evex(), "");
 9896   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9897   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9898   attributes.set_is_evex_instruction();
 9899   attributes.set_embedded_opmask_register_specifier(mask);
 9900   if (merge) {
 9901     attributes.reset_is_clear_context();
 9902   }
 9903   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9904   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9905 }
 9906 
 9907 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9908   assert(VM_Version::supports_evex(), "");
 9909   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9910   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9911   attributes.set_is_evex_instruction();
 9912   attributes.set_embedded_opmask_register_specifier(mask);
 9913   if (merge) {
 9914     attributes.reset_is_clear_context();
 9915   }
 9916   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9917   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9918 }
 9919 
 9920 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9921   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9922   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9923   attributes.set_is_evex_instruction();
 9924   attributes.set_embedded_opmask_register_specifier(mask);
 9925   if (merge) {
 9926     attributes.reset_is_clear_context();
 9927   }
 9928   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9929   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9930 }
 9931 
 9932 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9933   assert(VM_Version::supports_evex(), "");
 9934   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9935   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9936   attributes.set_is_evex_instruction();
 9937   attributes.set_embedded_opmask_register_specifier(mask);
 9938   if (merge) {
 9939     attributes.reset_is_clear_context();
 9940   }
 9941   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9942   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9943 }
 9944 
 9945 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9946   assert(VM_Version::supports_evex(), "");
 9947   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9948   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9949   attributes.set_is_evex_instruction();
 9950   attributes.set_embedded_opmask_register_specifier(mask);
 9951   if (merge) {
 9952     attributes.reset_is_clear_context();
 9953   }
 9954   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9955   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9956 }
 9957 
 9958 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9959   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9960   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9961   attributes.set_is_evex_instruction();
 9962   attributes.set_embedded_opmask_register_specifier(mask);
 9963   if (merge) {
 9964     attributes.reset_is_clear_context();
 9965   }
 9966   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9967   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9968 }
 9969 
 9970 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9971   assert(VM_Version::supports_evex(), "");
 9972   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9973   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9974   attributes.set_is_evex_instruction();
 9975   attributes.set_embedded_opmask_register_specifier(mask);
 9976   if (merge) {
 9977     attributes.reset_is_clear_context();
 9978   }
 9979   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9980   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9981 }
 9982 
 9983 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9984   assert(VM_Version::supports_evex(), "");
 9985   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9986   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9987   attributes.set_is_evex_instruction();
 9988   attributes.set_embedded_opmask_register_specifier(mask);
 9989   if (merge) {
 9990     attributes.reset_is_clear_context();
 9991   }
 9992   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9993   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9994 }
 9995 
 9996 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9997   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9998   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9999   attributes.set_is_evex_instruction();
10000   attributes.set_embedded_opmask_register_specifier(mask);
10001   if (merge) {
10002     attributes.reset_is_clear_context();
10003   }
10004   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10005   emit_int16((unsigned char)0xE1, (0xC0 | encode));
10006 }
10007 
10008 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10009   assert(VM_Version::supports_evex(), "");
10010   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10011   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10012   attributes.set_is_evex_instruction();
10013   attributes.set_embedded_opmask_register_specifier(mask);
10014   if (merge) {
10015     attributes.reset_is_clear_context();
10016   }
10017   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10018   emit_int16((unsigned char)0xE2, (0xC0 | encode));
10019 }
10020 
10021 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10022   assert(VM_Version::supports_evex(), "");
10023   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10024   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10025   attributes.set_is_evex_instruction();
10026   attributes.set_embedded_opmask_register_specifier(mask);
10027   if (merge) {
10028     attributes.reset_is_clear_context();
10029   }
10030   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10031   emit_int16((unsigned char)0xE2, (0xC0 | encode));
10032 }
10033 
10034 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10035   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10036   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10037   attributes.set_is_evex_instruction();
10038   attributes.set_embedded_opmask_register_specifier(mask);
10039   if (merge) {
10040     attributes.reset_is_clear_context();
10041   }
10042   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10043   emit_int16(0x12, (0xC0 | encode));
10044 }
10045 
10046 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10047   assert(VM_Version::supports_evex(), "");
10048   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10049   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10050   attributes.set_is_evex_instruction();
10051   attributes.set_embedded_opmask_register_specifier(mask);
10052   if (merge) {
10053     attributes.reset_is_clear_context();
10054   }
10055   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10056   emit_int16(0x47, (0xC0 | encode));
10057 }
10058 
10059 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10060   assert(VM_Version::supports_evex(), "");
10061   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10062   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10063   attributes.set_is_evex_instruction();
10064   attributes.set_embedded_opmask_register_specifier(mask);
10065   if (merge) {
10066     attributes.reset_is_clear_context();
10067   }
10068   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10069   emit_int16(0x47, (0xC0 | encode));
10070 }
10071 
10072 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10073   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10074   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10075   attributes.set_is_evex_instruction();
10076   attributes.set_embedded_opmask_register_specifier(mask);
10077   if (merge) {
10078     attributes.reset_is_clear_context();
10079   }
10080   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10081   emit_int16(0x10, (0xC0 | encode));
10082 }
10083 
10084 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10085   assert(VM_Version::supports_evex(), "");
10086   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10088   attributes.set_is_evex_instruction();
10089   attributes.set_embedded_opmask_register_specifier(mask);
10090   if (merge) {
10091     attributes.reset_is_clear_context();
10092   }
10093   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10094   emit_int16(0x45, (0xC0 | encode));
10095 }
10096 
10097 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10098   assert(VM_Version::supports_evex(), "");
10099   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10100   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10101   attributes.set_is_evex_instruction();
10102   attributes.set_embedded_opmask_register_specifier(mask);
10103   if (merge) {
10104     attributes.reset_is_clear_context();
10105   }
10106   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10107   emit_int16(0x45, (0xC0 | encode));
10108 }
10109 
10110 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10111   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10112   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10113   attributes.set_is_evex_instruction();
10114   attributes.set_embedded_opmask_register_specifier(mask);
10115   if (merge) {
10116     attributes.reset_is_clear_context();
10117   }
10118   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10119   emit_int16(0x11, (0xC0 | encode));
10120 }
10121 
10122 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10123   assert(VM_Version::supports_evex(), "");
10124   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10126   attributes.set_is_evex_instruction();
10127   attributes.set_embedded_opmask_register_specifier(mask);
10128   if (merge) {
10129     attributes.reset_is_clear_context();
10130   }
10131   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10132   emit_int16(0x46, (0xC0 | encode));
10133 }
10134 
10135 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10136   assert(VM_Version::supports_evex(), "");
10137   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10138   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10139   attributes.set_is_evex_instruction();
10140   attributes.set_embedded_opmask_register_specifier(mask);
10141   if (merge) {
10142     attributes.reset_is_clear_context();
10143   }
10144   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10145   emit_int16(0x46, (0xC0 | encode));
10146 }
10147 
10148 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10149   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10150   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10151   attributes.set_is_evex_instruction();
10152   attributes.set_embedded_opmask_register_specifier(mask);
10153   if (merge) {
10154     attributes.reset_is_clear_context();
10155   }
10156   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10157   emit_int16(0x38, (0xC0 | encode));
10158 }
10159 
10160 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10161   assert(VM_Version::supports_avx512bw(), "");
10162   InstructionMark im(this);
10163   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10164   attributes.set_is_evex_instruction();
10165   attributes.set_embedded_opmask_register_specifier(mask);
10166   if (merge) {
10167     attributes.reset_is_clear_context();
10168   }
10169   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10170   emit_int8(0x38);
10171   emit_operand(dst, src, 0);
10172 }
10173 
10174 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10175   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10176   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10177   attributes.set_is_evex_instruction();
10178   attributes.set_embedded_opmask_register_specifier(mask);
10179   if (merge) {
10180     attributes.reset_is_clear_context();
10181   }
10182   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10183   emit_int16((unsigned char)0xEA, (0xC0 | encode));
10184 }
10185 
10186 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10187   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10188   InstructionMark im(this);
10189   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10190   attributes.set_is_evex_instruction();
10191   attributes.set_embedded_opmask_register_specifier(mask);
10192   if (merge) {
10193     attributes.reset_is_clear_context();
10194   }
10195   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10196   emit_int8((unsigned char)0xEA);
10197   emit_operand(dst, src, 0);
10198 }
10199 
10200 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10201   assert(VM_Version::supports_evex(), "");
10202   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10204   attributes.set_is_evex_instruction();
10205   attributes.set_embedded_opmask_register_specifier(mask);
10206   if (merge) {
10207     attributes.reset_is_clear_context();
10208   }
10209   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10210   emit_int16(0x39, (0xC0 | encode));
10211 }
10212 
10213 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10214   assert(VM_Version::supports_evex(), "");
10215   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10216   InstructionMark im(this);
10217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10218   attributes.set_is_evex_instruction();
10219   attributes.set_embedded_opmask_register_specifier(mask);
10220   if (merge) {
10221     attributes.reset_is_clear_context();
10222   }
10223   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10224   emit_int8(0x39);
10225   emit_operand(dst, src, 0);
10226 }
10227 
10228 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10229   assert(VM_Version::supports_evex(), "");
10230   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10231   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10232   attributes.set_is_evex_instruction();
10233   attributes.set_embedded_opmask_register_specifier(mask);
10234   if (merge) {
10235     attributes.reset_is_clear_context();
10236   }
10237   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10238   emit_int16(0x39, (0xC0 | encode));
10239 }
10240 
10241 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10242   assert(VM_Version::supports_evex(), "");
10243   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10244   InstructionMark im(this);
10245   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10246   attributes.set_is_evex_instruction();
10247   attributes.set_embedded_opmask_register_specifier(mask);
10248   if (merge) {
10249     attributes.reset_is_clear_context();
10250   }
10251   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10252   emit_int8(0x39);
10253   emit_operand(dst, src, 0);
10254 }
10255 
10256 
10257 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10258   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10259   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10260   attributes.set_is_evex_instruction();
10261   attributes.set_embedded_opmask_register_specifier(mask);
10262   if (merge) {
10263     attributes.reset_is_clear_context();
10264   }
10265   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10266   emit_int16(0x3C, (0xC0 | encode));
10267 }
10268 
10269 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10270   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10271   InstructionMark im(this);
10272   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10273   attributes.set_is_evex_instruction();
10274   attributes.set_embedded_opmask_register_specifier(mask);
10275   if (merge) {
10276     attributes.reset_is_clear_context();
10277   }
10278   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10279   emit_int8(0x3C);
10280   emit_operand(dst, src, 0);
10281 }
10282 
10283 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10284   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10285   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10286   attributes.set_is_evex_instruction();
10287   attributes.set_embedded_opmask_register_specifier(mask);
10288   if (merge) {
10289     attributes.reset_is_clear_context();
10290   }
10291   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10292   emit_int16((unsigned char)0xEE, (0xC0 | encode));
10293 }
10294 
10295 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10296   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10297   InstructionMark im(this);
10298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10299   attributes.set_is_evex_instruction();
10300   attributes.set_embedded_opmask_register_specifier(mask);
10301   if (merge) {
10302     attributes.reset_is_clear_context();
10303   }
10304   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10305   emit_int8((unsigned char)0xEE);
10306   emit_operand(dst, src, 0);
10307 }
10308 
10309 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10310   assert(VM_Version::supports_evex(), "");
10311   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10312   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10313   attributes.set_is_evex_instruction();
10314   attributes.set_embedded_opmask_register_specifier(mask);
10315   if (merge) {
10316     attributes.reset_is_clear_context();
10317   }
10318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10319   emit_int16(0x3D, (0xC0 | encode));
10320 }
10321 
10322 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10323   assert(VM_Version::supports_evex(), "");
10324   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10325   InstructionMark im(this);
10326   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10327   attributes.set_is_evex_instruction();
10328   attributes.set_embedded_opmask_register_specifier(mask);
10329   if (merge) {
10330     attributes.reset_is_clear_context();
10331   }
10332   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10333   emit_int8(0x3D);
10334   emit_operand(dst, src, 0);
10335 }
10336 
10337 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10338   assert(VM_Version::supports_evex(), "");
10339   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10340   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10341   attributes.set_is_evex_instruction();
10342   attributes.set_embedded_opmask_register_specifier(mask);
10343   if (merge) {
10344     attributes.reset_is_clear_context();
10345   }
10346   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10347   emit_int16(0x3D, (0xC0 | encode));
10348 }
10349 
10350 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10351   assert(VM_Version::supports_evex(), "");
10352   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10353   InstructionMark im(this);
10354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10355   attributes.set_is_evex_instruction();
10356   attributes.set_embedded_opmask_register_specifier(mask);
10357   if (merge) {
10358     attributes.reset_is_clear_context();
10359   }
10360   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10361   emit_int8(0x3D);
10362   emit_operand(dst, src, 0);
10363 }
10364 
10365 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10366   assert(VM_Version::supports_evex(), "requires EVEX support");
10367   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10368   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10369   attributes.set_is_evex_instruction();
10370   attributes.set_embedded_opmask_register_specifier(mask);
10371   if (merge) {
10372     attributes.reset_is_clear_context();
10373   }
10374   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10375   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10376 }
10377 
10378 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10379   assert(VM_Version::supports_evex(), "requires EVEX support");
10380   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10381   assert(dst != xnoreg, "sanity");
10382   InstructionMark im(this);
10383   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10384   attributes.set_is_evex_instruction();
10385   attributes.set_embedded_opmask_register_specifier(mask);
10386   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10387   if (merge) {
10388     attributes.reset_is_clear_context();
10389   }
10390   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10391   emit_int8(0x25);
10392   emit_operand(dst, src3, 1);
10393   emit_int8(imm8);
10394 }
10395 
10396 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10397   assert(VM_Version::supports_evex(), "requires EVEX support");
10398   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10399   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10400   attributes.set_is_evex_instruction();
10401   attributes.set_embedded_opmask_register_specifier(mask);
10402   if (merge) {
10403     attributes.reset_is_clear_context();
10404   }
10405   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10406   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10407 }
10408 
10409 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10410   assert(VM_Version::supports_evex(), "requires EVEX support");
10411   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10412   assert(dst != xnoreg, "sanity");
10413   InstructionMark im(this);
10414   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10415   attributes.set_is_evex_instruction();
10416   attributes.set_embedded_opmask_register_specifier(mask);
10417   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10418   if (merge) {
10419     attributes.reset_is_clear_context();
10420   }
10421   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10422   emit_int8(0x25);
10423   emit_operand(dst, src3, 1);
10424   emit_int8(imm8);
10425 }
10426 
10427 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10428   assert(VM_Version::supports_gfni(), "");
10429   assert(VM_Version::supports_sse(), "");
10430   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10431   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10432   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10433 }
10434 
10435 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10436   assert(VM_Version::supports_gfni(), "requires GFNI support");
10437   assert(VM_Version::supports_sse(), "");
10438   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10439   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10440   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10441 }
10442 
10443 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10444 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10445   assert(UseAVX >= 2, "");
10446   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10447   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10448   emit_int16(0x58, (0xC0 | encode));
10449 }
10450 
10451 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10452   assert(VM_Version::supports_avx2(), "");
10453   assert(dst != xnoreg, "sanity");
10454   InstructionMark im(this);
10455   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10456   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10457   // swap src<->dst for encoding
10458   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10459   emit_int8(0x58);
10460   emit_operand(dst, src, 0);
10461 }
10462 
10463 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10464 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10465   assert(VM_Version::supports_avx2(), "");
10466   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10467   attributes.set_rex_vex_w_reverted();
10468   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10469   emit_int16(0x59, (0xC0 | encode));
10470 }
10471 
10472 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10473   assert(VM_Version::supports_avx2(), "");
10474   assert(dst != xnoreg, "sanity");
10475   InstructionMark im(this);
10476   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10477   attributes.set_rex_vex_w_reverted();
10478   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10479   // swap src<->dst for encoding
10480   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10481   emit_int8(0x59);
10482   emit_operand(dst, src, 0);
10483 }
10484 
10485 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10486   assert(vector_len != Assembler::AVX_128bit, "");
10487   assert(VM_Version::supports_evex(), "");
10488   assert(dst != xnoreg, "sanity");
10489   InstructionMark im(this);
10490   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10491   attributes.set_rex_vex_w_reverted();
10492   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10493   // swap src<->dst for encoding
10494   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10495   emit_int8(0x5A);
10496   emit_operand(dst, src, 0);
10497 }
10498 
10499 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10500   assert(vector_len != Assembler::AVX_128bit, "");
10501   assert(VM_Version::supports_avx512dq(), "");
10502   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10503   attributes.set_rex_vex_w_reverted();
10504   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10505   emit_int16(0x5A, (0xC0 | encode));
10506 }
10507 
10508 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10509   assert(vector_len != Assembler::AVX_128bit, "");
10510   assert(VM_Version::supports_avx512dq(), "");
10511   assert(dst != xnoreg, "sanity");
10512   InstructionMark im(this);
10513   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10514   attributes.set_rex_vex_w_reverted();
10515   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10516   // swap src<->dst for encoding
10517   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10518   emit_int8(0x5A);
10519   emit_operand(dst, src, 0);
10520 }
10521 
10522 // scalar single/double precision replicate
10523 
10524 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10525 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10526   assert(VM_Version::supports_avx2(), "");
10527   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10528   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10529   emit_int16(0x18, (0xC0 | encode));
10530 }
10531 
10532 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10533   assert(VM_Version::supports_avx(), "");
10534   assert(dst != xnoreg, "sanity");
10535   InstructionMark im(this);
10536   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10537   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10538   // swap src<->dst for encoding
10539   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10540   emit_int8(0x18);
10541   emit_operand(dst, src, 0);
10542 }
10543 
10544 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10545 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10546   assert(VM_Version::supports_avx2(), "");
10547   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10548   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10549   attributes.set_rex_vex_w_reverted();
10550   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10551   emit_int16(0x19, (0xC0 | encode));
10552 }
10553 
10554 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10555   assert(VM_Version::supports_avx(), "");
10556   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10557   assert(dst != xnoreg, "sanity");
10558   InstructionMark im(this);
10559   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10560   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10561   attributes.set_rex_vex_w_reverted();
10562   // swap src<->dst for encoding
10563   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10564   emit_int8(0x19);
10565   emit_operand(dst, src, 0);
10566 }
10567 
10568 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10569   assert(VM_Version::supports_avx(), "");
10570   assert(vector_len == AVX_256bit, "");
10571   assert(dst != xnoreg, "sanity");
10572   InstructionMark im(this);
10573   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10574   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10575   // swap src<->dst for encoding
10576   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10577   emit_int8(0x1A);
10578   emit_operand(dst, src, 0);
10579 }
10580 
10581 // gpr source broadcast forms
10582 
10583 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10584 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10585   assert(VM_Version::supports_avx512bw(), "");
10586   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10587   attributes.set_is_evex_instruction();
10588   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10589   emit_int16(0x7A, (0xC0 | encode));
10590 }
10591 
10592 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10593 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10594   assert(VM_Version::supports_avx512bw(), "");
10595   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10596   attributes.set_is_evex_instruction();
10597   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10598   emit_int16(0x7B, (0xC0 | encode));
10599 }
10600 
10601 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10602 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10603   assert(VM_Version::supports_evex(), "");
10604   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10605   attributes.set_is_evex_instruction();
10606   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10607   emit_int16(0x7C, (0xC0 | encode));
10608 }
10609 
10610 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10611 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10612   assert(VM_Version::supports_evex(), "");
10613   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10614   attributes.set_is_evex_instruction();
10615   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10616   emit_int16(0x7C, (0xC0 | encode));
10617 }
10618 
10619 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10620   assert(VM_Version::supports_avx2(), "");
10621   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10622   assert(dst != xnoreg, "sanity");
10623   assert(src.isxmmindex(),"expected to be xmm index");
10624   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10625   InstructionMark im(this);
10626   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10627   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10628   emit_int8((unsigned char)0x90);
10629   emit_operand(dst, src, 0);
10630 }
10631 
10632 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10633   assert(VM_Version::supports_avx2(), "");
10634   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10635   assert(dst != xnoreg, "sanity");
10636   assert(src.isxmmindex(),"expected to be xmm index");
10637   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10638   InstructionMark im(this);
10639   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10640   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10641   emit_int8((unsigned char)0x90);
10642   emit_operand(dst, src, 0);
10643 }
10644 
10645 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10646   assert(VM_Version::supports_avx2(), "");
10647   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10648   assert(dst != xnoreg, "sanity");
10649   assert(src.isxmmindex(),"expected to be xmm index");
10650   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10651   InstructionMark im(this);
10652   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10653   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10654   emit_int8((unsigned char)0x92);
10655   emit_operand(dst, src, 0);
10656 }
10657 
10658 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10659   assert(VM_Version::supports_avx2(), "");
10660   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10661   assert(dst != xnoreg, "sanity");
10662   assert(src.isxmmindex(),"expected to be xmm index");
10663   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10664   InstructionMark im(this);
10665   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10666   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10667   emit_int8((unsigned char)0x92);
10668   emit_operand(dst, src, 0);
10669 }
10670 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10671   assert(VM_Version::supports_evex(), "");
10672   assert(dst != xnoreg, "sanity");
10673   assert(src.isxmmindex(),"expected to be xmm index");
10674   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10675   assert(mask != k0, "instruction will #UD if mask is in k0");
10676   InstructionMark im(this);
10677   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10678   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10679   attributes.reset_is_clear_context();
10680   attributes.set_embedded_opmask_register_specifier(mask);
10681   attributes.set_is_evex_instruction();
10682   // swap src<->dst for encoding
10683   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10684   emit_int8((unsigned char)0x90);
10685   emit_operand(dst, src, 0);
10686 }
10687 
10688 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10689   assert(VM_Version::supports_evex(), "");
10690   assert(dst != xnoreg, "sanity");
10691   assert(src.isxmmindex(),"expected to be xmm index");
10692   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10693   assert(mask != k0, "instruction will #UD if mask is in k0");
10694   InstructionMark im(this);
10695   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10696   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10697   attributes.reset_is_clear_context();
10698   attributes.set_embedded_opmask_register_specifier(mask);
10699   attributes.set_is_evex_instruction();
10700   // swap src<->dst for encoding
10701   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10702   emit_int8((unsigned char)0x90);
10703   emit_operand(dst, src, 0);
10704 }
10705 
10706 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10707   assert(VM_Version::supports_evex(), "");
10708   assert(dst != xnoreg, "sanity");
10709   assert(src.isxmmindex(),"expected to be xmm index");
10710   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10711   assert(mask != k0, "instruction will #UD if mask is in k0");
10712   InstructionMark im(this);
10713   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10714   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10715   attributes.reset_is_clear_context();
10716   attributes.set_embedded_opmask_register_specifier(mask);
10717   attributes.set_is_evex_instruction();
10718   // swap src<->dst for encoding
10719   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10720   emit_int8((unsigned char)0x92);
10721   emit_operand(dst, src, 0);
10722 }
10723 
10724 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10725   assert(VM_Version::supports_evex(), "");
10726   assert(dst != xnoreg, "sanity");
10727   assert(src.isxmmindex(),"expected to be xmm index");
10728   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10729   assert(mask != k0, "instruction will #UD if mask is in k0");
10730   InstructionMark im(this);
10731   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10732   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10733   attributes.reset_is_clear_context();
10734   attributes.set_embedded_opmask_register_specifier(mask);
10735   attributes.set_is_evex_instruction();
10736   // swap src<->dst for encoding
10737   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10738   emit_int8((unsigned char)0x92);
10739   emit_operand(dst, src, 0);
10740 }
10741 
10742 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10743   assert(VM_Version::supports_evex(), "");
10744   assert(mask != k0, "instruction will #UD if mask is in k0");
10745   InstructionMark im(this);
10746   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10747   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10748   attributes.reset_is_clear_context();
10749   attributes.set_embedded_opmask_register_specifier(mask);
10750   attributes.set_is_evex_instruction();
10751   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10752   emit_int8((unsigned char)0xA0);
10753   emit_operand(src, dst, 0);
10754 }
10755 
10756 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10757   assert(VM_Version::supports_evex(), "");
10758   assert(mask != k0, "instruction will #UD if mask is in k0");
10759   InstructionMark im(this);
10760   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10761   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10762   attributes.reset_is_clear_context();
10763   attributes.set_embedded_opmask_register_specifier(mask);
10764   attributes.set_is_evex_instruction();
10765   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10766   emit_int8((unsigned char)0xA0);
10767   emit_operand(src, dst, 0);
10768 }
10769 
10770 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10771   assert(VM_Version::supports_evex(), "");
10772   assert(mask != k0, "instruction will #UD if mask is in k0");
10773   InstructionMark im(this);
10774   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10775   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10776   attributes.reset_is_clear_context();
10777   attributes.set_embedded_opmask_register_specifier(mask);
10778   attributes.set_is_evex_instruction();
10779   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10780   emit_int8((unsigned char)0xA2);
10781   emit_operand(src, dst, 0);
10782 }
10783 
10784 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10785   assert(VM_Version::supports_evex(), "");
10786   assert(mask != k0, "instruction will #UD if mask is in k0");
10787   InstructionMark im(this);
10788   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10789   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10790   attributes.reset_is_clear_context();
10791   attributes.set_embedded_opmask_register_specifier(mask);
10792   attributes.set_is_evex_instruction();
10793   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10794   emit_int8((unsigned char)0xA2);
10795   emit_operand(src, dst, 0);
10796 }
10797 // Carry-Less Multiplication Quadword
10798 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10799   assert(VM_Version::supports_clmul(), "");
10800   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10801   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10802   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10803 }
10804 
10805 // Carry-Less Multiplication Quadword
10806 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10807   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10808   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10809   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10810   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10811 }
10812 
10813 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10814   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10815   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10816   attributes.set_is_evex_instruction();
10817   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10818   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10819 }
10820 
10821 void Assembler::vzeroupper_uncached() {
10822   if (VM_Version::supports_vzeroupper()) {
10823     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10824     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10825     emit_int8(0x77);
10826   }
10827 }
10828 
10829 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10830   // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10831   assert(VM_Version::supports_evex(), "");
10832   assert(VM_Version::supports_avx512dq(), "");
10833   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10834   attributes.set_is_evex_instruction();
10835   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10836   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10837 }
10838 
10839 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10840   // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10841   assert(VM_Version::supports_evex(), "");
10842   assert(VM_Version::supports_avx512dq(), "");
10843   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10844   attributes.set_is_evex_instruction();
10845   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10846   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10847 }
10848 
10849 void Assembler::fld_x(Address adr) {
10850   InstructionMark im(this);
10851   emit_int8((unsigned char)0xDB);
10852   emit_operand32(rbp, adr, 0);
10853 }
10854 
10855 void Assembler::fstp_x(Address adr) {
10856   InstructionMark im(this);
10857   emit_int8((unsigned char)0xDB);
10858   emit_operand32(rdi, adr, 0);
10859 }
10860 
10861 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10862   assert(reg->encoding() < 8, "no extended registers");
10863   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10864   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10865 }
10866 
10867 #ifndef _LP64
10868 // 32bit only pieces of the assembler
10869 
10870 void Assembler::emms() {
10871   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10872   emit_int16(0x0F, 0x77);
10873 }
10874 
10875 void Assembler::vzeroupper() {
10876   vzeroupper_uncached();
10877 }
10878 
10879 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10880   // NO PREFIX AS NEVER 64BIT
10881   InstructionMark im(this);
10882   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10883   emit_data(imm32, rspec, 0);
10884 }
10885 
10886 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10887   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10888   InstructionMark im(this);
10889   emit_int8((unsigned char)0x81);
10890   emit_operand(rdi, src1, 4);
10891   emit_data(imm32, rspec, 0);
10892 }
10893 
10894 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10895 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10896 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10897 void Assembler::cmpxchg8(Address adr) {
10898   InstructionMark im(this);
10899   emit_int16(0x0F, (unsigned char)0xC7);
10900   emit_operand(rcx, adr, 0);
10901 }
10902 
10903 void Assembler::decl(Register dst) {
10904   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10905  emit_int8(0x48 | dst->encoding());
10906 }
10907 
10908 // 64bit doesn't use the x87
10909 
10910 void Assembler::emit_farith(int b1, int b2, int i) {
10911   assert(isByte(b1) && isByte(b2), "wrong opcode");
10912   assert(0 <= i &&  i < 8, "illegal stack offset");
10913   emit_int16(b1, b2 + i);
10914 }
10915 
10916 void Assembler::fabs() {
10917   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10918 }
10919 
10920 void Assembler::fadd(int i) {
10921   emit_farith(0xD8, 0xC0, i);
10922 }
10923 
10924 void Assembler::fadd_d(Address src) {
10925   InstructionMark im(this);
10926   emit_int8((unsigned char)0xDC);
10927   emit_operand32(rax, src, 0);
10928 }
10929 
10930 void Assembler::fadd_s(Address src) {
10931   InstructionMark im(this);
10932   emit_int8((unsigned char)0xD8);
10933   emit_operand32(rax, src, 0);
10934 }
10935 
10936 void Assembler::fadda(int i) {
10937   emit_farith(0xDC, 0xC0, i);
10938 }
10939 
10940 void Assembler::faddp(int i) {
10941   emit_farith(0xDE, 0xC0, i);
10942 }
10943 
10944 void Assembler::fchs() {
10945   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10946 }
10947 
10948 void Assembler::fcom(int i) {
10949   emit_farith(0xD8, 0xD0, i);
10950 }
10951 
10952 void Assembler::fcomp(int i) {
10953   emit_farith(0xD8, 0xD8, i);
10954 }
10955 
10956 void Assembler::fcomp_d(Address src) {
10957   InstructionMark im(this);
10958   emit_int8((unsigned char)0xDC);
10959   emit_operand32(rbx, src, 0);
10960 }
10961 
10962 void Assembler::fcomp_s(Address src) {
10963   InstructionMark im(this);
10964   emit_int8((unsigned char)0xD8);
10965   emit_operand32(rbx, src, 0);
10966 }
10967 
10968 void Assembler::fcompp() {
10969   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10970 }
10971 
10972 void Assembler::fcos() {
10973   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10974 }
10975 
10976 void Assembler::fdecstp() {
10977   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10978 }
10979 
10980 void Assembler::fdiv(int i) {
10981   emit_farith(0xD8, 0xF0, i);
10982 }
10983 
10984 void Assembler::fdiv_d(Address src) {
10985   InstructionMark im(this);
10986   emit_int8((unsigned char)0xDC);
10987   emit_operand32(rsi, src, 0);
10988 }
10989 
10990 void Assembler::fdiv_s(Address src) {
10991   InstructionMark im(this);
10992   emit_int8((unsigned char)0xD8);
10993   emit_operand32(rsi, src, 0);
10994 }
10995 
10996 void Assembler::fdiva(int i) {
10997   emit_farith(0xDC, 0xF8, i);
10998 }
10999 
11000 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
11001 //       is erroneous for some of the floating-point instructions below.
11002 
11003 void Assembler::fdivp(int i) {
11004   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
11005 }
11006 
11007 void Assembler::fdivr(int i) {
11008   emit_farith(0xD8, 0xF8, i);
11009 }
11010 
11011 void Assembler::fdivr_d(Address src) {
11012   InstructionMark im(this);
11013   emit_int8((unsigned char)0xDC);
11014   emit_operand32(rdi, src, 0);
11015 }
11016 
11017 void Assembler::fdivr_s(Address src) {
11018   InstructionMark im(this);
11019   emit_int8((unsigned char)0xD8);
11020   emit_operand32(rdi, src, 0);
11021 }
11022 
11023 void Assembler::fdivra(int i) {
11024   emit_farith(0xDC, 0xF0, i);
11025 }
11026 
11027 void Assembler::fdivrp(int i) {
11028   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
11029 }
11030 
11031 void Assembler::ffree(int i) {
11032   emit_farith(0xDD, 0xC0, i);
11033 }
11034 
11035 void Assembler::fild_d(Address adr) {
11036   InstructionMark im(this);
11037   emit_int8((unsigned char)0xDF);
11038   emit_operand32(rbp, adr, 0);
11039 }
11040 
11041 void Assembler::fild_s(Address adr) {
11042   InstructionMark im(this);
11043   emit_int8((unsigned char)0xDB);
11044   emit_operand32(rax, adr, 0);
11045 }
11046 
11047 void Assembler::fincstp() {
11048   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
11049 }
11050 
11051 void Assembler::finit() {
11052   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
11053 }
11054 
11055 void Assembler::fist_s(Address adr) {
11056   InstructionMark im(this);
11057   emit_int8((unsigned char)0xDB);
11058   emit_operand32(rdx, adr, 0);
11059 }
11060 
11061 void Assembler::fistp_d(Address adr) {
11062   InstructionMark im(this);
11063   emit_int8((unsigned char)0xDF);
11064   emit_operand32(rdi, adr, 0);
11065 }
11066 
11067 void Assembler::fistp_s(Address adr) {
11068   InstructionMark im(this);
11069   emit_int8((unsigned char)0xDB);
11070   emit_operand32(rbx, adr, 0);
11071 }
11072 
11073 void Assembler::fld1() {
11074   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
11075 }
11076 
11077 void Assembler::fld_d(Address adr) {
11078   InstructionMark im(this);
11079   emit_int8((unsigned char)0xDD);
11080   emit_operand32(rax, adr, 0);
11081 }
11082 
11083 void Assembler::fld_s(Address adr) {
11084   InstructionMark im(this);
11085   emit_int8((unsigned char)0xD9);
11086   emit_operand32(rax, adr, 0);
11087 }
11088 
11089 
11090 void Assembler::fld_s(int index) {
11091   emit_farith(0xD9, 0xC0, index);
11092 }
11093 
11094 void Assembler::fldcw(Address src) {
11095   InstructionMark im(this);
11096   emit_int8((unsigned char)0xD9);
11097   emit_operand32(rbp, src, 0);
11098 }
11099 
11100 void Assembler::fldenv(Address src) {
11101   InstructionMark im(this);
11102   emit_int8((unsigned char)0xD9);
11103   emit_operand32(rsp, src, 0);
11104 }
11105 
11106 void Assembler::fldlg2() {
11107   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
11108 }
11109 
11110 void Assembler::fldln2() {
11111   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
11112 }
11113 
11114 void Assembler::fldz() {
11115   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
11116 }
11117 
11118 void Assembler::flog() {
11119   fldln2();
11120   fxch();
11121   fyl2x();
11122 }
11123 
11124 void Assembler::flog10() {
11125   fldlg2();
11126   fxch();
11127   fyl2x();
11128 }
11129 
11130 void Assembler::fmul(int i) {
11131   emit_farith(0xD8, 0xC8, i);
11132 }
11133 
11134 void Assembler::fmul_d(Address src) {
11135   InstructionMark im(this);
11136   emit_int8((unsigned char)0xDC);
11137   emit_operand32(rcx, src, 0);
11138 }
11139 
11140 void Assembler::fmul_s(Address src) {
11141   InstructionMark im(this);
11142   emit_int8((unsigned char)0xD8);
11143   emit_operand32(rcx, src, 0);
11144 }
11145 
11146 void Assembler::fmula(int i) {
11147   emit_farith(0xDC, 0xC8, i);
11148 }
11149 
11150 void Assembler::fmulp(int i) {
11151   emit_farith(0xDE, 0xC8, i);
11152 }
11153 
11154 void Assembler::fnsave(Address dst) {
11155   InstructionMark im(this);
11156   emit_int8((unsigned char)0xDD);
11157   emit_operand32(rsi, dst, 0);
11158 }
11159 
11160 void Assembler::fnstcw(Address src) {
11161   InstructionMark im(this);
11162   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
11163   emit_operand32(rdi, src, 0);
11164 }
11165 
11166 void Assembler::fnstsw_ax() {
11167   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
11168 }
11169 
11170 void Assembler::fprem() {
11171   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
11172 }
11173 
11174 void Assembler::fprem1() {
11175   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
11176 }
11177 
11178 void Assembler::frstor(Address src) {
11179   InstructionMark im(this);
11180   emit_int8((unsigned char)0xDD);
11181   emit_operand32(rsp, src, 0);
11182 }
11183 
11184 void Assembler::fsin() {
11185   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
11186 }
11187 
11188 void Assembler::fsqrt() {
11189   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11190 }
11191 
11192 void Assembler::fst_d(Address adr) {
11193   InstructionMark im(this);
11194   emit_int8((unsigned char)0xDD);
11195   emit_operand32(rdx, adr, 0);
11196 }
11197 
11198 void Assembler::fst_s(Address adr) {
11199   InstructionMark im(this);
11200   emit_int8((unsigned char)0xD9);
11201   emit_operand32(rdx, adr, 0);
11202 }
11203 
11204 void Assembler::fstp_d(Address adr) {
11205   InstructionMark im(this);
11206   emit_int8((unsigned char)0xDD);
11207   emit_operand32(rbx, adr, 0);
11208 }
11209 
11210 void Assembler::fstp_d(int index) {
11211   emit_farith(0xDD, 0xD8, index);
11212 }
11213 
11214 void Assembler::fstp_s(Address adr) {
11215   InstructionMark im(this);
11216   emit_int8((unsigned char)0xD9);
11217   emit_operand32(rbx, adr, 0);
11218 }
11219 
11220 void Assembler::fsub(int i) {
11221   emit_farith(0xD8, 0xE0, i);
11222 }
11223 
11224 void Assembler::fsub_d(Address src) {
11225   InstructionMark im(this);
11226   emit_int8((unsigned char)0xDC);
11227   emit_operand32(rsp, src, 0);
11228 }
11229 
11230 void Assembler::fsub_s(Address src) {
11231   InstructionMark im(this);
11232   emit_int8((unsigned char)0xD8);
11233   emit_operand32(rsp, src, 0);
11234 }
11235 
11236 void Assembler::fsuba(int i) {
11237   emit_farith(0xDC, 0xE8, i);
11238 }
11239 
11240 void Assembler::fsubp(int i) {
11241   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11242 }
11243 
11244 void Assembler::fsubr(int i) {
11245   emit_farith(0xD8, 0xE8, i);
11246 }
11247 
11248 void Assembler::fsubr_d(Address src) {
11249   InstructionMark im(this);
11250   emit_int8((unsigned char)0xDC);
11251   emit_operand32(rbp, src, 0);
11252 }
11253 
11254 void Assembler::fsubr_s(Address src) {
11255   InstructionMark im(this);
11256   emit_int8((unsigned char)0xD8);
11257   emit_operand32(rbp, src, 0);
11258 }
11259 
11260 void Assembler::fsubra(int i) {
11261   emit_farith(0xDC, 0xE0, i);
11262 }
11263 
11264 void Assembler::fsubrp(int i) {
11265   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11266 }
11267 
11268 void Assembler::ftan() {
11269   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11270 }
11271 
11272 void Assembler::ftst() {
11273   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11274 }
11275 
11276 void Assembler::fucomi(int i) {
11277   // make sure the instruction is supported (introduced for P6, together with cmov)
11278   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11279   emit_farith(0xDB, 0xE8, i);
11280 }
11281 
11282 void Assembler::fucomip(int i) {
11283   // make sure the instruction is supported (introduced for P6, together with cmov)
11284   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11285   emit_farith(0xDF, 0xE8, i);
11286 }
11287 
11288 void Assembler::fwait() {
11289   emit_int8((unsigned char)0x9B);
11290 }
11291 
11292 void Assembler::fxch(int i) {
11293   emit_farith(0xD9, 0xC8, i);
11294 }
11295 
11296 void Assembler::fyl2x() {
11297   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11298 }
11299 
11300 void Assembler::frndint() {
11301   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11302 }
11303 
11304 void Assembler::f2xm1() {
11305   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11306 }
11307 
11308 void Assembler::fldl2e() {
11309   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11310 }
11311 #endif // !_LP64
11312 
11313 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11314 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11315 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11316 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
11317 
11318 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11319 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11320   if (pre > 0) {
11321     emit_int8(simd_pre[pre]);
11322   }
11323   if (rex_w) {
11324     prefixq(adr, xreg);
11325   } else {
11326     prefix(adr, xreg);
11327   }
11328   if (opc > 0) {
11329     emit_int8(0x0F);
11330     int opc2 = simd_opc[opc];
11331     if (opc2 > 0) {
11332       emit_int8(opc2);
11333     }
11334   }
11335 }
11336 
11337 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11338   if (pre > 0) {
11339     emit_int8(simd_pre[pre]);
11340   }
11341   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11342   if (opc > 0) {
11343     emit_int8(0x0F);
11344     int opc2 = simd_opc[opc];
11345     if (opc2 > 0) {
11346       emit_int8(opc2);
11347     }
11348   }
11349   return encode;
11350 }
11351 
11352 
11353 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11354   int vector_len = _attributes->get_vector_len();
11355   bool vex_w = _attributes->is_rex_vex_w();
11356   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11357     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11358     byte1 = (~byte1) & 0xE0;
11359     byte1 |= opc;
11360 
11361     int byte2 = ((~nds_enc) & 0xf) << 3;
11362     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11363 
11364     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11365   } else {
11366     int byte1 = vex_r ? VEX_R : 0;
11367     byte1 = (~byte1) & 0x80;
11368     byte1 |= ((~nds_enc) & 0xf) << 3;
11369     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11370     emit_int16((unsigned char)VEX_2bytes, byte1);
11371   }
11372 }
11373 
11374 // This is a 4 byte encoding
11375 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11376   // EVEX 0x62 prefix
11377   // byte1 = EVEX_4bytes;
11378 
11379   bool vex_w = _attributes->is_rex_vex_w();
11380   int evex_encoding = (vex_w ? VEX_W : 0);
11381   // EVEX.b is not currently used for broadcast of single element or data rounding modes
11382   _attributes->set_evex_encoding(evex_encoding);
11383 
11384   // P0: byte 2, initialized to RXBR`00mm
11385   // instead of not'd
11386   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11387   byte2 = (~byte2) & 0xF0;
11388   // confine opc opcode extensions in mm bits to lower two bits
11389   // of form {0F, 0F_38, 0F_3A}
11390   byte2 |= opc;
11391 
11392   // P1: byte 3 as Wvvvv1pp
11393   int byte3 = ((~nds_enc) & 0xf) << 3;
11394   // p[10] is always 1
11395   byte3 |= EVEX_F;
11396   byte3 |= (vex_w & 1) << 7;
11397   // confine pre opcode extensions in pp bits to lower two bits
11398   // of form {66, F3, F2}
11399   byte3 |= pre;
11400 
11401   // P2: byte 4 as zL'Lbv'aaa
11402   // kregs are implemented in the low 3 bits as aaa
11403   int byte4 = (_attributes->is_no_reg_mask()) ?
11404               0 :
11405               _attributes->get_embedded_opmask_register_specifier();
11406   // EVEX.v` for extending EVEX.vvvv or VIDX
11407   byte4 |= (evex_v ? 0: EVEX_V);
11408   // third EXEC.b for broadcast actions
11409   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11410   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11411   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11412   // last is EVEX.z for zero/merge actions
11413   if (_attributes->is_no_reg_mask() == false &&
11414       _attributes->get_embedded_opmask_register_specifier() != 0) {
11415     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11416   }
11417 
11418   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11419 }
11420 
11421 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11422   bool vex_r = (xreg_enc & 8) == 8;
11423   bool vex_b = adr.base_needs_rex();
11424   bool vex_x;
11425   if (adr.isxmmindex()) {
11426     vex_x = adr.xmmindex_needs_rex();
11427   } else {
11428     vex_x = adr.index_needs_rex();
11429   }
11430   set_attributes(attributes);
11431   attributes->set_current_assembler(this);
11432 
11433   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11434   // is allowed in legacy mode and has resources which will fit in it.
11435   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11436   if (!attributes->is_legacy_mode()) {
11437     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11438       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11439           attributes->set_is_legacy_mode();
11440       }
11441     }
11442   }
11443 
11444   if (UseAVX > 2) {
11445     assert(((!attributes->uses_vl()) ||
11446             (attributes->get_vector_len() == AVX_512bit) ||
11447             (!_legacy_mode_vl) ||
11448             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11449     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11450   }
11451 
11452   clear_managed();
11453   if (UseAVX > 2 && !attributes->is_legacy_mode())
11454   {
11455     bool evex_r = (xreg_enc >= 16);
11456     bool evex_v;
11457     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11458     if (adr.isxmmindex())  {
11459       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11460     } else {
11461       evex_v = (nds_enc >= 16);
11462     }
11463     attributes->set_is_evex_instruction();
11464     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11465   } else {
11466     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11467       attributes->set_rex_vex_w(false);
11468     }
11469     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11470   }
11471 }
11472 
11473 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11474   bool vex_r = (dst_enc & 8) == 8;
11475   bool vex_b = (src_enc & 8) == 8;
11476   bool vex_x = false;
11477   set_attributes(attributes);
11478   attributes->set_current_assembler(this);
11479 
11480   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11481   // is allowed in legacy mode and has resources which will fit in it.
11482   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11483   if (!attributes->is_legacy_mode()) {
11484     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11485       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11486           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11487           attributes->set_is_legacy_mode();
11488       }
11489     }
11490   }
11491 
11492   if (UseAVX > 2) {
11493     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11494     // Instruction with uses_vl true are vector instructions
11495     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11496     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11497     // Rest all should have legacy_mode set as true
11498     assert(((!attributes->uses_vl()) ||
11499             (attributes->get_vector_len() == AVX_512bit) ||
11500             (!_legacy_mode_vl) ||
11501             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11502     // Instruction with legacy_mode true should have dst, nds and src < 15
11503     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11504   }
11505 
11506   clear_managed();
11507   if (UseAVX > 2 && !attributes->is_legacy_mode())
11508   {
11509     bool evex_r = (dst_enc >= 16);
11510     bool evex_v = (nds_enc >= 16);
11511     // can use vex_x as bank extender on rm encoding
11512     vex_x = (src_enc >= 16);
11513     attributes->set_is_evex_instruction();
11514     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11515   } else {
11516     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11517       attributes->set_rex_vex_w(false);
11518     }
11519     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11520   }
11521 
11522   // return modrm byte components for operands
11523   return (((dst_enc & 7) << 3) | (src_enc & 7));
11524 }
11525 
11526 
11527 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11528                             VexOpcode opc, InstructionAttr *attributes) {
11529   if (UseAVX > 0) {
11530     int xreg_enc = xreg->encoding();
11531     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11532     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11533   } else {
11534     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11535     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11536   }
11537 }
11538 
11539 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11540                                       VexOpcode opc, InstructionAttr *attributes) {
11541   int dst_enc = dst->encoding();
11542   int src_enc = src->encoding();
11543   if (UseAVX > 0) {
11544     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11545     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11546   } else {
11547     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11548     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11549   }
11550 }
11551 
11552 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11553   assert(VM_Version::supports_avx(), "");
11554   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11555   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11556   emit_int16(0x5F, (0xC0 | encode));
11557 }
11558 
11559 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11560   assert(VM_Version::supports_avx(), "");
11561   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11562   attributes.set_rex_vex_w_reverted();
11563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11564   emit_int16(0x5F, (0xC0 | encode));
11565 }
11566 
11567 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11568   assert(VM_Version::supports_avx(), "");
11569   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11570   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11571   emit_int16(0x5D, (0xC0 | encode));
11572 }
11573 
11574 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11575   assert(VM_Version::supports_avx(), "");
11576   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11577   attributes.set_rex_vex_w_reverted();
11578   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11579   emit_int16(0x5D, (0xC0 | encode));
11580 }
11581 
11582 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11583   assert(VM_Version::supports_avx(), "");
11584   assert(vector_len <= AVX_256bit, "");
11585   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11586   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11587   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11588 }
11589 
11590 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11591   assert(VM_Version::supports_avx(), "");
11592   assert(vector_len <= AVX_256bit, "");
11593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11595   int src2_enc = src2->encoding();
11596   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11597 }
11598 
11599 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11600   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11601   assert(vector_len <= AVX_256bit, "");
11602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11603   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11604   int src2_enc = src2->encoding();
11605   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11606 }
11607 
11608 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11609   assert(VM_Version::supports_avx2(), "");
11610   assert(vector_len <= AVX_256bit, "");
11611   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11613   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11614 }
11615 
11616 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11617   assert(VM_Version::supports_avx(), "");
11618   assert(vector_len <= AVX_256bit, "");
11619   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11620   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11621   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11622 }
11623 
11624 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11625                         ComparisonPredicateFP comparison, int vector_len) {
11626   assert(VM_Version::supports_evex(), "");
11627   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11628   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11629   attributes.set_is_evex_instruction();
11630   attributes.set_embedded_opmask_register_specifier(mask);
11631   attributes.reset_is_clear_context();
11632   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11633   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11634 }
11635 
11636 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11637                         ComparisonPredicateFP comparison, int vector_len) {
11638   assert(VM_Version::supports_evex(), "");
11639   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11640   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11641   attributes.set_is_evex_instruction();
11642   attributes.set_embedded_opmask_register_specifier(mask);
11643   attributes.reset_is_clear_context();
11644   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11645   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11646 }
11647 
11648 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11649   assert(VM_Version::supports_sse4_1(), "");
11650   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11651   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11652   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11653   emit_int16(0x14, (0xC0 | encode));
11654 }
11655 
11656 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11657   assert(VM_Version::supports_sse4_1(), "");
11658   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11659   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11660   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11661   emit_int16(0x15, (0xC0 | encode));
11662 }
11663 
11664 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11665   assert(VM_Version::supports_sse4_1(), "");
11666   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11667   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11668   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11669   emit_int16(0x10, (0xC0 | encode));
11670 }
11671 
11672 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11673   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11674   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11675   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11676   int src2_enc = src2->encoding();
11677   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11678 }
11679 
11680 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11681   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11682   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11683   emit_int24(0x0C, (0xC0 | encode), imm8);
11684 }
11685 
11686 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11687   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11688   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11689   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11690   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11691   emit_int16(0x64, (0xC0 | encode));
11692 }
11693 
11694 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11695   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11696   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11697   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11698   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11699   emit_int16(0x65, (0xC0 | encode));
11700 }
11701 
11702 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11703   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11704   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11705   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11706   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11707   emit_int16(0x66, (0xC0 | encode));
11708 }
11709 
11710 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11711   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11712   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11713   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11714   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11715   emit_int16(0x37, (0xC0 | encode));
11716 }
11717 
11718 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11719                         int comparison, bool is_signed, int vector_len) {
11720   assert(VM_Version::supports_evex(), "");
11721   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11722   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11723   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11724   attributes.set_is_evex_instruction();
11725   attributes.set_embedded_opmask_register_specifier(mask);
11726   attributes.reset_is_clear_context();
11727   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11728   int opcode = is_signed ? 0x1F : 0x1E;
11729   emit_int24(opcode, (0xC0 | encode), comparison);
11730 }
11731 
11732 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11733                         int comparison, bool is_signed, int vector_len) {
11734   assert(VM_Version::supports_evex(), "");
11735   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11736   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11737   InstructionMark im(this);
11738   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11739   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11740   attributes.set_is_evex_instruction();
11741   attributes.set_embedded_opmask_register_specifier(mask);
11742   attributes.reset_is_clear_context();
11743   int dst_enc = kdst->encoding();
11744   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11745   int opcode = is_signed ? 0x1F : 0x1E;
11746   emit_int8((unsigned char)opcode);
11747   emit_operand(as_Register(dst_enc), src, 1);
11748   emit_int8((unsigned char)comparison);
11749 }
11750 
11751 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11752                         int comparison, bool is_signed, int vector_len) {
11753   assert(VM_Version::supports_evex(), "");
11754   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11755   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11756   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11757   attributes.set_is_evex_instruction();
11758   attributes.set_embedded_opmask_register_specifier(mask);
11759   attributes.reset_is_clear_context();
11760   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11761   int opcode = is_signed ? 0x1F : 0x1E;
11762   emit_int24(opcode, (0xC0 | encode), comparison);
11763 }
11764 
11765 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11766                         int comparison, bool is_signed, int vector_len) {
11767   assert(VM_Version::supports_evex(), "");
11768   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11769   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11770   InstructionMark im(this);
11771   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11772   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11773   attributes.set_is_evex_instruction();
11774   attributes.set_embedded_opmask_register_specifier(mask);
11775   attributes.reset_is_clear_context();
11776   int dst_enc = kdst->encoding();
11777   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11778   int opcode = is_signed ? 0x1F : 0x1E;
11779   emit_int8((unsigned char)opcode);
11780   emit_operand(as_Register(dst_enc), src, 1);
11781   emit_int8((unsigned char)comparison);
11782 }
11783 
11784 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11785                         int comparison, bool is_signed, int vector_len) {
11786   assert(VM_Version::supports_evex(), "");
11787   assert(VM_Version::supports_avx512bw(), "");
11788   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11789   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11790   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11791   attributes.set_is_evex_instruction();
11792   attributes.set_embedded_opmask_register_specifier(mask);
11793   attributes.reset_is_clear_context();
11794   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11795   int opcode = is_signed ? 0x3F : 0x3E;
11796   emit_int24(opcode, (0xC0 | encode), comparison);
11797 }
11798 
11799 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11800                         int comparison, bool is_signed, int vector_len) {
11801   assert(VM_Version::supports_evex(), "");
11802   assert(VM_Version::supports_avx512bw(), "");
11803   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11804   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11805   InstructionMark im(this);
11806   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11807   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11808   attributes.set_is_evex_instruction();
11809   attributes.set_embedded_opmask_register_specifier(mask);
11810   attributes.reset_is_clear_context();
11811   int dst_enc = kdst->encoding();
11812   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11813   int opcode = is_signed ? 0x3F : 0x3E;
11814   emit_int8((unsigned char)opcode);
11815   emit_operand(as_Register(dst_enc), src, 1);
11816   emit_int8((unsigned char)comparison);
11817 }
11818 
11819 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11820                         int comparison, bool is_signed, int vector_len) {
11821   assert(VM_Version::supports_evex(), "");
11822   assert(VM_Version::supports_avx512bw(), "");
11823   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11824   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11825   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11826   attributes.set_is_evex_instruction();
11827   attributes.set_embedded_opmask_register_specifier(mask);
11828   attributes.reset_is_clear_context();
11829   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11830   int opcode = is_signed ? 0x3F : 0x3E;
11831   emit_int24(opcode, (0xC0 | encode), comparison);
11832 }
11833 
11834 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11835                         int comparison, bool is_signed, int vector_len) {
11836   assert(VM_Version::supports_evex(), "");
11837   assert(VM_Version::supports_avx512bw(), "");
11838   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11839   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11840   InstructionMark im(this);
11841   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11842   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11843   attributes.set_is_evex_instruction();
11844   attributes.set_embedded_opmask_register_specifier(mask);
11845   attributes.reset_is_clear_context();
11846   int dst_enc = kdst->encoding();
11847   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11848   int opcode = is_signed ? 0x3F : 0x3E;
11849   emit_int8((unsigned char)opcode);
11850   emit_operand(as_Register(dst_enc), src, 1);
11851   emit_int8((unsigned char)comparison);
11852 }
11853 
11854 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11855   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11857   attributes.set_is_evex_instruction();
11858   attributes.set_embedded_opmask_register_specifier(mask);
11859   if (merge) {
11860     attributes.reset_is_clear_context();
11861   }
11862   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11863   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11864 }
11865 
11866 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11867   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11868   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11869   attributes.set_is_evex_instruction();
11870   attributes.set_embedded_opmask_register_specifier(mask);
11871   if (merge) {
11872     attributes.reset_is_clear_context();
11873   }
11874   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11875   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11876 }
11877 
11878 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11879   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11880   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11881   attributes.set_is_evex_instruction();
11882   attributes.set_embedded_opmask_register_specifier(mask);
11883   if (merge) {
11884     attributes.reset_is_clear_context();
11885   }
11886   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11887   emit_int16(0x14, (0xC0 | encode));
11888 }
11889 
11890 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11891   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11892   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11893   attributes.set_is_evex_instruction();
11894   attributes.set_embedded_opmask_register_specifier(mask);
11895   if (merge) {
11896     attributes.reset_is_clear_context();
11897   }
11898   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11899   emit_int16(0x14, (0xC0 | encode));
11900 }
11901 
11902 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11903   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11904   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11905   attributes.set_is_evex_instruction();
11906   attributes.set_embedded_opmask_register_specifier(mask);
11907   if (merge) {
11908     attributes.reset_is_clear_context();
11909   }
11910   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11911   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11912 }
11913 
11914 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11915   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11916   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11917   attributes.set_is_evex_instruction();
11918   attributes.set_embedded_opmask_register_specifier(mask);
11919   if (merge) {
11920     attributes.reset_is_clear_context();
11921   }
11922   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11923   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11924 }
11925 
11926 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11927   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11928   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11929   attributes.set_is_evex_instruction();
11930   attributes.set_embedded_opmask_register_specifier(mask);
11931   if (merge) {
11932     attributes.reset_is_clear_context();
11933   }
11934   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11935   emit_int16(0x15, (0xC0 | encode));
11936 }
11937 
11938 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11939   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11940   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11941   attributes.set_is_evex_instruction();
11942   attributes.set_embedded_opmask_register_specifier(mask);
11943   if (merge) {
11944     attributes.reset_is_clear_context();
11945   }
11946   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11947   emit_int16(0x15, (0xC0 | encode));
11948 }
11949 
11950 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11951   assert(VM_Version::supports_avx(), "");
11952   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11953   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11954   int mask_enc = mask->encoding();
11955   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11956 }
11957 
11958 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11959   assert(VM_Version::supports_evex(), "");
11960   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11961   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11962   attributes.set_is_evex_instruction();
11963   attributes.set_embedded_opmask_register_specifier(mask);
11964   if (merge) {
11965     attributes.reset_is_clear_context();
11966   }
11967   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11968   emit_int16(0x65, (0xC0 | encode));
11969 }
11970 
11971 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11972   assert(VM_Version::supports_evex(), "");
11973   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11974   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11975   attributes.set_is_evex_instruction();
11976   attributes.set_embedded_opmask_register_specifier(mask);
11977   if (merge) {
11978     attributes.reset_is_clear_context();
11979   }
11980   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11981   emit_int16(0x65, (0xC0 | encode));
11982 }
11983 
11984 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11985   assert(VM_Version::supports_evex(), "");
11986   assert(VM_Version::supports_avx512bw(), "");
11987   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11988   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11989   attributes.set_is_evex_instruction();
11990   attributes.set_embedded_opmask_register_specifier(mask);
11991   if (merge) {
11992     attributes.reset_is_clear_context();
11993   }
11994   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11995   emit_int16(0x66, (0xC0 | encode));
11996 }
11997 
11998 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11999   assert(VM_Version::supports_evex(), "");
12000   assert(VM_Version::supports_avx512bw(), "");
12001   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
12002   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
12003   attributes.set_is_evex_instruction();
12004   attributes.set_embedded_opmask_register_specifier(mask);
12005   if (merge) {
12006     attributes.reset_is_clear_context();
12007   }
12008   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12009   emit_int16(0x66, (0xC0 | encode));
12010 }
12011 
12012 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12013   assert(VM_Version::supports_evex(), "");
12014   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
12015   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12016   attributes.set_is_evex_instruction();
12017   attributes.set_embedded_opmask_register_specifier(mask);
12018   if (merge) {
12019     attributes.reset_is_clear_context();
12020   }
12021   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12022   emit_int16(0x64, (0xC0 | encode));
12023 }
12024 
12025 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12026   assert(VM_Version::supports_evex(), "");
12027   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
12028   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12029   attributes.set_is_evex_instruction();
12030   attributes.set_embedded_opmask_register_specifier(mask);
12031   if (merge) {
12032     attributes.reset_is_clear_context();
12033   }
12034   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12035   emit_int16(0x64, (0xC0 | encode));
12036 }
12037 
12038 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
12039   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12040   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12041   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12042   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12043 }
12044 
12045 void Assembler::pextl(Register dst, Register src1, Register src2) {
12046   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12047   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12048   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12049   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12050 }
12051 
12052 void Assembler::pdepl(Register dst, Register src1, Register src2) {
12053   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12054   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12055   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12056   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12057 }
12058 
12059 void Assembler::pextq(Register dst, Register src1, Register src2) {
12060   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12061   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12062   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12063   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12064 }
12065 
12066 void Assembler::pdepq(Register dst, Register src1, Register src2) {
12067   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12068   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12069   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12070   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12071 }
12072 
12073 void Assembler::pextl(Register dst, Register src1, Address src2) {
12074   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12075   InstructionMark im(this);
12076   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12077   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12078   emit_int8((unsigned char)0xF5);
12079   emit_operand(dst, src2, 0);
12080 }
12081 
12082 void Assembler::pdepl(Register dst, Register src1, Address src2) {
12083   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12084   InstructionMark im(this);
12085   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12086   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12087   emit_int8((unsigned char)0xF5);
12088   emit_operand(dst, src2, 0);
12089 }
12090 
12091 void Assembler::pextq(Register dst, Register src1, Address src2) {
12092   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12093   InstructionMark im(this);
12094   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12095   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12096   emit_int8((unsigned char)0xF5);
12097   emit_operand(dst, src2, 0);
12098 }
12099 
12100 void Assembler::pdepq(Register dst, Register src1, Address src2) {
12101   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12102   InstructionMark im(this);
12103   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12104   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12105   emit_int8((unsigned char)0xF5);
12106   emit_operand(dst, src2, 0);
12107 }
12108 
12109 void Assembler::sarxl(Register dst, Register src1, Register src2) {
12110   assert(VM_Version::supports_bmi2(), "");
12111   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12112   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12113   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12114 }
12115 
12116 void Assembler::sarxl(Register dst, Address src1, Register src2) {
12117   assert(VM_Version::supports_bmi2(), "");
12118   InstructionMark im(this);
12119   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12120   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12121   emit_int8((unsigned char)0xF7);
12122   emit_operand(dst, src1, 0);
12123 }
12124 
12125 void Assembler::sarxq(Register dst, Register src1, Register src2) {
12126   assert(VM_Version::supports_bmi2(), "");
12127   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12128   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12129   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12130 }
12131 
12132 void Assembler::sarxq(Register dst, Address src1, Register src2) {
12133   assert(VM_Version::supports_bmi2(), "");
12134   InstructionMark im(this);
12135   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12136   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12137   emit_int8((unsigned char)0xF7);
12138   emit_operand(dst, src1, 0);
12139 }
12140 
12141 void Assembler::shlxl(Register dst, Register src1, Register src2) {
12142   assert(VM_Version::supports_bmi2(), "");
12143   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12144   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12145   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12146 }
12147 
12148 void Assembler::shlxl(Register dst, Address src1, Register src2) {
12149   assert(VM_Version::supports_bmi2(), "");
12150   InstructionMark im(this);
12151   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12152   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12153   emit_int8((unsigned char)0xF7);
12154   emit_operand(dst, src1, 0);
12155 }
12156 
12157 void Assembler::shlxq(Register dst, Register src1, Register src2) {
12158   assert(VM_Version::supports_bmi2(), "");
12159   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12160   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12161   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12162 }
12163 
12164 void Assembler::shlxq(Register dst, Address src1, Register src2) {
12165   assert(VM_Version::supports_bmi2(), "");
12166   InstructionMark im(this);
12167   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12168   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12169   emit_int8((unsigned char)0xF7);
12170   emit_operand(dst, src1, 0);
12171 }
12172 
12173 void Assembler::shrxl(Register dst, Register src1, Register src2) {
12174   assert(VM_Version::supports_bmi2(), "");
12175   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12176   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12177   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12178 }
12179 
12180 void Assembler::shrxl(Register dst, Address src1, Register src2) {
12181   assert(VM_Version::supports_bmi2(), "");
12182   InstructionMark im(this);
12183   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12184   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12185   emit_int8((unsigned char)0xF7);
12186   emit_operand(dst, src1, 0);
12187 }
12188 
12189 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12190   assert(VM_Version::supports_bmi2(), "");
12191   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12192   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12193   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12194 }
12195 
12196 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12197   assert(VM_Version::supports_bmi2(), "");
12198   InstructionMark im(this);
12199   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12200   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12201   emit_int8((unsigned char)0xF7);
12202   emit_operand(dst, src1, 0);
12203 }
12204 
12205 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12206   assert(VM_Version::supports_avx512vldq(), "");
12207   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12208   attributes.set_is_evex_instruction();
12209   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12210   emit_int16(0x39, (0xC0 | encode));
12211 }
12212 
12213 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12214   assert(VM_Version::supports_avx512vldq(), "");
12215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12216   attributes.set_is_evex_instruction();
12217   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12218   emit_int16(0x39, (0xC0 | encode));
12219 }
12220 
12221 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12222   assert(VM_Version::supports_avx512vlbw(), "");
12223   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12224   attributes.set_is_evex_instruction();
12225   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12226   emit_int16(0x29, (0xC0 | encode));
12227 }
12228 
12229 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12230   assert(VM_Version::supports_avx512vlbw(), "");
12231   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12232   attributes.set_is_evex_instruction();
12233   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12234   emit_int16(0x29, (0xC0 | encode));
12235 }
12236 
12237 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12238   assert(VM_Version::supports_avx512vldq(), "");
12239   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12240   attributes.set_is_evex_instruction();
12241   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12242   emit_int16(0x38, (0xC0 | encode));
12243 }
12244 
12245 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12246   assert(VM_Version::supports_avx512vldq(), "");
12247   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12248   attributes.set_is_evex_instruction();
12249   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12250   emit_int16(0x38, (0xC0 | encode));
12251 }
12252 
12253 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12254   assert(VM_Version::supports_avx512vlbw(), "");
12255   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12256   attributes.set_is_evex_instruction();
12257   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12258   emit_int16(0x28, (0xC0 | encode));
12259 }
12260 
12261 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12262   assert(VM_Version::supports_avx512vlbw(), "");
12263   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12264   attributes.set_is_evex_instruction();
12265   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12266   emit_int16(0x28, (0xC0 | encode));
12267 }
12268 
12269 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12270   assert(VM_Version::supports_avx512_vbmi2(), "");
12271   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12272   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12273   attributes.set_embedded_opmask_register_specifier(mask);
12274   attributes.set_is_evex_instruction();
12275   if (merge) {
12276     attributes.reset_is_clear_context();
12277   }
12278   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12279   emit_int16((unsigned char)0x63, (0xC0 | encode));
12280 }
12281 
12282 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12283   assert(VM_Version::supports_avx512_vbmi2(), "");
12284   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12285   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12286   attributes.set_embedded_opmask_register_specifier(mask);
12287   attributes.set_is_evex_instruction();
12288   if (merge) {
12289     attributes.reset_is_clear_context();
12290   }
12291   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12292   emit_int16((unsigned char)0x63, (0xC0 | encode));
12293 }
12294 
12295 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12296   assert(VM_Version::supports_evex(), "");
12297   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12298   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12299   attributes.set_embedded_opmask_register_specifier(mask);
12300   attributes.set_is_evex_instruction();
12301   if (merge) {
12302     attributes.reset_is_clear_context();
12303   }
12304   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12305   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12306 }
12307 
12308 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12309   assert(VM_Version::supports_evex(), "");
12310   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12311   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12312   attributes.set_embedded_opmask_register_specifier(mask);
12313   attributes.set_is_evex_instruction();
12314   if (merge) {
12315     attributes.reset_is_clear_context();
12316   }
12317   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12318   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12319 }
12320 
12321 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12322   assert(VM_Version::supports_evex(), "");
12323   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12324   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12325   attributes.set_embedded_opmask_register_specifier(mask);
12326   attributes.set_is_evex_instruction();
12327   if (merge) {
12328     attributes.reset_is_clear_context();
12329   }
12330   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12331   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12332 }
12333 
12334 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12335   assert(VM_Version::supports_evex(), "");
12336   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12337   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12338   attributes.set_embedded_opmask_register_specifier(mask);
12339   attributes.set_is_evex_instruction();
12340   if (merge) {
12341     attributes.reset_is_clear_context();
12342   }
12343   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12344   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12345 }
12346 
12347 #ifndef _LP64
12348 
12349 void Assembler::incl(Register dst) {
12350   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12351   emit_int8(0x40 | dst->encoding());
12352 }
12353 
12354 void Assembler::lea(Register dst, Address src) {
12355   leal(dst, src);
12356 }
12357 
12358 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12359   InstructionMark im(this);
12360   emit_int8((unsigned char)0xC7);
12361   emit_operand(rax, dst, 4);
12362   emit_data((int)imm32, rspec, 0);
12363 }
12364 
12365 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12366   InstructionMark im(this);
12367   int encode = prefix_and_encode(dst->encoding());
12368   emit_int8((0xB8 | encode));
12369   emit_data((int)imm32, rspec, 0);
12370 }
12371 
12372 void Assembler::popa() { // 32bit
12373   emit_int8(0x61);
12374 }
12375 
12376 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12377   InstructionMark im(this);
12378   emit_int8(0x68);
12379   emit_data(imm32, rspec, 0);
12380 }
12381 
12382 void Assembler::pusha() { // 32bit
12383   emit_int8(0x60);
12384 }
12385 
12386 void Assembler::set_byte_if_not_zero(Register dst) {
12387   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12388 }
12389 
12390 #else // LP64
12391 
12392 // 64bit only pieces of the assembler
12393 
12394 void Assembler::set_byte_if_not_zero(Register dst) {
12395   int enc = prefix_and_encode(dst->encoding(), true);
12396   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12397 }
12398 
12399 // This should only be used by 64bit instructions that can use rip-relative
12400 // it cannot be used by instructions that want an immediate value.
12401 
12402 // Determine whether an address is always reachable in rip-relative addressing mode
12403 // when accessed from the code cache.
12404 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12405   switch (reloc_type) {
12406     // This should be rip-relative and easily reachable.
12407     case relocInfo::internal_word_type: {
12408       return true;
12409     }
12410     // This should be rip-relative within the code cache and easily
12411     // reachable until we get huge code caches. (At which point
12412     // IC code is going to have issues).
12413     case relocInfo::virtual_call_type:
12414     case relocInfo::opt_virtual_call_type:
12415     case relocInfo::static_call_type:
12416     case relocInfo::static_stub_type: {
12417       return true;
12418     }
12419     case relocInfo::runtime_call_type:
12420     case relocInfo::external_word_type:
12421     case relocInfo::poll_return_type: // these are really external_word but need special
12422     case relocInfo::poll_type: {      // relocs to identify them
12423       return CodeCache::contains(target);
12424     }
12425     default: {
12426       return false;
12427     }
12428   }
12429 }
12430 
12431 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12432 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12433   if (is_always_reachable(target, reloc_type)) {
12434     return true;
12435   }
12436   switch (reloc_type) {
12437     // None will force a 64bit literal to the code stream. Likely a placeholder
12438     // for something that will be patched later and we need to certain it will
12439     // always be reachable.
12440     case relocInfo::none: {
12441       return false;
12442     }
12443     case relocInfo::runtime_call_type:
12444     case relocInfo::external_word_type:
12445     case relocInfo::poll_return_type: // these are really external_word but need special
12446     case relocInfo::poll_type: {      // relocs to identify them
12447       assert(!CodeCache::contains(target), "always reachable");
12448       if (ForceUnreachable) {
12449         return false; // stress the correction code
12450       }
12451       // For external_word_type/runtime_call_type if it is reachable from where we
12452       // are now (possibly a temp buffer) and where we might end up
12453       // anywhere in the code cache then we are always reachable.
12454       // This would have to change if we ever save/restore shared code to be more pessimistic.
12455       // Code buffer has to be allocated in the code cache, so check against
12456       // code cache boundaries cover that case.
12457       //
12458       // In rip-relative addressing mode, an effective address is formed by adding displacement
12459       // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12460       // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12461       // to account for that.
12462       return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12463              Assembler::is_simm32(target - CodeCache::high_bound());
12464     }
12465     default: {
12466       return false;
12467     }
12468   }
12469 }
12470 
12471 bool Assembler::reachable(AddressLiteral adr) {
12472   assert(CodeCache::contains(pc()), "required");
12473   if (adr.is_lval()) {
12474     return false;
12475   }
12476   return is_reachable(adr.target(), adr.reloc());
12477 }
12478 
12479 bool Assembler::always_reachable(AddressLiteral adr) {
12480   assert(CodeCache::contains(pc()), "required");
12481   if (adr.is_lval()) {
12482     return false;
12483   }
12484   return is_always_reachable(adr.target(), adr.reloc());
12485 }
12486 
12487 void Assembler::emit_data64(jlong data,
12488                             relocInfo::relocType rtype,
12489                             int format) {
12490   if (rtype == relocInfo::none) {
12491     emit_int64(data);
12492   } else {
12493     emit_data64(data, Relocation::spec_simple(rtype), format);
12494   }
12495 }
12496 
12497 void Assembler::emit_data64(jlong data,
12498                             RelocationHolder const& rspec,
12499                             int format) {
12500   assert(imm_operand == 0, "default format must be immediate in this file");
12501   assert(imm_operand == format, "must be immediate");
12502   assert(inst_mark() != NULL, "must be inside InstructionMark");
12503   // Do not use AbstractAssembler::relocate, which is not intended for
12504   // embedded words.  Instead, relocate to the enclosing instruction.
12505   code_section()->relocate(inst_mark(), rspec, format);
12506 #ifdef ASSERT
12507   check_relocation(rspec, format);
12508 #endif
12509   emit_int64(data);
12510 }
12511 
12512 void Assembler::prefix(Register reg) {
12513   if (reg->encoding() >= 8) {
12514     prefix(REX_B);
12515   }
12516 }
12517 
12518 void Assembler::prefix(Register dst, Register src, Prefix p) {
12519   if (src->encoding() >= 8) {
12520     p = (Prefix)(p | REX_B);
12521   }
12522   if (dst->encoding() >= 8) {
12523     p = (Prefix)(p | REX_R);
12524   }
12525   if (p != Prefix_EMPTY) {
12526     // do not generate an empty prefix
12527     prefix(p);
12528   }
12529 }
12530 
12531 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12532   if (adr.base_needs_rex()) {
12533     if (adr.index_needs_rex()) {
12534       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12535     } else {
12536       prefix(REX_B);
12537     }
12538   } else {
12539     if (adr.index_needs_rex()) {
12540       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12541     }
12542   }
12543   if (dst->encoding() >= 8) {
12544     p = (Prefix)(p | REX_R);
12545   }
12546   if (p != Prefix_EMPTY) {
12547     // do not generate an empty prefix
12548     prefix(p);
12549   }
12550 }
12551 
12552 void Assembler::prefix(Address adr) {
12553   if (adr.base_needs_rex()) {
12554     if (adr.index_needs_rex()) {
12555       prefix(REX_XB);
12556     } else {
12557       prefix(REX_B);
12558     }
12559   } else {
12560     if (adr.index_needs_rex()) {
12561       prefix(REX_X);
12562     }
12563   }
12564 }
12565 
12566 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12567   if (reg->encoding() < 8) {
12568     if (adr.base_needs_rex()) {
12569       if (adr.index_needs_rex()) {
12570         prefix(REX_XB);
12571       } else {
12572         prefix(REX_B);
12573       }
12574     } else {
12575       if (adr.index_needs_rex()) {
12576         prefix(REX_X);
12577       } else if (byteinst && reg->encoding() >= 4) {
12578         prefix(REX);
12579       }
12580     }
12581   } else {
12582     if (adr.base_needs_rex()) {
12583       if (adr.index_needs_rex()) {
12584         prefix(REX_RXB);
12585       } else {
12586         prefix(REX_RB);
12587       }
12588     } else {
12589       if (adr.index_needs_rex()) {
12590         prefix(REX_RX);
12591       } else {
12592         prefix(REX_R);
12593       }
12594     }
12595   }
12596 }
12597 
12598 void Assembler::prefix(Address adr, XMMRegister reg) {
12599   if (reg->encoding() < 8) {
12600     if (adr.base_needs_rex()) {
12601       if (adr.index_needs_rex()) {
12602         prefix(REX_XB);
12603       } else {
12604         prefix(REX_B);
12605       }
12606     } else {
12607       if (adr.index_needs_rex()) {
12608         prefix(REX_X);
12609       }
12610     }
12611   } else {
12612     if (adr.base_needs_rex()) {
12613       if (adr.index_needs_rex()) {
12614         prefix(REX_RXB);
12615       } else {
12616         prefix(REX_RB);
12617       }
12618     } else {
12619       if (adr.index_needs_rex()) {
12620         prefix(REX_RX);
12621       } else {
12622         prefix(REX_R);
12623       }
12624     }
12625   }
12626 }
12627 
12628 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12629   if (reg_enc >= 8) {
12630     prefix(REX_B);
12631     reg_enc -= 8;
12632   } else if (byteinst && reg_enc >= 4) {
12633     prefix(REX);
12634   }
12635   return reg_enc;
12636 }
12637 
12638 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12639   if (dst_enc < 8) {
12640     if (src_enc >= 8) {
12641       prefix(REX_B);
12642       src_enc -= 8;
12643     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12644       prefix(REX);
12645     }
12646   } else {
12647     if (src_enc < 8) {
12648       prefix(REX_R);
12649     } else {
12650       prefix(REX_RB);
12651       src_enc -= 8;
12652     }
12653     dst_enc -= 8;
12654   }
12655   return dst_enc << 3 | src_enc;
12656 }
12657 
12658 int8_t Assembler::get_prefixq(Address adr) {
12659   int8_t prfx = get_prefixq(adr, rax);
12660   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12661   return prfx;
12662 }
12663 
12664 int8_t Assembler::get_prefixq(Address adr, Register src) {
12665   int8_t prfx = (int8_t)(REX_W +
12666                          ((int)adr.base_needs_rex()) +
12667                          ((int)adr.index_needs_rex() << 1) +
12668                          ((int)(src->encoding() >= 8) << 2));
12669 #ifdef ASSERT
12670   if (src->encoding() < 8) {
12671     if (adr.base_needs_rex()) {
12672       if (adr.index_needs_rex()) {
12673         assert(prfx == REX_WXB, "must be");
12674       } else {
12675         assert(prfx == REX_WB, "must be");
12676       }
12677     } else {
12678       if (adr.index_needs_rex()) {
12679         assert(prfx == REX_WX, "must be");
12680       } else {
12681         assert(prfx == REX_W, "must be");
12682       }
12683     }
12684   } else {
12685     if (adr.base_needs_rex()) {
12686       if (adr.index_needs_rex()) {
12687         assert(prfx == REX_WRXB, "must be");
12688       } else {
12689         assert(prfx == REX_WRB, "must be");
12690       }
12691     } else {
12692       if (adr.index_needs_rex()) {
12693         assert(prfx == REX_WRX, "must be");
12694       } else {
12695         assert(prfx == REX_WR, "must be");
12696       }
12697     }
12698   }
12699 #endif
12700   return prfx;
12701 }
12702 
12703 void Assembler::prefixq(Address adr) {
12704   emit_int8(get_prefixq(adr));
12705 }
12706 
12707 void Assembler::prefixq(Address adr, Register src) {
12708   emit_int8(get_prefixq(adr, src));
12709 }
12710 
12711 void Assembler::prefixq(Address adr, XMMRegister src) {
12712   if (src->encoding() < 8) {
12713     if (adr.base_needs_rex()) {
12714       if (adr.index_needs_rex()) {
12715         prefix(REX_WXB);
12716       } else {
12717         prefix(REX_WB);
12718       }
12719     } else {
12720       if (adr.index_needs_rex()) {
12721         prefix(REX_WX);
12722       } else {
12723         prefix(REX_W);
12724       }
12725     }
12726   } else {
12727     if (adr.base_needs_rex()) {
12728       if (adr.index_needs_rex()) {
12729         prefix(REX_WRXB);
12730       } else {
12731         prefix(REX_WRB);
12732       }
12733     } else {
12734       if (adr.index_needs_rex()) {
12735         prefix(REX_WRX);
12736       } else {
12737         prefix(REX_WR);
12738       }
12739     }
12740   }
12741 }
12742 
12743 int Assembler::prefixq_and_encode(int reg_enc) {
12744   if (reg_enc < 8) {
12745     prefix(REX_W);
12746   } else {
12747     prefix(REX_WB);
12748     reg_enc -= 8;
12749   }
12750   return reg_enc;
12751 }
12752 
12753 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12754   if (dst_enc < 8) {
12755     if (src_enc < 8) {
12756       prefix(REX_W);
12757     } else {
12758       prefix(REX_WB);
12759       src_enc -= 8;
12760     }
12761   } else {
12762     if (src_enc < 8) {
12763       prefix(REX_WR);
12764     } else {
12765       prefix(REX_WRB);
12766       src_enc -= 8;
12767     }
12768     dst_enc -= 8;
12769   }
12770   return dst_enc << 3 | src_enc;
12771 }
12772 
12773 void Assembler::adcq(Register dst, int32_t imm32) {
12774   (void) prefixq_and_encode(dst->encoding());
12775   emit_arith(0x81, 0xD0, dst, imm32);
12776 }
12777 
12778 void Assembler::adcq(Register dst, Address src) {
12779   InstructionMark im(this);
12780   emit_int16(get_prefixq(src, dst), 0x13);
12781   emit_operand(dst, src, 0);
12782 }
12783 
12784 void Assembler::adcq(Register dst, Register src) {
12785   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12786   emit_arith(0x13, 0xC0, dst, src);
12787 }
12788 
12789 void Assembler::addq(Address dst, int32_t imm32) {
12790   InstructionMark im(this);
12791   prefixq(dst);
12792   emit_arith_operand(0x81, rax, dst, imm32);
12793 }
12794 
12795 void Assembler::addq(Address dst, Register src) {
12796   InstructionMark im(this);
12797   emit_int16(get_prefixq(dst, src), 0x01);
12798   emit_operand(src, dst, 0);
12799 }
12800 
12801 void Assembler::addq(Register dst, int32_t imm32) {
12802   (void) prefixq_and_encode(dst->encoding());
12803   emit_arith(0x81, 0xC0, dst, imm32);
12804 }
12805 
12806 void Assembler::addq(Register dst, Address src) {
12807   InstructionMark im(this);
12808   emit_int16(get_prefixq(src, dst), 0x03);
12809   emit_operand(dst, src, 0);
12810 }
12811 
12812 void Assembler::addq(Register dst, Register src) {
12813   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12814   emit_arith(0x03, 0xC0, dst, src);
12815 }
12816 
12817 void Assembler::adcxq(Register dst, Register src) {
12818   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12819   emit_int8(0x66);
12820   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12821   emit_int32(0x0F,
12822              0x38,
12823              (unsigned char)0xF6,
12824              (0xC0 | encode));
12825 }
12826 
12827 void Assembler::adoxq(Register dst, Register src) {
12828   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12829   emit_int8((unsigned char)0xF3);
12830   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12831   emit_int32(0x0F,
12832              0x38,
12833              (unsigned char)0xF6,
12834              (0xC0 | encode));
12835 }
12836 
12837 void Assembler::andq(Address dst, int32_t imm32) {
12838   InstructionMark im(this);
12839   prefixq(dst);
12840   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12841 }
12842 
12843 void Assembler::andq(Register dst, int32_t imm32) {
12844   (void) prefixq_and_encode(dst->encoding());
12845   emit_arith(0x81, 0xE0, dst, imm32);
12846 }
12847 
12848 void Assembler::andq(Register dst, Address src) {
12849   InstructionMark im(this);
12850   emit_int16(get_prefixq(src, dst), 0x23);
12851   emit_operand(dst, src, 0);
12852 }
12853 
12854 void Assembler::andq(Register dst, Register src) {
12855   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12856   emit_arith(0x23, 0xC0, dst, src);
12857 }
12858 
12859 void Assembler::andq(Address dst, Register src) {
12860   InstructionMark im(this);
12861   emit_int16(get_prefixq(dst, src), 0x21);
12862   emit_operand(src, dst, 0);
12863 }
12864 
12865 void Assembler::andnq(Register dst, Register src1, Register src2) {
12866   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12867   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12868   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12869   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12870 }
12871 
12872 void Assembler::andnq(Register dst, Register src1, Address src2) {
12873   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12874   InstructionMark im(this);
12875   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12876   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12877   emit_int8((unsigned char)0xF2);
12878   emit_operand(dst, src2, 0);
12879 }
12880 
12881 void Assembler::bsfq(Register dst, Register src) {
12882   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12883   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12884 }
12885 
12886 void Assembler::bsrq(Register dst, Register src) {
12887   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12888   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12889 }
12890 
12891 void Assembler::bswapq(Register reg) {
12892   int encode = prefixq_and_encode(reg->encoding());
12893   emit_int16(0x0F, (0xC8 | encode));
12894 }
12895 
12896 void Assembler::blsiq(Register dst, Register src) {
12897   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12898   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12899   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12900   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12901 }
12902 
12903 void Assembler::blsiq(Register dst, Address src) {
12904   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12905   InstructionMark im(this);
12906   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12907   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12908   emit_int8((unsigned char)0xF3);
12909   emit_operand(rbx, src, 0);
12910 }
12911 
12912 void Assembler::blsmskq(Register dst, Register src) {
12913   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12914   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12915   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12916   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12917 }
12918 
12919 void Assembler::blsmskq(Register dst, Address src) {
12920   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12921   InstructionMark im(this);
12922   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12923   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12924   emit_int8((unsigned char)0xF3);
12925   emit_operand(rdx, src, 0);
12926 }
12927 
12928 void Assembler::blsrq(Register dst, Register src) {
12929   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12930   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12931   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12932   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12933 }
12934 
12935 void Assembler::blsrq(Register dst, Address src) {
12936   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12937   InstructionMark im(this);
12938   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12939   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12940   emit_int8((unsigned char)0xF3);
12941   emit_operand(rcx, src, 0);
12942 }
12943 
12944 void Assembler::cdqq() {
12945   emit_int16(REX_W, (unsigned char)0x99);
12946 }
12947 
12948 void Assembler::clflush(Address adr) {
12949   assert(VM_Version::supports_clflush(), "should do");
12950   prefix(adr);
12951   emit_int16(0x0F, (unsigned char)0xAE);
12952   emit_operand(rdi, adr, 0);
12953 }
12954 
12955 void Assembler::clflushopt(Address adr) {
12956   assert(VM_Version::supports_clflushopt(), "should do!");
12957   // adr should be base reg only with no index or offset
12958   assert(adr.index() == noreg, "index should be noreg");
12959   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12960   assert(adr.disp() == 0, "displacement should be 0");
12961   // instruction prefix is 0x66
12962   emit_int8(0x66);
12963   prefix(adr);
12964   // opcode family is 0x0F 0xAE
12965   emit_int16(0x0F, (unsigned char)0xAE);
12966   // extended opcode byte is 7 == rdi
12967   emit_operand(rdi, adr, 0);
12968 }
12969 
12970 void Assembler::clwb(Address adr) {
12971   assert(VM_Version::supports_clwb(), "should do!");
12972   // adr should be base reg only with no index or offset
12973   assert(adr.index() == noreg, "index should be noreg");
12974   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12975   assert(adr.disp() == 0, "displacement should be 0");
12976   // instruction prefix is 0x66
12977   emit_int8(0x66);
12978   prefix(adr);
12979   // opcode family is 0x0f 0xAE
12980   emit_int16(0x0F, (unsigned char)0xAE);
12981   // extended opcode byte is 6 == rsi
12982   emit_operand(rsi, adr, 0);
12983 }
12984 
12985 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12986   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12987   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12988 }
12989 
12990 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12991   InstructionMark im(this);
12992   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12993   emit_operand(dst, src, 0);
12994 }
12995 
12996 void Assembler::cmpq(Address dst, int32_t imm32) {
12997   InstructionMark im(this);
12998   prefixq(dst);
12999   emit_arith_operand(0x81, as_Register(7), dst, imm32);
13000 }
13001 
13002 void Assembler::cmpq(Register dst, int32_t imm32) {
13003   (void) prefixq_and_encode(dst->encoding());
13004   emit_arith(0x81, 0xF8, dst, imm32);
13005 }
13006 
13007 void Assembler::cmpq(Address dst, Register src) {
13008   InstructionMark im(this);
13009   emit_int16(get_prefixq(dst, src), 0x39);
13010   emit_operand(src, dst, 0);
13011 }
13012 
13013 void Assembler::cmpq(Register dst, Register src) {
13014   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13015   emit_arith(0x3B, 0xC0, dst, src);
13016 }
13017 
13018 void Assembler::cmpq(Register dst, Address src) {
13019   InstructionMark im(this);
13020   emit_int16(get_prefixq(src, dst), 0x3B);
13021   emit_operand(dst, src, 0);
13022 }
13023 
13024 void Assembler::cmpxchgq(Register reg, Address adr) {
13025   InstructionMark im(this);
13026   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
13027   emit_operand(reg, adr, 0);
13028 }
13029 
13030 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
13031   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13032   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13033   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13034   emit_int16(0x2A, (0xC0 | encode));
13035 }
13036 
13037 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
13038   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13039   InstructionMark im(this);
13040   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13041   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13042   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13043   emit_int8(0x2A);
13044   emit_operand(dst, src, 0);
13045 }
13046 
13047 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
13048   NOT_LP64(assert(VM_Version::supports_sse(), ""));
13049   InstructionMark im(this);
13050   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13051   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13052   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13053   emit_int8(0x2A);
13054   emit_operand(dst, src, 0);
13055 }
13056 
13057 void Assembler::cvttsd2siq(Register dst, Address src) {
13058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13059   // F2 REX.W 0F 2C /r
13060   // CVTTSD2SI r64, xmm1/m64
13061   InstructionMark im(this);
13062   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
13063   emit_operand(dst, src, 0);
13064 }
13065 
13066 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
13067   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13068   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13069   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13070   emit_int16(0x2C, (0xC0 | encode));
13071 }
13072 
13073 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
13074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13075   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13076   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13077   emit_int16(0x2D, (0xC0 | encode));
13078 }
13079 
13080 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
13081   NOT_LP64(assert(VM_Version::supports_sse(), ""));
13082   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13083   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13084   emit_int16(0x2C, (0xC0 | encode));
13085 }
13086 
13087 void Assembler::decl(Register dst) {
13088   // Don't use it directly. Use MacroAssembler::decrementl() instead.
13089   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
13090   int encode = prefix_and_encode(dst->encoding());
13091   emit_int16((unsigned char)0xFF, (0xC8 | encode));
13092 }
13093 
13094 void Assembler::decq(Register dst) {
13095   // Don't use it directly. Use MacroAssembler::decrementq() instead.
13096   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13097   int encode = prefixq_and_encode(dst->encoding());
13098   emit_int16((unsigned char)0xFF, 0xC8 | encode);
13099 }
13100 
13101 void Assembler::decq(Address dst) {
13102   // Don't use it directly. Use MacroAssembler::decrementq() instead.
13103   InstructionMark im(this);
13104   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13105   emit_operand(rcx, dst, 0);
13106 }
13107 
13108 void Assembler::fxrstor(Address src) {
13109   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13110   emit_operand(as_Register(1), src, 0);
13111 }
13112 
13113 void Assembler::xrstor(Address src) {
13114   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13115   emit_operand(as_Register(5), src, 0);
13116 }
13117 
13118 void Assembler::fxsave(Address dst) {
13119   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13120   emit_operand(as_Register(0), dst, 0);
13121 }
13122 
13123 void Assembler::xsave(Address dst) {
13124   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13125   emit_operand(as_Register(4), dst, 0);
13126 }
13127 
13128 void Assembler::idivq(Register src) {
13129   int encode = prefixq_and_encode(src->encoding());
13130   emit_int16((unsigned char)0xF7, (0xF8 | encode));
13131 }
13132 
13133 void Assembler::divq(Register src) {
13134   int encode = prefixq_and_encode(src->encoding());
13135   emit_int16((unsigned char)0xF7, (0xF0 | encode));
13136 }
13137 
13138 void Assembler::imulq(Register dst, Register src) {
13139   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13140   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
13141 }
13142 
13143 void Assembler::imulq(Register src) {
13144   int encode = prefixq_and_encode(src->encoding());
13145   emit_int16((unsigned char)0xF7, (0xE8 | encode));
13146 }
13147 
13148 void Assembler::imulq(Register dst, Address src, int32_t value) {
13149   InstructionMark im(this);
13150   prefixq(src, dst);
13151   if (is8bit(value)) {
13152     emit_int8((unsigned char)0x6B);
13153     emit_operand(dst, src, 1);
13154     emit_int8(value);
13155   } else {
13156     emit_int8((unsigned char)0x69);
13157     emit_operand(dst, src, 4);
13158     emit_int32(value);
13159   }
13160 }
13161 
13162 void Assembler::imulq(Register dst, Register src, int value) {
13163   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13164   if (is8bit(value)) {
13165     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
13166   } else {
13167     emit_int16(0x69, (0xC0 | encode));
13168     emit_int32(value);
13169   }
13170 }
13171 
13172 void Assembler::imulq(Register dst, Address src) {
13173   InstructionMark im(this);
13174   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
13175   emit_operand(dst, src, 0);
13176 }
13177 
13178 void Assembler::incl(Register dst) {
13179   // Don't use it directly. Use MacroAssembler::incrementl() instead.
13180   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13181   int encode = prefix_and_encode(dst->encoding());
13182   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13183 }
13184 
13185 void Assembler::incq(Register dst) {
13186   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13187   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13188   int encode = prefixq_and_encode(dst->encoding());
13189   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13190 }
13191 
13192 void Assembler::incq(Address dst) {
13193   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13194   InstructionMark im(this);
13195   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13196   emit_operand(rax, dst, 0);
13197 }
13198 
13199 void Assembler::lea(Register dst, Address src) {
13200   leaq(dst, src);
13201 }
13202 
13203 void Assembler::leaq(Register dst, Address src) {
13204   InstructionMark im(this);
13205   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13206   emit_operand(dst, src, 0);
13207 }
13208 
13209 void Assembler::mov64(Register dst, int64_t imm64) {
13210   InstructionMark im(this);
13211   int encode = prefixq_and_encode(dst->encoding());
13212   emit_int8(0xB8 | encode);
13213   emit_int64(imm64);
13214 }
13215 
13216 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13217   InstructionMark im(this);
13218   int encode = prefixq_and_encode(dst->encoding());
13219   emit_int8(0xB8 | encode);
13220   emit_data64(imm64, rtype, format);
13221 }
13222 
13223 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13224   InstructionMark im(this);
13225   int encode = prefixq_and_encode(dst->encoding());
13226   emit_int8(0xB8 | encode);
13227   emit_data64(imm64, rspec);
13228 }
13229 
13230 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13231   InstructionMark im(this);
13232   int encode = prefix_and_encode(dst->encoding());
13233   emit_int8(0xB8 | encode);
13234   emit_data((int)imm32, rspec, narrow_oop_operand);
13235 }
13236 
13237 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
13238   InstructionMark im(this);
13239   prefix(dst);
13240   emit_int8((unsigned char)0xC7);
13241   emit_operand(rax, dst, 4);
13242   emit_data((int)imm32, rspec, narrow_oop_operand);
13243 }
13244 
13245 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13246   InstructionMark im(this);
13247   int encode = prefix_and_encode(src1->encoding());
13248   emit_int16((unsigned char)0x81, (0xF8 | encode));
13249   emit_data((int)imm32, rspec, narrow_oop_operand);
13250 }
13251 
13252 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13253   InstructionMark im(this);
13254   prefix(src1);
13255   emit_int8((unsigned char)0x81);
13256   emit_operand(rax, src1, 4);
13257   emit_data((int)imm32, rspec, narrow_oop_operand);
13258 }
13259 
13260 void Assembler::lzcntq(Register dst, Register src) {
13261   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13262   emit_int8((unsigned char)0xF3);
13263   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13264   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13265 }
13266 
13267 void Assembler::lzcntq(Register dst, Address src) {
13268   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13269   InstructionMark im(this);
13270   emit_int8((unsigned char)0xF3);
13271   prefixq(src, dst);
13272   emit_int16(0x0F, (unsigned char)0xBD);
13273   emit_operand(dst, src, 0);
13274 }
13275 
13276 void Assembler::movdq(XMMRegister dst, Register src) {
13277   // table D-1 says MMX/SSE2
13278   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13279   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13280   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13281   emit_int16(0x6E, (0xC0 | encode));
13282 }
13283 
13284 void Assembler::movdq(Register dst, XMMRegister src) {
13285   // table D-1 says MMX/SSE2
13286   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13287   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13288   // swap src/dst to get correct prefix
13289   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13290   emit_int16(0x7E,
13291              (0xC0 | encode));
13292 }
13293 
13294 void Assembler::movq(Register dst, Register src) {
13295   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13296   emit_int16((unsigned char)0x8B,
13297              (0xC0 | encode));
13298 }
13299 
13300 void Assembler::movq(Register dst, Address src) {
13301   InstructionMark im(this);
13302   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13303   emit_operand(dst, src, 0);
13304 }
13305 
13306 void Assembler::movq(Address dst, Register src) {
13307   InstructionMark im(this);
13308   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13309   emit_operand(src, dst, 0);
13310 }
13311 
13312 void Assembler::movq(Address dst, int32_t imm32) {
13313   InstructionMark im(this);
13314   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13315   emit_operand(as_Register(0), dst, 4);
13316   emit_int32(imm32);
13317 }
13318 
13319 void Assembler::movq(Register dst, int32_t imm32) {
13320   int encode = prefixq_and_encode(dst->encoding());
13321   emit_int16((unsigned char)0xC7, (0xC0 | encode));
13322   emit_int32(imm32);
13323 }
13324 
13325 void Assembler::movsbq(Register dst, Address src) {
13326   InstructionMark im(this);
13327   emit_int24(get_prefixq(src, dst),
13328              0x0F,
13329              (unsigned char)0xBE);
13330   emit_operand(dst, src, 0);
13331 }
13332 
13333 void Assembler::movsbq(Register dst, Register src) {
13334   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13335   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13336 }
13337 
13338 void Assembler::movslq(Register dst, int32_t imm32) {
13339   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
13340   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
13341   // as a result we shouldn't use until tested at runtime...
13342   ShouldNotReachHere();
13343   InstructionMark im(this);
13344   int encode = prefixq_and_encode(dst->encoding());
13345   emit_int8(0xC7 | encode);
13346   emit_int32(imm32);
13347 }
13348 
13349 void Assembler::movslq(Address dst, int32_t imm32) {
13350   assert(is_simm32(imm32), "lost bits");
13351   InstructionMark im(this);
13352   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13353   emit_operand(rax, dst, 4);
13354   emit_int32(imm32);
13355 }
13356 
13357 void Assembler::movslq(Register dst, Address src) {
13358   InstructionMark im(this);
13359   emit_int16(get_prefixq(src, dst), 0x63);
13360   emit_operand(dst, src, 0);
13361 }
13362 
13363 void Assembler::movslq(Register dst, Register src) {
13364   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13365   emit_int16(0x63, (0xC0 | encode));
13366 }
13367 
13368 void Assembler::movswq(Register dst, Address src) {
13369   InstructionMark im(this);
13370   emit_int24(get_prefixq(src, dst),
13371              0x0F,
13372              (unsigned char)0xBF);
13373   emit_operand(dst, src, 0);
13374 }
13375 
13376 void Assembler::movswq(Register dst, Register src) {
13377   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13378   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13379 }
13380 
13381 void Assembler::movzbq(Register dst, Address src) {
13382   InstructionMark im(this);
13383   emit_int24(get_prefixq(src, dst),
13384              0x0F,
13385              (unsigned char)0xB6);
13386   emit_operand(dst, src, 0);
13387 }
13388 
13389 void Assembler::movzbq(Register dst, Register src) {
13390   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13391   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13392 }
13393 
13394 void Assembler::movzwq(Register dst, Address src) {
13395   InstructionMark im(this);
13396   emit_int24(get_prefixq(src, dst),
13397              0x0F,
13398              (unsigned char)0xB7);
13399   emit_operand(dst, src, 0);
13400 }
13401 
13402 void Assembler::movzwq(Register dst, Register src) {
13403   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13404   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13405 }
13406 
13407 void Assembler::mulq(Address src) {
13408   InstructionMark im(this);
13409   emit_int16(get_prefixq(src), (unsigned char)0xF7);
13410   emit_operand(rsp, src, 0);
13411 }
13412 
13413 void Assembler::mulq(Register src) {
13414   int encode = prefixq_and_encode(src->encoding());
13415   emit_int16((unsigned char)0xF7, (0xE0 | encode));
13416 }
13417 
13418 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13419   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13420   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13421   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13422   emit_int16((unsigned char)0xF6, (0xC0 | encode));
13423 }
13424 
13425 void Assembler::negq(Register dst) {
13426   int encode = prefixq_and_encode(dst->encoding());
13427   emit_int16((unsigned char)0xF7, (0xD8 | encode));
13428 }
13429 
13430 void Assembler::negq(Address dst) {
13431   InstructionMark im(this);
13432   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13433   emit_operand(as_Register(3), dst, 0);
13434 }
13435 
13436 void Assembler::notq(Register dst) {
13437   int encode = prefixq_and_encode(dst->encoding());
13438   emit_int16((unsigned char)0xF7, (0xD0 | encode));
13439 }
13440 
13441 void Assembler::btsq(Address dst, int imm8) {
13442   assert(isByte(imm8), "not a byte");
13443   InstructionMark im(this);
13444   emit_int24(get_prefixq(dst),
13445              0x0F,
13446              (unsigned char)0xBA);
13447   emit_operand(rbp /* 5 */, dst, 1);
13448   emit_int8(imm8);
13449 }
13450 
13451 void Assembler::btrq(Address dst, int imm8) {
13452   assert(isByte(imm8), "not a byte");
13453   InstructionMark im(this);
13454   emit_int24(get_prefixq(dst),
13455              0x0F,
13456              (unsigned char)0xBA);
13457   emit_operand(rsi /* 6 */, dst, 1);
13458   emit_int8(imm8);
13459 }
13460 
13461 void Assembler::orq(Address dst, int32_t imm32) {
13462   InstructionMark im(this);
13463   prefixq(dst);
13464   emit_arith_operand(0x81, as_Register(1), dst, imm32);
13465 }
13466 
13467 void Assembler::orq(Address dst, Register src) {
13468   InstructionMark im(this);
13469   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13470   emit_operand(src, dst, 0);
13471 }
13472 
13473 void Assembler::orq(Register dst, int32_t imm32) {
13474   (void) prefixq_and_encode(dst->encoding());
13475   emit_arith(0x81, 0xC8, dst, imm32);
13476 }
13477 
13478 void Assembler::orq_imm32(Register dst, int32_t imm32) {
13479   (void) prefixq_and_encode(dst->encoding());
13480   emit_arith_imm32(0x81, 0xC8, dst, imm32);
13481 }
13482 
13483 void Assembler::orq(Register dst, Address src) {
13484   InstructionMark im(this);
13485   emit_int16(get_prefixq(src, dst), 0x0B);
13486   emit_operand(dst, src, 0);
13487 }
13488 
13489 void Assembler::orq(Register dst, Register src) {
13490   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13491   emit_arith(0x0B, 0xC0, dst, src);
13492 }
13493 
13494 void Assembler::popcntq(Register dst, Address src) {
13495   assert(VM_Version::supports_popcnt(), "must support");
13496   InstructionMark im(this);
13497   emit_int32((unsigned char)0xF3,
13498              get_prefixq(src, dst),
13499              0x0F,
13500              (unsigned char)0xB8);
13501   emit_operand(dst, src, 0);
13502 }
13503 
13504 void Assembler::popcntq(Register dst, Register src) {
13505   assert(VM_Version::supports_popcnt(), "must support");
13506   emit_int8((unsigned char)0xF3);
13507   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13508   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13509 }
13510 
13511 void Assembler::popq(Address dst) {
13512   InstructionMark im(this);
13513   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13514   emit_operand(rax, dst, 0);
13515 }
13516 
13517 void Assembler::popq(Register dst) {
13518   emit_int8((unsigned char)0x58 | dst->encoding());
13519 }
13520 
13521 // Precomputable: popa, pusha, vzeroupper
13522 
13523 // The result of these routines are invariant from one invocation to another
13524 // invocation for the duration of a run. Caching the result on bootstrap
13525 // and copying it out on subsequent invocations can thus be beneficial
13526 static bool     precomputed = false;
13527 
13528 static u_char* popa_code  = NULL;
13529 static int     popa_len   = 0;
13530 
13531 static u_char* pusha_code = NULL;
13532 static int     pusha_len  = 0;
13533 
13534 static u_char* vzup_code  = NULL;
13535 static int     vzup_len   = 0;
13536 
13537 void Assembler::precompute_instructions() {
13538   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13539   guarantee(!precomputed, "only once");
13540   precomputed = true;
13541   ResourceMark rm;
13542 
13543   // Make a temporary buffer big enough for the routines we're capturing
13544   int size = 256;
13545   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13546   CodeBuffer buffer((address)tmp_code, size);
13547   MacroAssembler masm(&buffer);
13548 
13549   address begin_popa  = masm.code_section()->end();
13550   masm.popa_uncached();
13551   address end_popa    = masm.code_section()->end();
13552   masm.pusha_uncached();
13553   address end_pusha   = masm.code_section()->end();
13554   masm.vzeroupper_uncached();
13555   address end_vzup    = masm.code_section()->end();
13556 
13557   // Save the instructions to permanent buffers.
13558   popa_len = (int)(end_popa - begin_popa);
13559   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13560   memcpy(popa_code, begin_popa, popa_len);
13561 
13562   pusha_len = (int)(end_pusha - end_popa);
13563   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13564   memcpy(pusha_code, end_popa, pusha_len);
13565 
13566   vzup_len = (int)(end_vzup - end_pusha);
13567   if (vzup_len > 0) {
13568     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13569     memcpy(vzup_code, end_pusha, vzup_len);
13570   } else {
13571     vzup_code = pusha_code; // dummy
13572   }
13573 
13574   assert(masm.code()->total_oop_size() == 0 &&
13575          masm.code()->total_metadata_size() == 0 &&
13576          masm.code()->total_relocation_size() == 0,
13577          "pre-computed code can't reference oops, metadata or contain relocations");
13578 }
13579 
13580 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13581   assert(src != NULL, "code to copy must have been pre-computed");
13582   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13583   address end = code_section->end();
13584   memcpy(end, src, src_len);
13585   code_section->set_end(end + src_len);
13586 }
13587 
13588 void Assembler::popa() { // 64bit
13589   emit_copy(code_section(), popa_code, popa_len);
13590 }
13591 
13592 void Assembler::popa_uncached() { // 64bit
13593   movq(r15, Address(rsp, 0));
13594   movq(r14, Address(rsp, wordSize));
13595   movq(r13, Address(rsp, 2 * wordSize));
13596   movq(r12, Address(rsp, 3 * wordSize));
13597   movq(r11, Address(rsp, 4 * wordSize));
13598   movq(r10, Address(rsp, 5 * wordSize));
13599   movq(r9,  Address(rsp, 6 * wordSize));
13600   movq(r8,  Address(rsp, 7 * wordSize));
13601   movq(rdi, Address(rsp, 8 * wordSize));
13602   movq(rsi, Address(rsp, 9 * wordSize));
13603   movq(rbp, Address(rsp, 10 * wordSize));
13604   // Skip rsp as it is restored automatically to the value
13605   // before the corresponding pusha when popa is done.
13606   movq(rbx, Address(rsp, 12 * wordSize));
13607   movq(rdx, Address(rsp, 13 * wordSize));
13608   movq(rcx, Address(rsp, 14 * wordSize));
13609   movq(rax, Address(rsp, 15 * wordSize));
13610 
13611   addq(rsp, 16 * wordSize);
13612 }
13613 
13614 // Does not actually store the value of rsp on the stack.
13615 // The slot for rsp just contains an arbitrary value.
13616 void Assembler::pusha() { // 64bit
13617   emit_copy(code_section(), pusha_code, pusha_len);
13618 }
13619 
13620 // Does not actually store the value of rsp on the stack.
13621 // The slot for rsp just contains an arbitrary value.
13622 void Assembler::pusha_uncached() { // 64bit
13623   subq(rsp, 16 * wordSize);
13624 
13625   movq(Address(rsp, 15 * wordSize), rax);
13626   movq(Address(rsp, 14 * wordSize), rcx);
13627   movq(Address(rsp, 13 * wordSize), rdx);
13628   movq(Address(rsp, 12 * wordSize), rbx);
13629   // Skip rsp as the value is normally not used. There are a few places where
13630   // the original value of rsp needs to be known but that can be computed
13631   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13632   movq(Address(rsp, 10 * wordSize), rbp);
13633   movq(Address(rsp, 9 * wordSize), rsi);
13634   movq(Address(rsp, 8 * wordSize), rdi);
13635   movq(Address(rsp, 7 * wordSize), r8);
13636   movq(Address(rsp, 6 * wordSize), r9);
13637   movq(Address(rsp, 5 * wordSize), r10);
13638   movq(Address(rsp, 4 * wordSize), r11);
13639   movq(Address(rsp, 3 * wordSize), r12);
13640   movq(Address(rsp, 2 * wordSize), r13);
13641   movq(Address(rsp, wordSize), r14);
13642   movq(Address(rsp, 0), r15);
13643 }
13644 
13645 void Assembler::vzeroupper() {
13646   emit_copy(code_section(), vzup_code, vzup_len);
13647 }
13648 
13649 void Assembler::vzeroall() {
13650   assert(VM_Version::supports_avx(), "requires AVX");
13651   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13652   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
13653   emit_int8(0x77);
13654 }
13655 
13656 void Assembler::pushq(Address src) {
13657   InstructionMark im(this);
13658   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13659   emit_operand(rsi, src, 0);
13660 }
13661 
13662 void Assembler::rclq(Register dst, int imm8) {
13663   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13664   int encode = prefixq_and_encode(dst->encoding());
13665   if (imm8 == 1) {
13666     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13667   } else {
13668     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13669   }
13670 }
13671 
13672 void Assembler::rcrq(Register dst, int imm8) {
13673   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13674   int encode = prefixq_and_encode(dst->encoding());
13675   if (imm8 == 1) {
13676     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13677   } else {
13678     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13679   }
13680 }
13681 
13682 void Assembler::rorxl(Register dst, Register src, int imm8) {
13683   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13684   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13685   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13686   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13687 }
13688 
13689 void Assembler::rorxl(Register dst, Address src, int imm8) {
13690   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13691   InstructionMark im(this);
13692   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13693   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13694   emit_int8((unsigned char)0xF0);
13695   emit_operand(dst, src, 1);
13696   emit_int8(imm8);
13697 }
13698 
13699 void Assembler::rorxq(Register dst, Register src, int imm8) {
13700   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13701   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13702   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13703   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13704 }
13705 
13706 void Assembler::rorxq(Register dst, Address src, int imm8) {
13707   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13708   InstructionMark im(this);
13709   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13710   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13711   emit_int8((unsigned char)0xF0);
13712   emit_operand(dst, src, 1);
13713   emit_int8(imm8);
13714 }
13715 
13716 #ifdef _LP64
13717 void Assembler::salq(Address dst, int imm8) {
13718   InstructionMark im(this);
13719   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13720   if (imm8 == 1) {
13721     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13722     emit_operand(as_Register(4), dst, 0);
13723   }
13724   else {
13725     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13726     emit_operand(as_Register(4), dst, 1);
13727     emit_int8(imm8);
13728   }
13729 }
13730 
13731 void Assembler::salq(Address dst) {
13732   InstructionMark im(this);
13733   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13734   emit_operand(as_Register(4), dst, 0);
13735 }
13736 
13737 void Assembler::salq(Register dst, int imm8) {
13738   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13739   int encode = prefixq_and_encode(dst->encoding());
13740   if (imm8 == 1) {
13741     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13742   } else {
13743     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13744   }
13745 }
13746 
13747 void Assembler::salq(Register dst) {
13748   int encode = prefixq_and_encode(dst->encoding());
13749   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13750 }
13751 
13752 void Assembler::sarq(Address dst, int imm8) {
13753   InstructionMark im(this);
13754   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13755   if (imm8 == 1) {
13756     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13757     emit_operand(as_Register(7), dst, 0);
13758   }
13759   else {
13760     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13761     emit_operand(as_Register(7), dst, 1);
13762     emit_int8(imm8);
13763   }
13764 }
13765 
13766 void Assembler::sarq(Address dst) {
13767   InstructionMark im(this);
13768   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13769   emit_operand(as_Register(7), dst, 0);
13770 }
13771 
13772 void Assembler::sarq(Register dst, int imm8) {
13773   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13774   int encode = prefixq_and_encode(dst->encoding());
13775   if (imm8 == 1) {
13776     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13777   } else {
13778     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13779   }
13780 }
13781 
13782 void Assembler::sarq(Register dst) {
13783   int encode = prefixq_and_encode(dst->encoding());
13784   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13785 }
13786 #endif
13787 
13788 void Assembler::sbbq(Address dst, int32_t imm32) {
13789   InstructionMark im(this);
13790   prefixq(dst);
13791   emit_arith_operand(0x81, rbx, dst, imm32);
13792 }
13793 
13794 void Assembler::sbbq(Register dst, int32_t imm32) {
13795   (void) prefixq_and_encode(dst->encoding());
13796   emit_arith(0x81, 0xD8, dst, imm32);
13797 }
13798 
13799 void Assembler::sbbq(Register dst, Address src) {
13800   InstructionMark im(this);
13801   emit_int16(get_prefixq(src, dst), 0x1B);
13802   emit_operand(dst, src, 0);
13803 }
13804 
13805 void Assembler::sbbq(Register dst, Register src) {
13806   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13807   emit_arith(0x1B, 0xC0, dst, src);
13808 }
13809 
13810 void Assembler::shlq(Register dst, int imm8) {
13811   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13812   int encode = prefixq_and_encode(dst->encoding());
13813   if (imm8 == 1) {
13814     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13815   } else {
13816     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13817   }
13818 }
13819 
13820 void Assembler::shlq(Register dst) {
13821   int encode = prefixq_and_encode(dst->encoding());
13822   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13823 }
13824 
13825 void Assembler::shrq(Register dst, int imm8) {
13826   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13827   int encode = prefixq_and_encode(dst->encoding());
13828   if (imm8 == 1) {
13829     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13830   }
13831   else {
13832     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13833   }
13834 }
13835 
13836 void Assembler::shrq(Register dst) {
13837   int encode = prefixq_and_encode(dst->encoding());
13838   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13839 }
13840 
13841 void Assembler::shrq(Address dst) {
13842   InstructionMark im(this);
13843   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13844   emit_operand(as_Register(5), dst, 0);
13845 }
13846 
13847 void Assembler::shrq(Address dst, int imm8) {
13848   InstructionMark im(this);
13849   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13850   if (imm8 == 1) {
13851     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13852     emit_operand(as_Register(5), dst, 0);
13853   }
13854   else {
13855     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13856     emit_operand(as_Register(5), dst, 1);
13857     emit_int8(imm8);
13858   }
13859 }
13860 
13861 void Assembler::subq(Address dst, int32_t imm32) {
13862   InstructionMark im(this);
13863   prefixq(dst);
13864   emit_arith_operand(0x81, rbp, dst, imm32);
13865 }
13866 
13867 void Assembler::subq(Address dst, Register src) {
13868   InstructionMark im(this);
13869   emit_int16(get_prefixq(dst, src), 0x29);
13870   emit_operand(src, dst, 0);
13871 }
13872 
13873 void Assembler::subq(Register dst, int32_t imm32) {
13874   (void) prefixq_and_encode(dst->encoding());
13875   emit_arith(0x81, 0xE8, dst, imm32);
13876 }
13877 
13878 // Force generation of a 4 byte immediate value even if it fits into 8bit
13879 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13880   (void) prefixq_and_encode(dst->encoding());
13881   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13882 }
13883 
13884 void Assembler::subq(Register dst, Address src) {
13885   InstructionMark im(this);
13886   emit_int16(get_prefixq(src, dst), 0x2B);
13887   emit_operand(dst, src, 0);
13888 }
13889 
13890 void Assembler::subq(Register dst, Register src) {
13891   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13892   emit_arith(0x2B, 0xC0, dst, src);
13893 }
13894 
13895 void Assembler::testq(Address dst, int32_t imm32) {
13896   InstructionMark im(this);
13897   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13898   emit_operand(as_Register(0), dst, 4);
13899   emit_int32(imm32);
13900 }
13901 
13902 void Assembler::testq(Register dst, int32_t imm32) {
13903   // not using emit_arith because test
13904   // doesn't support sign-extension of
13905   // 8bit operands
13906   if (dst == rax) {
13907     prefix(REX_W);
13908     emit_int8((unsigned char)0xA9);
13909     emit_int32(imm32);
13910   } else {
13911     int encode = dst->encoding();
13912     encode = prefixq_and_encode(encode);
13913     emit_int16((unsigned char)0xF7, (0xC0 | encode));
13914     emit_int32(imm32);
13915   }
13916 }
13917 
13918 void Assembler::testq(Register dst, Register src) {
13919   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13920   emit_arith(0x85, 0xC0, dst, src);
13921 }
13922 
13923 void Assembler::testq(Register dst, Address src) {
13924   InstructionMark im(this);
13925   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13926   emit_operand(dst, src, 0);
13927 }
13928 
13929 void Assembler::xaddq(Address dst, Register src) {
13930   InstructionMark im(this);
13931   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13932   emit_operand(src, dst, 0);
13933 }
13934 
13935 void Assembler::xchgq(Register dst, Address src) {
13936   InstructionMark im(this);
13937   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13938   emit_operand(dst, src, 0);
13939 }
13940 
13941 void Assembler::xchgq(Register dst, Register src) {
13942   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13943   emit_int16((unsigned char)0x87, (0xc0 | encode));
13944 }
13945 
13946 void Assembler::xorq(Register dst, Register src) {
13947   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13948   emit_arith(0x33, 0xC0, dst, src);
13949 }
13950 
13951 void Assembler::xorq(Register dst, Address src) {
13952   InstructionMark im(this);
13953   emit_int16(get_prefixq(src, dst), 0x33);
13954   emit_operand(dst, src, 0);
13955 }
13956 
13957 void Assembler::xorq(Register dst, int32_t imm32) {
13958   (void) prefixq_and_encode(dst->encoding());
13959   emit_arith(0x81, 0xF0, dst, imm32);
13960 }
13961 
13962 void Assembler::xorq(Address dst, int32_t imm32) {
13963   InstructionMark im(this);
13964   prefixq(dst);
13965   emit_arith_operand(0x81, as_Register(6), dst, imm32);
13966 }
13967 
13968 void Assembler::xorq(Address dst, Register src) {
13969   InstructionMark im(this);
13970   emit_int16(get_prefixq(dst, src), 0x31);
13971   emit_operand(src, dst, 0);
13972 }
13973 
13974 #endif // !LP64
13975 
13976 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13977   if (VM_Version::supports_evex()) {
13978     _tuple_type = tuple_type;
13979     _input_size_in_bits = input_size_in_bits;
13980   }
13981 }