1 /*
    2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert((op1 & 0x01) == 1, "should be 32bit operation");
  304   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  305   if (is8bit(imm32)) {
  306     emit_int24(op1 | 0x02,        // set sign bit
  307                op2 | encode(dst),
  308                imm32 & 0xFF);
  309   } else {
  310     emit_int16(op1, (op2 | encode(dst)));
  311     emit_int32(imm32);
  312   }
  313 }
  314 
  315 // Force generation of a 4 byte immediate value even if it fits into 8bit
  316 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  317   assert(isByte(op1) && isByte(op2), "wrong opcode");
  318   assert((op1 & 0x01) == 1, "should be 32bit operation");
  319   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  320   emit_int16(op1, (op2 | encode(dst)));
  321   emit_int32(imm32);
  322 }
  323 
  324 // immediate-to-memory forms
  325 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  326   assert((op1 & 0x01) == 1, "should be 32bit operation");
  327   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  328   if (is8bit(imm32)) {
  329     emit_int8(op1 | 0x02); // set sign bit
  330     emit_operand(rm, adr, 1);
  331     emit_int8(imm32 & 0xFF);
  332   } else {
  333     emit_int8(op1);
  334     emit_operand(rm, adr, 4);
  335     emit_int32(imm32);
  336   }
  337 }
  338 
  339 
  340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  341   assert(isByte(op1) && isByte(op2), "wrong opcode");
  342   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  343 }
  344 
  345 
  346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  347                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  348   int mod_idx = 0;
  349   // We will test if the displacement fits the compressed format and if so
  350   // apply the compression to the displacment iff the result is8bit.
  351   if (VM_Version::supports_evex() && is_evex_inst) {
  352     switch (cur_tuple_type) {
  353     case EVEX_FV:
  354       if ((cur_encoding & VEX_W) == VEX_W) {
  355         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  356       } else {
  357         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  358       }
  359       break;
  360 
  361     case EVEX_HV:
  362       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  363       break;
  364 
  365     case EVEX_FVM:
  366       break;
  367 
  368     case EVEX_T1S:
  369       switch (in_size_in_bits) {
  370       case EVEX_8bit:
  371         break;
  372 
  373       case EVEX_16bit:
  374         mod_idx = 1;
  375         break;
  376 
  377       case EVEX_32bit:
  378         mod_idx = 2;
  379         break;
  380 
  381       case EVEX_64bit:
  382         mod_idx = 3;
  383         break;
  384       }
  385       break;
  386 
  387     case EVEX_T1F:
  388     case EVEX_T2:
  389     case EVEX_T4:
  390       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  391       break;
  392 
  393     case EVEX_T8:
  394       break;
  395 
  396     case EVEX_HVM:
  397       break;
  398 
  399     case EVEX_QVM:
  400       break;
  401 
  402     case EVEX_OVM:
  403       break;
  404 
  405     case EVEX_M128:
  406       break;
  407 
  408     case EVEX_DUP:
  409       break;
  410 
  411     default:
  412       assert(0, "no valid evex tuple_table entry");
  413       break;
  414     }
  415 
  416     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  417       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  418       if ((disp % disp_factor) == 0) {
  419         int new_disp = disp / disp_factor;
  420         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  421           disp = new_disp;
  422         }
  423       } else {
  424         return false;
  425       }
  426     }
  427   }
  428   return (-0x80 <= disp && disp < 0x80);
  429 }
  430 
  431 
  432 bool Assembler::emit_compressed_disp_byte(int &disp) {
  433   int mod_idx = 0;
  434   // We will test if the displacement fits the compressed format and if so
  435   // apply the compression to the displacment iff the result is8bit.
  436   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  437     int evex_encoding = _attributes->get_evex_encoding();
  438     int tuple_type = _attributes->get_tuple_type();
  439     switch (tuple_type) {
  440     case EVEX_FV:
  441       if ((evex_encoding & VEX_W) == VEX_W) {
  442         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  443       } else {
  444         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  445       }
  446       break;
  447 
  448     case EVEX_HV:
  449       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  450       break;
  451 
  452     case EVEX_FVM:
  453       break;
  454 
  455     case EVEX_T1S:
  456       switch (_attributes->get_input_size()) {
  457       case EVEX_8bit:
  458         break;
  459 
  460       case EVEX_16bit:
  461         mod_idx = 1;
  462         break;
  463 
  464       case EVEX_32bit:
  465         mod_idx = 2;
  466         break;
  467 
  468       case EVEX_64bit:
  469         mod_idx = 3;
  470         break;
  471       }
  472       break;
  473 
  474     case EVEX_T1F:
  475     case EVEX_T2:
  476     case EVEX_T4:
  477       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  478       break;
  479 
  480     case EVEX_T8:
  481       break;
  482 
  483     case EVEX_HVM:
  484       break;
  485 
  486     case EVEX_QVM:
  487       break;
  488 
  489     case EVEX_OVM:
  490       break;
  491 
  492     case EVEX_M128:
  493       break;
  494 
  495     case EVEX_DUP:
  496       break;
  497 
  498     default:
  499       assert(0, "no valid evex tuple_table entry");
  500       break;
  501     }
  502 
  503     int vector_len = _attributes->get_vector_len();
  504     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  505       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  506       if ((disp % disp_factor) == 0) {
  507         int new_disp = disp / disp_factor;
  508         if (is8bit(new_disp)) {
  509           disp = new_disp;
  510         }
  511       } else {
  512         return false;
  513       }
  514     }
  515   }
  516   return is8bit(disp);
  517 }
  518 
  519 static bool is_valid_encoding(int reg_enc) {
  520   return reg_enc >= 0;
  521 }
  522 
  523 static int raw_encode(Register reg) {
  524   assert(reg == noreg || reg->is_valid(), "sanity");
  525   int reg_enc = (intptr_t)reg;
  526   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  527   return reg_enc;
  528 }
  529 
  530 static int raw_encode(XMMRegister xmmreg) {
  531   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  532   int xmmreg_enc = (intptr_t)xmmreg;
  533   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  534   return xmmreg_enc;
  535 }
  536 
  537 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  538   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  539 }
  540 
  541 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  542   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  543 }
  544 
  545 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  546   assert((mod & 3) != 0b11, "forbidden");
  547   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  548   emit_int8(modrm);
  549 }
  550 
  551 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  552                                         int disp) {
  553   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  554   emit_int16(modrm, disp & 0xFF);
  555 }
  556 
  557 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  558                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   int sib = sib_encoding(scale, index_enc, base_enc);
  561   emit_int16(modrm, sib);
  562 }
  563 
  564 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  565                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  566                                             int disp) {
  567   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  568   int sib = sib_encoding(scale, index_enc, base_enc);
  569   emit_int24(modrm, sib, disp & 0xFF);
  570 }
  571 
  572 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  573                                     Address::ScaleFactor scale, int disp,
  574                                     RelocationHolder const& rspec,
  575                                     int rip_relative_correction) {
  576   bool no_relocation = (rspec.type() == relocInfo::none);
  577 
  578   if (is_valid_encoding(base_enc)) {
  579     if (is_valid_encoding(index_enc)) {
  580       assert(scale != Address::no_scale, "inconsistent address");
  581       // [base + index*scale + disp]
  582       if (disp == 0 && no_relocation &&
  583           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  584         // [base + index*scale]
  585         // [00 reg 100][ss index base]
  586         emit_modrm_sib(0b00, reg_enc, 0b100,
  587                        scale, index_enc, base_enc);
  588       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  589         // [base + index*scale + imm8]
  590         // [01 reg 100][ss index base] imm8
  591         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  592                              scale, index_enc, base_enc,
  593                              disp);
  594       } else {
  595         // [base + index*scale + disp32]
  596         // [10 reg 100][ss index base] disp32
  597         emit_modrm_sib(0b10, reg_enc, 0b100,
  598                        scale, index_enc, base_enc);
  599         emit_data(disp, rspec, disp32_operand);
  600       }
  601     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  602       // [rsp + disp]
  603       if (disp == 0 && no_relocation) {
  604         // [rsp]
  605         // [00 reg 100][00 100 100]
  606         emit_modrm_sib(0b00, reg_enc, 0b100,
  607                        Address::times_1, 0b100, 0b100);
  608       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  609         // [rsp + imm8]
  610         // [01 reg 100][00 100 100] disp8
  611         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  612                              Address::times_1, 0b100, 0b100,
  613                              disp);
  614       } else {
  615         // [rsp + imm32]
  616         // [10 reg 100][00 100 100] disp32
  617         emit_modrm_sib(0b10, reg_enc, 0b100,
  618                        Address::times_1, 0b100, 0b100);
  619         emit_data(disp, rspec, disp32_operand);
  620       }
  621     } else {
  622       // [base + disp]
  623       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  624       if (disp == 0 && no_relocation &&
  625           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  626         // [base]
  627         // [00 reg base]
  628         emit_modrm(0, reg_enc, base_enc);
  629       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  630         // [base + disp8]
  631         // [01 reg base] disp8
  632         emit_modrm_disp8(0b01, reg_enc, base_enc,
  633                          disp);
  634       } else {
  635         // [base + disp32]
  636         // [10 reg base] disp32
  637         emit_modrm(0b10, reg_enc, base_enc);
  638         emit_data(disp, rspec, disp32_operand);
  639       }
  640     }
  641   } else {
  642     if (is_valid_encoding(index_enc)) {
  643       assert(scale != Address::no_scale, "inconsistent address");
  644       // base == noreg
  645       // [index*scale + disp]
  646       // [00 reg 100][ss index 101] disp32
  647       emit_modrm_sib(0b00, reg_enc, 0b100,
  648                      scale, index_enc, 0b101 /* no base */);
  649       emit_data(disp, rspec, disp32_operand);
  650     } else if (!no_relocation) {
  651       // base == noreg, index == noreg
  652       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  653       // [00 reg 101] disp32
  654 
  655       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  656       // Note that the RIP-rel. correction applies to the generated
  657       // disp field, but _not_ to the target address in the rspec.
  658 
  659       // disp was created by converting the target address minus the pc
  660       // at the start of the instruction. That needs more correction here.
  661       // intptr_t disp = target - next_ip;
  662       assert(inst_mark() != NULL, "must be inside InstructionMark");
  663       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  664       int64_t adjusted = disp;
  665       // Do rip-rel adjustment for 64bit
  666       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  667       assert(is_simm32(adjusted),
  668              "must be 32bit offset (RIP relative address)");
  669       emit_data((int32_t) adjusted, rspec, disp32_operand);
  670 
  671     } else {
  672       // base == noreg, index == noreg, no_relocation == true
  673       // 32bit never did this, did everything as the rip-rel/disp code above
  674       // [disp] ABSOLUTE
  675       // [00 reg 100][00 100 101] disp32
  676       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  677                      Address::times_1, 0b100, 0b101);
  678       emit_data(disp, rspec, disp32_operand);
  679     }
  680   }
  681 }
  682 
  683 void Assembler::emit_operand(Register reg, Register base, Register index,
  684                              Address::ScaleFactor scale, int disp,
  685                              RelocationHolder const& rspec,
  686                              int rip_relative_correction) {
  687   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  688   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  689                       scale, disp, rspec, rip_relative_correction);
  690 
  691 }
  692 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  693                              Address::ScaleFactor scale, int disp,
  694                              RelocationHolder const& rspec) {
  695   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  696   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  697   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  698                       scale, disp, rspec);
  699 }
  700 
  701 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  702                              Address::ScaleFactor scale, int disp,
  703                              RelocationHolder const& rspec) {
  704   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  705   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  706   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  707                       scale, disp, rspec, /* rip_relative_correction */ 0);
  708 }
  709 
  710 // Secret local extension to Assembler::WhichOperand:
  711 #define end_pc_operand (_WhichOperand_limit)
  712 
  713 address Assembler::locate_operand(address inst, WhichOperand which) {
  714   // Decode the given instruction, and return the address of
  715   // an embedded 32-bit operand word.
  716 
  717   // If "which" is disp32_operand, selects the displacement portion
  718   // of an effective address specifier.
  719   // If "which" is imm64_operand, selects the trailing immediate constant.
  720   // If "which" is call32_operand, selects the displacement of a call or jump.
  721   // Caller is responsible for ensuring that there is such an operand,
  722   // and that it is 32/64 bits wide.
  723 
  724   // If "which" is end_pc_operand, find the end of the instruction.
  725 
  726   address ip = inst;
  727   bool is_64bit = false;
  728 
  729   debug_only(bool has_disp32 = false);
  730   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  731 
  732   again_after_prefix:
  733   switch (0xFF & *ip++) {
  734 
  735   // These convenience macros generate groups of "case" labels for the switch.
  736 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  737 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  738              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  739 #define REP16(x) REP8((x)+0): \
  740               case REP8((x)+8)
  741 
  742   case CS_segment:
  743   case SS_segment:
  744   case DS_segment:
  745   case ES_segment:
  746   case FS_segment:
  747   case GS_segment:
  748     // Seems dubious
  749     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  750     assert(ip == inst+1, "only one prefix allowed");
  751     goto again_after_prefix;
  752 
  753   case 0x67:
  754   case REX:
  755   case REX_B:
  756   case REX_X:
  757   case REX_XB:
  758   case REX_R:
  759   case REX_RB:
  760   case REX_RX:
  761   case REX_RXB:
  762     NOT_LP64(assert(false, "64bit prefixes"));
  763     goto again_after_prefix;
  764 
  765   case REX_W:
  766   case REX_WB:
  767   case REX_WX:
  768   case REX_WXB:
  769   case REX_WR:
  770   case REX_WRB:
  771   case REX_WRX:
  772   case REX_WRXB:
  773     NOT_LP64(assert(false, "64bit prefixes"));
  774     is_64bit = true;
  775     goto again_after_prefix;
  776 
  777   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  778   case 0x88: // movb a, r
  779   case 0x89: // movl a, r
  780   case 0x8A: // movb r, a
  781   case 0x8B: // movl r, a
  782   case 0x8F: // popl a
  783     debug_only(has_disp32 = true);
  784     break;
  785 
  786   case 0x68: // pushq #32
  787     if (which == end_pc_operand) {
  788       return ip + 4;
  789     }
  790     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  791     return ip;                  // not produced by emit_operand
  792 
  793   case 0x66: // movw ... (size prefix)
  794     again_after_size_prefix2:
  795     switch (0xFF & *ip++) {
  796     case REX:
  797     case REX_B:
  798     case REX_X:
  799     case REX_XB:
  800     case REX_R:
  801     case REX_RB:
  802     case REX_RX:
  803     case REX_RXB:
  804     case REX_W:
  805     case REX_WB:
  806     case REX_WX:
  807     case REX_WXB:
  808     case REX_WR:
  809     case REX_WRB:
  810     case REX_WRX:
  811     case REX_WRXB:
  812       NOT_LP64(assert(false, "64bit prefix found"));
  813       goto again_after_size_prefix2;
  814     case 0x8B: // movw r, a
  815     case 0x89: // movw a, r
  816       debug_only(has_disp32 = true);
  817       break;
  818     case 0xC7: // movw a, #16
  819       debug_only(has_disp32 = true);
  820       tail_size = 2;  // the imm16
  821       break;
  822     case 0x0F: // several SSE/SSE2 variants
  823       ip--;    // reparse the 0x0F
  824       goto again_after_prefix;
  825     default:
  826       ShouldNotReachHere();
  827     }
  828     break;
  829 
  830   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  831     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  832     // these asserts are somewhat nonsensical
  833 #ifndef _LP64
  834     assert(which == imm_operand || which == disp32_operand,
  835            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  836 #else
  837     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  838            which == narrow_oop_operand && !is_64bit,
  839            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  840 #endif // _LP64
  841     return ip;
  842 
  843   case 0x69: // imul r, a, #32
  844   case 0xC7: // movl a, #32(oop?)
  845     tail_size = 4;
  846     debug_only(has_disp32 = true); // has both kinds of operands!
  847     break;
  848 
  849   case 0x0F: // movx..., etc.
  850     switch (0xFF & *ip++) {
  851     case 0x3A: // pcmpestri
  852       tail_size = 1;
  853     case 0x38: // ptest, pmovzxbw
  854       ip++; // skip opcode
  855       debug_only(has_disp32 = true); // has both kinds of operands!
  856       break;
  857 
  858     case 0x70: // pshufd r, r/a, #8
  859       debug_only(has_disp32 = true); // has both kinds of operands!
  860     case 0x73: // psrldq r, #8
  861       tail_size = 1;
  862       break;
  863 
  864     case 0x12: // movlps
  865     case 0x28: // movaps
  866     case 0x2E: // ucomiss
  867     case 0x2F: // comiss
  868     case 0x54: // andps
  869     case 0x55: // andnps
  870     case 0x56: // orps
  871     case 0x57: // xorps
  872     case 0x58: // addpd
  873     case 0x59: // mulpd
  874     case 0x6E: // movd
  875     case 0x7E: // movd
  876     case 0x6F: // movdq
  877     case 0x7F: // movdq
  878     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  879     case 0xFE: // paddd
  880       debug_only(has_disp32 = true);
  881       break;
  882 
  883     case 0xAD: // shrd r, a, %cl
  884     case 0xAF: // imul r, a
  885     case 0xBE: // movsbl r, a (movsxb)
  886     case 0xBF: // movswl r, a (movsxw)
  887     case 0xB6: // movzbl r, a (movzxb)
  888     case 0xB7: // movzwl r, a (movzxw)
  889     case REP16(0x40): // cmovl cc, r, a
  890     case 0xB0: // cmpxchgb
  891     case 0xB1: // cmpxchg
  892     case 0xC1: // xaddl
  893     case 0xC7: // cmpxchg8
  894     case REP16(0x90): // setcc a
  895       debug_only(has_disp32 = true);
  896       // fall out of the switch to decode the address
  897       break;
  898 
  899     case 0xC4: // pinsrw r, a, #8
  900       debug_only(has_disp32 = true);
  901     case 0xC5: // pextrw r, r, #8
  902       tail_size = 1;  // the imm8
  903       break;
  904 
  905     case 0xAC: // shrd r, a, #8
  906       debug_only(has_disp32 = true);
  907       tail_size = 1;  // the imm8
  908       break;
  909 
  910     case REP16(0x80): // jcc rdisp32
  911       if (which == end_pc_operand)  return ip + 4;
  912       assert(which == call32_operand, "jcc has no disp32 or imm");
  913       return ip;
  914     default:
  915       ShouldNotReachHere();
  916     }
  917     break;
  918 
  919   case 0x81: // addl a, #32; addl r, #32
  920     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  921     // on 32bit in the case of cmpl, the imm might be an oop
  922     tail_size = 4;
  923     debug_only(has_disp32 = true); // has both kinds of operands!
  924     break;
  925 
  926   case 0x83: // addl a, #8; addl r, #8
  927     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  928     debug_only(has_disp32 = true); // has both kinds of operands!
  929     tail_size = 1;
  930     break;
  931 
  932   case 0x9B:
  933     switch (0xFF & *ip++) {
  934     case 0xD9: // fnstcw a
  935       debug_only(has_disp32 = true);
  936       break;
  937     default:
  938       ShouldNotReachHere();
  939     }
  940     break;
  941 
  942   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  943   case REP4(0x10): // adc...
  944   case REP4(0x20): // and...
  945   case REP4(0x30): // xor...
  946   case REP4(0x08): // or...
  947   case REP4(0x18): // sbb...
  948   case REP4(0x28): // sub...
  949   case 0xF7: // mull a
  950   case 0x8D: // lea r, a
  951   case 0x87: // xchg r, a
  952   case REP4(0x38): // cmp...
  953   case 0x85: // test r, a
  954     debug_only(has_disp32 = true); // has both kinds of operands!
  955     break;
  956 
  957   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  958   case 0xC6: // movb a, #8
  959   case 0x80: // cmpb a, #8
  960   case 0x6B: // imul r, a, #8
  961     debug_only(has_disp32 = true); // has both kinds of operands!
  962     tail_size = 1; // the imm8
  963     break;
  964 
  965   case 0xC4: // VEX_3bytes
  966   case 0xC5: // VEX_2bytes
  967     assert((UseAVX > 0), "shouldn't have VEX prefix");
  968     assert(ip == inst+1, "no prefixes allowed");
  969     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  970     // but they have prefix 0x0F and processed when 0x0F processed above.
  971     //
  972     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
  973     // instructions (these instructions are not supported in 64-bit mode).
  974     // To distinguish them bits [7:6] are set in the VEX second byte since
  975     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
  976     // those VEX bits REX and vvvv bits are inverted.
  977     //
  978     // Fortunately C2 doesn't generate these instructions so we don't need
  979     // to check for them in product version.
  980 
  981     // Check second byte
  982     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
  983 
  984     int vex_opcode;
  985     // First byte
  986     if ((0xFF & *inst) == VEX_3bytes) {
  987       vex_opcode = VEX_OPCODE_MASK & *ip;
  988       ip++; // third byte
  989       is_64bit = ((VEX_W & *ip) == VEX_W);
  990     } else {
  991       vex_opcode = VEX_OPCODE_0F;
  992     }
  993     ip++; // opcode
  994     // To find the end of instruction (which == end_pc_operand).
  995     switch (vex_opcode) {
  996       case VEX_OPCODE_0F:
  997         switch (0xFF & *ip) {
  998         case 0x70: // pshufd r, r/a, #8
  999         case 0x71: // ps[rl|ra|ll]w r, #8
 1000         case 0x72: // ps[rl|ra|ll]d r, #8
 1001         case 0x73: // ps[rl|ra|ll]q r, #8
 1002         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1003         case 0xC4: // pinsrw r, r, r/a, #8
 1004         case 0xC5: // pextrw r/a, r, #8
 1005         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1006           tail_size = 1;  // the imm8
 1007           break;
 1008         }
 1009         break;
 1010       case VEX_OPCODE_0F_3A:
 1011         tail_size = 1;
 1012         break;
 1013     }
 1014     ip++; // skip opcode
 1015     debug_only(has_disp32 = true); // has both kinds of operands!
 1016     break;
 1017 
 1018   case 0x62: // EVEX_4bytes
 1019     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1020     assert(ip == inst+1, "no prefixes allowed");
 1021     // no EVEX collisions, all instructions that have 0x62 opcodes
 1022     // have EVEX versions and are subopcodes of 0x66
 1023     ip++; // skip P0 and exmaine W in P1
 1024     is_64bit = ((VEX_W & *ip) == VEX_W);
 1025     ip++; // move to P2
 1026     ip++; // skip P2, move to opcode
 1027     // To find the end of instruction (which == end_pc_operand).
 1028     switch (0xFF & *ip) {
 1029     case 0x22: // pinsrd r, r/a, #8
 1030     case 0x61: // pcmpestri r, r/a, #8
 1031     case 0x70: // pshufd r, r/a, #8
 1032     case 0x73: // psrldq r, #8
 1033     case 0x1f: // evpcmpd/evpcmpq
 1034     case 0x3f: // evpcmpb/evpcmpw
 1035       tail_size = 1;  // the imm8
 1036       break;
 1037     default:
 1038       break;
 1039     }
 1040     ip++; // skip opcode
 1041     debug_only(has_disp32 = true); // has both kinds of operands!
 1042     break;
 1043 
 1044   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1045   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1046   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1047   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1048   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1049   case 0xDF: // fild_d a; fistp_d a
 1050   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1051   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1052   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1053     debug_only(has_disp32 = true);
 1054     break;
 1055 
 1056   case 0xE8: // call rdisp32
 1057   case 0xE9: // jmp  rdisp32
 1058     if (which == end_pc_operand)  return ip + 4;
 1059     assert(which == call32_operand, "call has no disp32 or imm");
 1060     return ip;
 1061 
 1062   case 0xF0:                    // Lock
 1063     goto again_after_prefix;
 1064 
 1065   case 0xF3:                    // For SSE
 1066   case 0xF2:                    // For SSE2
 1067     switch (0xFF & *ip++) {
 1068     case REX:
 1069     case REX_B:
 1070     case REX_X:
 1071     case REX_XB:
 1072     case REX_R:
 1073     case REX_RB:
 1074     case REX_RX:
 1075     case REX_RXB:
 1076     case REX_W:
 1077     case REX_WB:
 1078     case REX_WX:
 1079     case REX_WXB:
 1080     case REX_WR:
 1081     case REX_WRB:
 1082     case REX_WRX:
 1083     case REX_WRXB:
 1084       NOT_LP64(assert(false, "found 64bit prefix"));
 1085       ip++;
 1086     default:
 1087       ip++;
 1088     }
 1089     debug_only(has_disp32 = true); // has both kinds of operands!
 1090     break;
 1091 
 1092   default:
 1093     ShouldNotReachHere();
 1094 
 1095 #undef REP8
 1096 #undef REP16
 1097   }
 1098 
 1099   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1100 #ifdef _LP64
 1101   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1102 #else
 1103   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1104   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1105 #endif // LP64
 1106   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1107 
 1108   // parse the output of emit_operand
 1109   int op2 = 0xFF & *ip++;
 1110   int base = op2 & 0x07;
 1111   int op3 = -1;
 1112   const int b100 = 4;
 1113   const int b101 = 5;
 1114   if (base == b100 && (op2 >> 6) != 3) {
 1115     op3 = 0xFF & *ip++;
 1116     base = op3 & 0x07;   // refetch the base
 1117   }
 1118   // now ip points at the disp (if any)
 1119 
 1120   switch (op2 >> 6) {
 1121   case 0:
 1122     // [00 reg  100][ss index base]
 1123     // [00 reg  100][00   100  esp]
 1124     // [00 reg base]
 1125     // [00 reg  100][ss index  101][disp32]
 1126     // [00 reg  101]               [disp32]
 1127 
 1128     if (base == b101) {
 1129       if (which == disp32_operand)
 1130         return ip;              // caller wants the disp32
 1131       ip += 4;                  // skip the disp32
 1132     }
 1133     break;
 1134 
 1135   case 1:
 1136     // [01 reg  100][ss index base][disp8]
 1137     // [01 reg  100][00   100  esp][disp8]
 1138     // [01 reg base]               [disp8]
 1139     ip += 1;                    // skip the disp8
 1140     break;
 1141 
 1142   case 2:
 1143     // [10 reg  100][ss index base][disp32]
 1144     // [10 reg  100][00   100  esp][disp32]
 1145     // [10 reg base]               [disp32]
 1146     if (which == disp32_operand)
 1147       return ip;                // caller wants the disp32
 1148     ip += 4;                    // skip the disp32
 1149     break;
 1150 
 1151   case 3:
 1152     // [11 reg base]  (not a memory addressing mode)
 1153     break;
 1154   }
 1155 
 1156   if (which == end_pc_operand) {
 1157     return ip + tail_size;
 1158   }
 1159 
 1160 #ifdef _LP64
 1161   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1162 #else
 1163   assert(which == imm_operand, "instruction has only an imm field");
 1164 #endif // LP64
 1165   return ip;
 1166 }
 1167 
 1168 address Assembler::locate_next_instruction(address inst) {
 1169   // Secretly share code with locate_operand:
 1170   return locate_operand(inst, end_pc_operand);
 1171 }
 1172 
 1173 
 1174 #ifdef ASSERT
 1175 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1176   address inst = inst_mark();
 1177   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1178   address opnd;
 1179 
 1180   Relocation* r = rspec.reloc();
 1181   if (r->type() == relocInfo::none) {
 1182     return;
 1183   } else if (r->is_call() || format == call32_operand) {
 1184     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1185     opnd = locate_operand(inst, call32_operand);
 1186   } else if (r->is_data()) {
 1187     assert(format == imm_operand || format == disp32_operand
 1188            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1189     opnd = locate_operand(inst, (WhichOperand)format);
 1190   } else {
 1191     assert(format == imm_operand, "cannot specify a format");
 1192     return;
 1193   }
 1194   assert(opnd == pc(), "must put operand where relocs can find it");
 1195 }
 1196 #endif // ASSERT
 1197 
 1198 void Assembler::emit_operand(Register reg, Address adr,
 1199                              int rip_relative_correction) {
 1200   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1201                adr._rspec,
 1202                rip_relative_correction);
 1203 }
 1204 
 1205 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1206     if (adr.isxmmindex()) {
 1207        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1208     } else {
 1209        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1210        adr._rspec);
 1211     }
 1212 }
 1213 
 1214 // Now the Assembler instructions (identical for 32/64 bits)
 1215 
 1216 void Assembler::adcl(Address dst, int32_t imm32) {
 1217   InstructionMark im(this);
 1218   prefix(dst);
 1219   emit_arith_operand(0x81, rdx, dst, imm32);
 1220 }
 1221 
 1222 void Assembler::adcl(Address dst, Register src) {
 1223   InstructionMark im(this);
 1224   prefix(dst, src);
 1225   emit_int8(0x11);
 1226   emit_operand(src, dst);
 1227 }
 1228 
 1229 void Assembler::adcl(Register dst, int32_t imm32) {
 1230   prefix(dst);
 1231   emit_arith(0x81, 0xD0, dst, imm32);
 1232 }
 1233 
 1234 void Assembler::adcl(Register dst, Address src) {
 1235   InstructionMark im(this);
 1236   prefix(src, dst);
 1237   emit_int8(0x13);
 1238   emit_operand(dst, src);
 1239 }
 1240 
 1241 void Assembler::adcl(Register dst, Register src) {
 1242   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1243   emit_arith(0x13, 0xC0, dst, src);
 1244 }
 1245 
 1246 void Assembler::addl(Address dst, int32_t imm32) {
 1247   InstructionMark im(this);
 1248   prefix(dst);
 1249   emit_arith_operand(0x81, rax, dst, imm32);
 1250 }
 1251 
 1252 void Assembler::addb(Address dst, int imm8) {
 1253   InstructionMark im(this);
 1254   prefix(dst);
 1255   emit_int8((unsigned char)0x80);
 1256   emit_operand(rax, dst, 1);
 1257   emit_int8(imm8);
 1258 }
 1259 
 1260 void Assembler::addw(Register dst, Register src) {
 1261   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1262   emit_arith(0x03, 0xC0, dst, src);
 1263 }
 1264 
 1265 void Assembler::addw(Address dst, int imm16) {
 1266   InstructionMark im(this);
 1267   emit_int8(0x66);
 1268   prefix(dst);
 1269   emit_int8((unsigned char)0x81);
 1270   emit_operand(rax, dst, 2);
 1271   emit_int16(imm16);
 1272 }
 1273 
 1274 void Assembler::addl(Address dst, Register src) {
 1275   InstructionMark im(this);
 1276   prefix(dst, src);
 1277   emit_int8(0x01);
 1278   emit_operand(src, dst);
 1279 }
 1280 
 1281 void Assembler::addl(Register dst, int32_t imm32) {
 1282   prefix(dst);
 1283   emit_arith(0x81, 0xC0, dst, imm32);
 1284 }
 1285 
 1286 void Assembler::addl(Register dst, Address src) {
 1287   InstructionMark im(this);
 1288   prefix(src, dst);
 1289   emit_int8(0x03);
 1290   emit_operand(dst, src);
 1291 }
 1292 
 1293 void Assembler::addl(Register dst, Register src) {
 1294   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1295   emit_arith(0x03, 0xC0, dst, src);
 1296 }
 1297 
 1298 void Assembler::addr_nop_4() {
 1299   assert(UseAddressNop, "no CPU support");
 1300   // 4 bytes: NOP DWORD PTR [EAX+0]
 1301   emit_int32(0x0F,
 1302              0x1F,
 1303              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1304              0);   // 8-bits offset (1 byte)
 1305 }
 1306 
 1307 void Assembler::addr_nop_5() {
 1308   assert(UseAddressNop, "no CPU support");
 1309   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1310   emit_int32(0x0F,
 1311              0x1F,
 1312              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1313              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1314   emit_int8(0);     // 8-bits offset (1 byte)
 1315 }
 1316 
 1317 void Assembler::addr_nop_7() {
 1318   assert(UseAddressNop, "no CPU support");
 1319   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1320   emit_int24(0x0F,
 1321              0x1F,
 1322              (unsigned char)0x80);
 1323                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1324   emit_int32(0);   // 32-bits offset (4 bytes)
 1325 }
 1326 
 1327 void Assembler::addr_nop_8() {
 1328   assert(UseAddressNop, "no CPU support");
 1329   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1330   emit_int32(0x0F,
 1331              0x1F,
 1332              (unsigned char)0x84,
 1333                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1334              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1335   emit_int32(0);    // 32-bits offset (4 bytes)
 1336 }
 1337 
 1338 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1341   attributes.set_rex_vex_w_reverted();
 1342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1343   emit_int16(0x58, (0xC0 | encode));
 1344 }
 1345 
 1346 void Assembler::addsd(XMMRegister dst, Address src) {
 1347   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1348   InstructionMark im(this);
 1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1350   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1351   attributes.set_rex_vex_w_reverted();
 1352   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1353   emit_int8(0x58);
 1354   emit_operand(dst, src);
 1355 }
 1356 
 1357 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1358   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1360   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1361   emit_int16(0x58, (0xC0 | encode));
 1362 }
 1363 
 1364 void Assembler::addss(XMMRegister dst, Address src) {
 1365   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1366   InstructionMark im(this);
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1369   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1370   emit_int8(0x58);
 1371   emit_operand(dst, src);
 1372 }
 1373 
 1374 void Assembler::aesdec(XMMRegister dst, Address src) {
 1375   assert(VM_Version::supports_aes(), "");
 1376   InstructionMark im(this);
 1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1378   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1379   emit_int8((unsigned char)0xDE);
 1380   emit_operand(dst, src);
 1381 }
 1382 
 1383 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1384   assert(VM_Version::supports_aes(), "");
 1385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1386   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1387   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1388 }
 1389 
 1390 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1391   assert(VM_Version::supports_avx512_vaes(), "");
 1392   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1393   attributes.set_is_evex_instruction();
 1394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1395   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1396 }
 1397 
 1398 
 1399 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1400   assert(VM_Version::supports_aes(), "");
 1401   InstructionMark im(this);
 1402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1403   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1404   emit_int8((unsigned char)0xDF);
 1405   emit_operand(dst, src);
 1406 }
 1407 
 1408 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1409   assert(VM_Version::supports_aes(), "");
 1410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1411   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1412   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1413 }
 1414 
 1415 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1416   assert(VM_Version::supports_avx512_vaes(), "");
 1417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1418   attributes.set_is_evex_instruction();
 1419   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1420   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1421 }
 1422 
 1423 void Assembler::aesenc(XMMRegister dst, Address src) {
 1424   assert(VM_Version::supports_aes(), "");
 1425   InstructionMark im(this);
 1426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1427   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1428   emit_int8((unsigned char)0xDC);
 1429   emit_operand(dst, src);
 1430 }
 1431 
 1432 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1433   assert(VM_Version::supports_aes(), "");
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1437 }
 1438 
 1439 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1440   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1442   attributes.set_is_evex_instruction();
 1443   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1448   assert(VM_Version::supports_aes(), "");
 1449   InstructionMark im(this);
 1450   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1451   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int8((unsigned char)0xDD);
 1453   emit_operand(dst, src);
 1454 }
 1455 
 1456 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1460   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1461 }
 1462 
 1463 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1464   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1465   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1466   attributes.set_is_evex_instruction();
 1467   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1468   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1469 }
 1470 
 1471 void Assembler::andb(Address dst, Register src) {
 1472   InstructionMark im(this);
 1473   prefix(dst, src, true);
 1474   emit_int8(0x20);
 1475   emit_operand(src, dst);
 1476 }
 1477 
 1478 void Assembler::andw(Register dst, Register src) {
 1479   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1480   emit_arith(0x23, 0xC0, dst, src);
 1481 }
 1482 
 1483 void Assembler::andl(Address dst, int32_t imm32) {
 1484   InstructionMark im(this);
 1485   prefix(dst);
 1486   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1487 }
 1488 
 1489 void Assembler::andl(Register dst, int32_t imm32) {
 1490   prefix(dst);
 1491   emit_arith(0x81, 0xE0, dst, imm32);
 1492 }
 1493 
 1494 void Assembler::andl(Address dst, Register src) {
 1495   InstructionMark im(this);
 1496   prefix(dst, src);
 1497   emit_int8(0x21);
 1498   emit_operand(src, dst);
 1499 }
 1500 
 1501 void Assembler::andl(Register dst, Address src) {
 1502   InstructionMark im(this);
 1503   prefix(src, dst);
 1504   emit_int8(0x23);
 1505   emit_operand(dst, src);
 1506 }
 1507 
 1508 void Assembler::andl(Register dst, Register src) {
 1509   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1510   emit_arith(0x23, 0xC0, dst, src);
 1511 }
 1512 
 1513 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1514   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1521   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1522   InstructionMark im(this);
 1523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1524   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int8((unsigned char)0xF2);
 1526   emit_operand(dst, src2);
 1527 }
 1528 
 1529 void Assembler::bsfl(Register dst, Register src) {
 1530   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1531   emit_int24(0x0F,
 1532              (unsigned char)0xBC,
 1533              0xC0 | encode);
 1534 }
 1535 
 1536 void Assembler::bsrl(Register dst, Register src) {
 1537   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1538   emit_int24(0x0F,
 1539              (unsigned char)0xBD,
 1540              0xC0 | encode);
 1541 }
 1542 
 1543 void Assembler::bswapl(Register reg) { // bswap
 1544   int encode = prefix_and_encode(reg->encoding());
 1545   emit_int16(0x0F, (0xC8 | encode));
 1546 }
 1547 
 1548 void Assembler::blsil(Register dst, Register src) {
 1549   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1550   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1553 }
 1554 
 1555 void Assembler::blsil(Register dst, Address src) {
 1556   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1557   InstructionMark im(this);
 1558   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1559   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1560   emit_int8((unsigned char)0xF3);
 1561   emit_operand(rbx, src);
 1562 }
 1563 
 1564 void Assembler::blsmskl(Register dst, Register src) {
 1565   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1566   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1567   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1568   emit_int16((unsigned char)0xF3,
 1569              0xC0 | encode);
 1570 }
 1571 
 1572 void Assembler::blsmskl(Register dst, Address src) {
 1573   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1574   InstructionMark im(this);
 1575   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1576   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1577   emit_int8((unsigned char)0xF3);
 1578   emit_operand(rdx, src);
 1579 }
 1580 
 1581 void Assembler::blsrl(Register dst, Register src) {
 1582   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1583   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1584   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1585   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1586 }
 1587 
 1588 void Assembler::blsrl(Register dst, Address src) {
 1589   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1590   InstructionMark im(this);
 1591   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1592   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1593   emit_int8((unsigned char)0xF3);
 1594   emit_operand(rcx, src);
 1595 }
 1596 
 1597 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1598   // suspect disp32 is always good
 1599   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1600 
 1601   if (L.is_bound()) {
 1602     const int long_size = 5;
 1603     int offs = (int)( target(L) - pc() );
 1604     assert(offs <= 0, "assembler error");
 1605     InstructionMark im(this);
 1606     // 1110 1000 #32-bit disp
 1607     emit_int8((unsigned char)0xE8);
 1608     emit_data(offs - long_size, rtype, operand);
 1609   } else {
 1610     InstructionMark im(this);
 1611     // 1110 1000 #32-bit disp
 1612     L.add_patch_at(code(), locator());
 1613 
 1614     emit_int8((unsigned char)0xE8);
 1615     emit_data(int(0), rtype, operand);
 1616   }
 1617 }
 1618 
 1619 void Assembler::call(Register dst) {
 1620   int encode = prefix_and_encode(dst->encoding());
 1621   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1622 }
 1623 
 1624 
 1625 void Assembler::call(Address adr) {
 1626   InstructionMark im(this);
 1627   prefix(adr);
 1628   emit_int8((unsigned char)0xFF);
 1629   emit_operand(rdx, adr);
 1630 }
 1631 
 1632 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1633   InstructionMark im(this);
 1634   emit_int8((unsigned char)0xE8);
 1635   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1636   // Entry is NULL in case of a scratch emit.
 1637   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1638   // Technically, should use call32_operand, but this format is
 1639   // implied by the fact that we're emitting a call instruction.
 1640 
 1641   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1642   emit_data((int) disp, rspec, operand);
 1643 }
 1644 
 1645 void Assembler::cdql() {
 1646   emit_int8((unsigned char)0x99);
 1647 }
 1648 
 1649 void Assembler::cld() {
 1650   emit_int8((unsigned char)0xFC);
 1651 }
 1652 
 1653 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1654   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1655   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1656   emit_int24(0x0F,
 1657              0x40 | cc,
 1658              0xC0 | encode);
 1659 }
 1660 
 1661 
 1662 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1663   InstructionMark im(this);
 1664   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1665   prefix(src, dst);
 1666   emit_int16(0x0F, (0x40 | cc));
 1667   emit_operand(dst, src);
 1668 }
 1669 
 1670 void Assembler::cmpb(Address dst, int imm8) {
 1671   InstructionMark im(this);
 1672   prefix(dst);
 1673   emit_int8((unsigned char)0x80);
 1674   emit_operand(rdi, dst, 1);
 1675   emit_int8(imm8);
 1676 }
 1677 
 1678 void Assembler::cmpl(Address dst, int32_t imm32) {
 1679   InstructionMark im(this);
 1680   prefix(dst);
 1681   emit_int8((unsigned char)0x81);
 1682   emit_operand(rdi, dst, 4);
 1683   emit_int32(imm32);
 1684 }
 1685 
 1686 void Assembler::cmp(Register dst, int32_t imm32) {
 1687   prefix(dst);
 1688   emit_int8((unsigned char)0x3D);
 1689   emit_int32(imm32);
 1690 }
 1691 
 1692 void Assembler::cmpl(Register dst, int32_t imm32) {
 1693   prefix(dst);
 1694   emit_arith(0x81, 0xF8, dst, imm32);
 1695 }
 1696 
 1697 void Assembler::cmpl(Register dst, Register src) {
 1698   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1699   emit_arith(0x3B, 0xC0, dst, src);
 1700 }
 1701 
 1702 void Assembler::cmpl(Register dst, Address  src) {
 1703   InstructionMark im(this);
 1704   prefix(src, dst);
 1705   emit_int8(0x3B);
 1706   emit_operand(dst, src);
 1707 }
 1708 
 1709 void Assembler::cmpw(Address dst, int imm16) {
 1710   InstructionMark im(this);
 1711   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1712   emit_int16(0x66, (unsigned char)0x81);
 1713   emit_operand(rdi, dst, 2);
 1714   emit_int16(imm16);
 1715 }
 1716 
 1717 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1718 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1719 // The ZF is set if the compared values were equal, and cleared otherwise.
 1720 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1721   InstructionMark im(this);
 1722   prefix(adr, reg);
 1723   emit_int16(0x0F, (unsigned char)0xB1);
 1724   emit_operand(reg, adr);
 1725 }
 1726 
 1727 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1728   InstructionMark im(this);
 1729   size_prefix();
 1730   prefix(adr, reg);
 1731   emit_int16(0x0F, (unsigned char)0xB1);
 1732   emit_operand(reg, adr);
 1733 }
 1734 
 1735 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1736 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1737 // The ZF is set if the compared values were equal, and cleared otherwise.
 1738 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1739   InstructionMark im(this);
 1740   prefix(adr, reg, true);
 1741   emit_int16(0x0F, (unsigned char)0xB0);
 1742   emit_operand(reg, adr);
 1743 }
 1744 
 1745 void Assembler::comisd(XMMRegister dst, Address src) {
 1746   // NOTE: dbx seems to decode this as comiss even though the
 1747   // 0x66 is there. Strangly ucomisd comes out correct
 1748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1749   InstructionMark im(this);
 1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1751   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1752   attributes.set_rex_vex_w_reverted();
 1753   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1754   emit_int8(0x2F);
 1755   emit_operand(dst, src);
 1756 }
 1757 
 1758 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1760   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1761   attributes.set_rex_vex_w_reverted();
 1762   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1763   emit_int16(0x2F, (0xC0 | encode));
 1764 }
 1765 
 1766 void Assembler::comiss(XMMRegister dst, Address src) {
 1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1768   InstructionMark im(this);
 1769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1770   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1771   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1772   emit_int8(0x2F);
 1773   emit_operand(dst, src);
 1774 }
 1775 
 1776 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1777   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1779   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1780   emit_int16(0x2F, (0xC0 | encode));
 1781 }
 1782 
 1783 void Assembler::cpuid() {
 1784   emit_int16(0x0F, (unsigned char)0xA2);
 1785 }
 1786 
 1787 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1788 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1789 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1790 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1791 //
 1792 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1793 //
 1794 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1795 //
 1796 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1797 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1798   assert(VM_Version::supports_sse4_2(), "");
 1799   int8_t w = 0x01;
 1800   Prefix p = Prefix_EMPTY;
 1801 
 1802   emit_int8((unsigned char)0xF2);
 1803   switch (sizeInBytes) {
 1804   case 1:
 1805     w = 0;
 1806     break;
 1807   case 2:
 1808   case 4:
 1809     break;
 1810   LP64_ONLY(case 8:)
 1811     // This instruction is not valid in 32 bits
 1812     // Note:
 1813     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1814     //
 1815     // Page B - 72   Vol. 2C says
 1816     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1817     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1818     //                                                                            F0!!!
 1819     // while 3 - 208 Vol. 2A
 1820     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1821     //
 1822     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1823     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1824     p = REX_W;
 1825     break;
 1826   default:
 1827     assert(0, "Unsupported value for a sizeInBytes argument");
 1828     break;
 1829   }
 1830   LP64_ONLY(prefix(crc, v, p);)
 1831   emit_int32(0x0F,
 1832              0x38,
 1833              0xF0 | w,
 1834              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1835 }
 1836 
 1837 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1838   assert(VM_Version::supports_sse4_2(), "");
 1839   InstructionMark im(this);
 1840   int8_t w = 0x01;
 1841   Prefix p = Prefix_EMPTY;
 1842 
 1843   emit_int8((int8_t)0xF2);
 1844   switch (sizeInBytes) {
 1845   case 1:
 1846     w = 0;
 1847     break;
 1848   case 2:
 1849   case 4:
 1850     break;
 1851   LP64_ONLY(case 8:)
 1852     // This instruction is not valid in 32 bits
 1853     p = REX_W;
 1854     break;
 1855   default:
 1856     assert(0, "Unsupported value for a sizeInBytes argument");
 1857     break;
 1858   }
 1859   LP64_ONLY(prefix(crc, adr, p);)
 1860   emit_int24(0x0F, 0x38, (0xF0 | w));
 1861   emit_operand(crc, adr);
 1862 }
 1863 
 1864 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1868   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1869 }
 1870 
 1871 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1872   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1875   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1876 }
 1877 
 1878 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1882   emit_int16(0x5B, (0xC0 | encode));
 1883 }
 1884 
 1885 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1886   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1887   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1889   emit_int16(0x5B, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1893   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1894   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1895   attributes.set_rex_vex_w_reverted();
 1896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1897   emit_int16(0x5A, (0xC0 | encode));
 1898 }
 1899 
 1900 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1901   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1902   InstructionMark im(this);
 1903   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1904   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1905   attributes.set_rex_vex_w_reverted();
 1906   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1907   emit_int8(0x5A);
 1908   emit_operand(dst, src);
 1909 }
 1910 
 1911 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1914   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1915   emit_int16(0x2A, (0xC0 | encode));
 1916 }
 1917 
 1918 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1919   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1920   InstructionMark im(this);
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1922   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1923   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1924   emit_int8(0x2A);
 1925   emit_operand(dst, src);
 1926 }
 1927 
 1928 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1929   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1931   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1932   emit_int16(0x2A, (0xC0 | encode));
 1933 }
 1934 
 1935 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1937   InstructionMark im(this);
 1938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1939   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1940   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1941   emit_int8(0x2A);
 1942   emit_operand(dst, src);
 1943 }
 1944 
 1945 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1946   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1947   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1948   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1949   emit_int16(0x2A, (0xC0 | encode));
 1950 }
 1951 
 1952 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1953   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1955   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1956   emit_int16(0x5A, (0xC0 | encode));
 1957 }
 1958 
 1959 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1961   InstructionMark im(this);
 1962   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1963   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1964   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1965   emit_int8(0x5A);
 1966   emit_operand(dst, src);
 1967 }
 1968 
 1969 
 1970 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1971   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1973   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1974   emit_int16(0x2C, (0xC0 | encode));
 1975 }
 1976 
 1977 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 1978   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1980   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1981   emit_int16(0x2C, (0xC0 | encode));
 1982 }
 1983 
 1984 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 1985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1986   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 1987   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1988   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1989   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1990 }
 1991 
 1992 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 1993   assert(VM_Version::supports_ssse3(), "");
 1994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 1995   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1996   emit_int16(0x1C, (0xC0 | encode));
 1997 }
 1998 
 1999 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2000   assert(VM_Version::supports_ssse3(), "");
 2001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2002   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2003   emit_int16(0x1D, (0xC0 | encode));
 2004 }
 2005 
 2006 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2007   assert(VM_Version::supports_ssse3(), "");
 2008   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2009   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2010   emit_int16(0x1E, (0xC0 | encode));
 2011 }
 2012 
 2013 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2014   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2015          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2016          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2017   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2018   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2019   emit_int16(0x1C, (0xC0 | encode));
 2020 }
 2021 
 2022 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2023   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2024          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2025          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2026   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2027   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2028   emit_int16(0x1D, (0xC0 | encode));
 2029 }
 2030 
 2031 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2032   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2033   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2034   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2035   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2037   emit_int16(0x1E, (0xC0 | encode));
 2038 }
 2039 
 2040 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2041   assert(UseAVX > 2, "");
 2042   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2043   attributes.set_is_evex_instruction();
 2044   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2045   emit_int16(0x1F, (0xC0 | encode));
 2046 }
 2047 
 2048 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2049   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2050   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2052   emit_int16(0x5A, (0xC0 | encode));
 2053 }
 2054 
 2055 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2056   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2057   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2058   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2059   attributes.set_rex_vex_w_reverted();
 2060   emit_int16(0x5A, (0xC0 | encode));
 2061 }
 2062 
 2063 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2064   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2065   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2066   attributes.set_is_evex_instruction();
 2067   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2068   emit_int16(0x5B, (0xC0 | encode));
 2069 }
 2070 
 2071 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2072   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2073   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2074   attributes.set_is_evex_instruction();
 2075   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2076   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2077 }
 2078 
 2079 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2080   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2082   attributes.set_is_evex_instruction();
 2083   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2084   emit_int16(0x30, (0xC0 | encode));
 2085 }
 2086 
 2087 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2088   assert(UseAVX > 2, "");
 2089   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2090   attributes.set_is_evex_instruction();
 2091   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2092   emit_int16(0x33, (0xC0 | encode));
 2093 }
 2094 
 2095 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2096   assert(UseAVX > 2, "");
 2097   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2098   attributes.set_is_evex_instruction();
 2099   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x31, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(UseAVX > 2, "");
 2105   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2106   attributes.set_is_evex_instruction();
 2107   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2108   emit_int16(0x35, (0xC0 | encode));
 2109 }
 2110 
 2111 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2112   assert(UseAVX > 2, "");
 2113   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2114   attributes.set_is_evex_instruction();
 2115   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2116   emit_int16(0x32, (0xC0 | encode));
 2117 }
 2118 
 2119 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2120   assert(UseAVX > 2, "");
 2121   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2122   attributes.set_is_evex_instruction();
 2123   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2124   emit_int16(0x34, (0xC0 | encode));
 2125 }
 2126 
 2127 void Assembler::decl(Address dst) {
 2128   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2129   InstructionMark im(this);
 2130   prefix(dst);
 2131   emit_int8((unsigned char)0xFF);
 2132   emit_operand(rcx, dst);
 2133 }
 2134 
 2135 void Assembler::divsd(XMMRegister dst, Address src) {
 2136   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2137   InstructionMark im(this);
 2138   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2139   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2140   attributes.set_rex_vex_w_reverted();
 2141   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2142   emit_int8(0x5E);
 2143   emit_operand(dst, src);
 2144 }
 2145 
 2146 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2147   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2148   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2149   attributes.set_rex_vex_w_reverted();
 2150   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2151   emit_int16(0x5E, (0xC0 | encode));
 2152 }
 2153 
 2154 void Assembler::divss(XMMRegister dst, Address src) {
 2155   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2156   InstructionMark im(this);
 2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2158   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2159   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2160   emit_int8(0x5E);
 2161   emit_operand(dst, src);
 2162 }
 2163 
 2164 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2168   emit_int16(0x5E, (0xC0 | encode));
 2169 }
 2170 
 2171 void Assembler::hlt() {
 2172   emit_int8((unsigned char)0xF4);
 2173 }
 2174 
 2175 void Assembler::idivl(Register src) {
 2176   int encode = prefix_and_encode(src->encoding());
 2177   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2178 }
 2179 
 2180 void Assembler::divl(Register src) { // Unsigned
 2181   int encode = prefix_and_encode(src->encoding());
 2182   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2183 }
 2184 
 2185 void Assembler::imull(Register src) {
 2186   int encode = prefix_and_encode(src->encoding());
 2187   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2188 }
 2189 
 2190 void Assembler::imull(Register dst, Register src) {
 2191   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2192   emit_int24(0x0F,
 2193              (unsigned char)0xAF,
 2194              (0xC0 | encode));
 2195 }
 2196 
 2197 void Assembler::imull(Register dst, Address src, int32_t value) {
 2198   InstructionMark im(this);
 2199   prefix(src, dst);
 2200   if (is8bit(value)) {
 2201     emit_int8((unsigned char)0x6B);
 2202     emit_operand(dst, src);
 2203     emit_int8(value);
 2204   } else {
 2205     emit_int8((unsigned char)0x69);
 2206     emit_operand(dst, src);
 2207     emit_int32(value);
 2208   }
 2209 }
 2210 
 2211 void Assembler::imull(Register dst, Register src, int value) {
 2212   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2213   if (is8bit(value)) {
 2214     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2215   } else {
 2216     emit_int16(0x69, (0xC0 | encode));
 2217     emit_int32(value);
 2218   }
 2219 }
 2220 
 2221 void Assembler::imull(Register dst, Address src) {
 2222   InstructionMark im(this);
 2223   prefix(src, dst);
 2224   emit_int16(0x0F, (unsigned char)0xAF);
 2225   emit_operand(dst, src);
 2226 }
 2227 
 2228 
 2229 void Assembler::incl(Address dst) {
 2230   // Don't use it directly. Use MacroAssembler::increment() instead.
 2231   InstructionMark im(this);
 2232   prefix(dst);
 2233   emit_int8((unsigned char)0xFF);
 2234   emit_operand(rax, dst);
 2235 }
 2236 
 2237 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2238   InstructionMark im(this);
 2239   assert((0 <= cc) && (cc < 16), "illegal cc");
 2240   if (L.is_bound()) {
 2241     address dst = target(L);
 2242     assert(dst != NULL, "jcc most probably wrong");
 2243 
 2244     const int short_size = 2;
 2245     const int long_size = 6;
 2246     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2247     if (maybe_short && is8bit(offs - short_size)) {
 2248       // 0111 tttn #8-bit disp
 2249       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2250     } else {
 2251       // 0000 1111 1000 tttn #32-bit disp
 2252       assert(is_simm32(offs - long_size),
 2253              "must be 32bit offset (call4)");
 2254       emit_int16(0x0F, (0x80 | cc));
 2255       emit_int32(offs - long_size);
 2256     }
 2257   } else {
 2258     // Note: could eliminate cond. jumps to this jump if condition
 2259     //       is the same however, seems to be rather unlikely case.
 2260     // Note: use jccb() if label to be bound is very close to get
 2261     //       an 8-bit displacement
 2262     L.add_patch_at(code(), locator());
 2263     emit_int16(0x0F, (0x80 | cc));
 2264     emit_int32(0);
 2265   }
 2266 }
 2267 
 2268 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2269   if (L.is_bound()) {
 2270     const int short_size = 2;
 2271     address entry = target(L);
 2272 #ifdef ASSERT
 2273     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2274     intptr_t delta = short_branch_delta();
 2275     if (delta != 0) {
 2276       dist += (dist < 0 ? (-delta) :delta);
 2277     }
 2278     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2279 #endif
 2280     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2281     // 0111 tttn #8-bit disp
 2282     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2283   } else {
 2284     InstructionMark im(this);
 2285     L.add_patch_at(code(), locator(), file, line);
 2286     emit_int16(0x70 | cc, 0);
 2287   }
 2288 }
 2289 
 2290 void Assembler::jmp(Address adr) {
 2291   InstructionMark im(this);
 2292   prefix(adr);
 2293   emit_int8((unsigned char)0xFF);
 2294   emit_operand(rsp, adr);
 2295 }
 2296 
 2297 void Assembler::jmp(Label& L, bool maybe_short) {
 2298   if (L.is_bound()) {
 2299     address entry = target(L);
 2300     assert(entry != NULL, "jmp most probably wrong");
 2301     InstructionMark im(this);
 2302     const int short_size = 2;
 2303     const int long_size = 5;
 2304     intptr_t offs = entry - pc();
 2305     if (maybe_short && is8bit(offs - short_size)) {
 2306       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2307     } else {
 2308       emit_int8((unsigned char)0xE9);
 2309       emit_int32(offs - long_size);
 2310     }
 2311   } else {
 2312     // By default, forward jumps are always 32-bit displacements, since
 2313     // we can't yet know where the label will be bound.  If you're sure that
 2314     // the forward jump will not run beyond 256 bytes, use jmpb to
 2315     // force an 8-bit displacement.
 2316     InstructionMark im(this);
 2317     L.add_patch_at(code(), locator());
 2318     emit_int8((unsigned char)0xE9);
 2319     emit_int32(0);
 2320   }
 2321 }
 2322 
 2323 void Assembler::jmp(Register entry) {
 2324   int encode = prefix_and_encode(entry->encoding());
 2325   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2326 }
 2327 
 2328 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2329   InstructionMark im(this);
 2330   emit_int8((unsigned char)0xE9);
 2331   assert(dest != NULL, "must have a target");
 2332   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2333   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2334   emit_data(disp, rspec.reloc(), call32_operand);
 2335 }
 2336 
 2337 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2338   if (L.is_bound()) {
 2339     const int short_size = 2;
 2340     address entry = target(L);
 2341     assert(entry != NULL, "jmp most probably wrong");
 2342 #ifdef ASSERT
 2343     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2344     intptr_t delta = short_branch_delta();
 2345     if (delta != 0) {
 2346       dist += (dist < 0 ? (-delta) :delta);
 2347     }
 2348     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2349 #endif
 2350     intptr_t offs = entry - pc();
 2351     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2352   } else {
 2353     InstructionMark im(this);
 2354     L.add_patch_at(code(), locator(), file, line);
 2355     emit_int16((unsigned char)0xEB, 0);
 2356   }
 2357 }
 2358 
 2359 void Assembler::ldmxcsr( Address src) {
 2360   if (UseAVX > 0 ) {
 2361     InstructionMark im(this);
 2362     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2363     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2364     emit_int8((unsigned char)0xAE);
 2365     emit_operand(as_Register(2), src);
 2366   } else {
 2367     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2368     InstructionMark im(this);
 2369     prefix(src);
 2370     emit_int16(0x0F, (unsigned char)0xAE);
 2371     emit_operand(as_Register(2), src);
 2372   }
 2373 }
 2374 
 2375 void Assembler::leal(Register dst, Address src) {
 2376   InstructionMark im(this);
 2377 #ifdef _LP64
 2378   emit_int8(0x67); // addr32
 2379   prefix(src, dst);
 2380 #endif // LP64
 2381   emit_int8((unsigned char)0x8D);
 2382   emit_operand(dst, src);
 2383 }
 2384 
 2385 void Assembler::lfence() {
 2386   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2387 }
 2388 
 2389 void Assembler::lock() {
 2390   emit_int8((unsigned char)0xF0);
 2391 }
 2392 
 2393 void Assembler::size_prefix() {
 2394   emit_int8(0x66);
 2395 }
 2396 
 2397 void Assembler::lzcntl(Register dst, Register src) {
 2398   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2399   emit_int8((unsigned char)0xF3);
 2400   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2401   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2402 }
 2403 
 2404 // Emit mfence instruction
 2405 void Assembler::mfence() {
 2406   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2407   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2408 }
 2409 
 2410 // Emit sfence instruction
 2411 void Assembler::sfence() {
 2412   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2413   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2414 }
 2415 
 2416 void Assembler::mov(Register dst, Register src) {
 2417   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2418 }
 2419 
 2420 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2422   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2423   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2424   attributes.set_rex_vex_w_reverted();
 2425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2426   emit_int16(0x28, (0xC0 | encode));
 2427 }
 2428 
 2429 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2430   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2431   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2432   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2433   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2434   emit_int16(0x28, (0xC0 | encode));
 2435 }
 2436 
 2437 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2438   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2440   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2441   emit_int16(0x16, (0xC0 | encode));
 2442 }
 2443 
 2444 void Assembler::movb(Register dst, Address src) {
 2445   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2446   InstructionMark im(this);
 2447   prefix(src, dst, true);
 2448   emit_int8((unsigned char)0x8A);
 2449   emit_operand(dst, src);
 2450 }
 2451 
 2452 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2453   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2454   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2455   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2456   attributes.set_rex_vex_w_reverted();
 2457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2458   emit_int16(0x12, 0xC0 | encode);
 2459 }
 2460 
 2461 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2462   assert(VM_Version::supports_avx512dq(), "");
 2463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2465   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2466 }
 2467 
 2468 void Assembler::kmovbl(KRegister dst, Register src) {
 2469   assert(VM_Version::supports_avx512dq(), "");
 2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2472   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2473 }
 2474 
 2475 void Assembler::kmovbl(Register dst, KRegister src) {
 2476   assert(VM_Version::supports_avx512dq(), "");
 2477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2478   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2479   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2480 }
 2481 
 2482 void Assembler::kmovwl(KRegister dst, Register src) {
 2483   assert(VM_Version::supports_evex(), "");
 2484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2485   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2486   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2487 }
 2488 
 2489 void Assembler::kmovwl(Register dst, KRegister src) {
 2490   assert(VM_Version::supports_evex(), "");
 2491   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2492   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2493   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2494 }
 2495 
 2496 void Assembler::kmovwl(KRegister dst, Address src) {
 2497   assert(VM_Version::supports_evex(), "");
 2498   InstructionMark im(this);
 2499   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2500   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2501   emit_int8((unsigned char)0x90);
 2502   emit_operand((Register)dst, src);
 2503 }
 2504 
 2505 void Assembler::kmovwl(Address dst, KRegister src) {
 2506   assert(VM_Version::supports_evex(), "");
 2507   InstructionMark im(this);
 2508   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2509   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2510   emit_int8((unsigned char)0x91);
 2511   emit_operand((Register)src, dst);
 2512 }
 2513 
 2514 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2515   assert(VM_Version::supports_evex(), "");
 2516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2517   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2518   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2519 }
 2520 
 2521 void Assembler::kmovdl(KRegister dst, Register src) {
 2522   assert(VM_Version::supports_avx512bw(), "");
 2523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2524   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2525   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2526 }
 2527 
 2528 void Assembler::kmovdl(Register dst, KRegister src) {
 2529   assert(VM_Version::supports_avx512bw(), "");
 2530   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2531   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2532   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2533 }
 2534 
 2535 void Assembler::kmovql(KRegister dst, KRegister src) {
 2536   assert(VM_Version::supports_avx512bw(), "");
 2537   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2538   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2539   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2540 }
 2541 
 2542 void Assembler::kmovql(KRegister dst, Address src) {
 2543   assert(VM_Version::supports_avx512bw(), "");
 2544   InstructionMark im(this);
 2545   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2546   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2547   emit_int8((unsigned char)0x90);
 2548   emit_operand((Register)dst, src);
 2549 }
 2550 
 2551 void Assembler::kmovql(Address dst, KRegister src) {
 2552   assert(VM_Version::supports_avx512bw(), "");
 2553   InstructionMark im(this);
 2554   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2555   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2556   emit_int8((unsigned char)0x91);
 2557   emit_operand((Register)src, dst);
 2558 }
 2559 
 2560 void Assembler::kmovql(KRegister dst, Register src) {
 2561   assert(VM_Version::supports_avx512bw(), "");
 2562   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2563   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2564   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2565 }
 2566 
 2567 void Assembler::kmovql(Register dst, KRegister src) {
 2568   assert(VM_Version::supports_avx512bw(), "");
 2569   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2570   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2571   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2572 }
 2573 
 2574 void Assembler::knotwl(KRegister dst, KRegister src) {
 2575   assert(VM_Version::supports_evex(), "");
 2576   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2578   emit_int16(0x44, (0xC0 | encode));
 2579 }
 2580 
 2581 void Assembler::knotbl(KRegister dst, KRegister src) {
 2582   assert(VM_Version::supports_evex(), "");
 2583   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2584   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2585   emit_int16(0x44, (0xC0 | encode));
 2586 }
 2587 
 2588 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2589   assert(VM_Version::supports_avx512dq(), "");
 2590   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2591   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2592   emit_int16(0x45, (0xC0 | encode));
 2593 }
 2594 
 2595 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2596   assert(VM_Version::supports_evex(), "");
 2597   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2598   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2599   emit_int16(0x45, (0xC0 | encode));
 2600 }
 2601 
 2602 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2603   assert(VM_Version::supports_avx512bw(), "");
 2604   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2605   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2606   emit_int16(0x45, (0xC0 | encode));
 2607 }
 2608 
 2609 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2610   assert(VM_Version::supports_avx512bw(), "");
 2611   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2612   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2613   emit_int16(0x45, (0xC0 | encode));
 2614 }
 2615 
 2616 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2617   assert(VM_Version::supports_avx512dq(), "");
 2618   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2619   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2620   emit_int16(0x47, (0xC0 | encode));
 2621 }
 2622 
 2623 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2624   assert(VM_Version::supports_evex(), "");
 2625   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2626   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2627   emit_int16(0x47, (0xC0 | encode));
 2628 }
 2629 
 2630 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2631   assert(VM_Version::supports_avx512bw(), "");
 2632   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2633   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2634   emit_int16(0x47, (0xC0 | encode));
 2635 }
 2636 
 2637 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2638   assert(VM_Version::supports_avx512bw(), "");
 2639   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2640   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2641   emit_int16(0x47, (0xC0 | encode));
 2642 }
 2643 
 2644 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2645   assert(VM_Version::supports_avx512dq(), "");
 2646   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2647   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2648   emit_int16(0x41, (0xC0 | encode));
 2649 }
 2650 
 2651 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2652   assert(VM_Version::supports_evex(), "");
 2653   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2654   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2655   emit_int16(0x41, (0xC0 | encode));
 2656 }
 2657 
 2658 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2659   assert(VM_Version::supports_avx512bw(), "");
 2660   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2661   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2662   emit_int16(0x41, (0xC0 | encode));
 2663 }
 2664 
 2665 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2666   assert(VM_Version::supports_avx512bw(), "");
 2667   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2668   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2669   emit_int16(0x41, (0xC0 | encode));
 2670 }
 2671 
 2672 void Assembler::knotdl(KRegister dst, KRegister src) {
 2673   assert(VM_Version::supports_avx512bw(), "");
 2674   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2675   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2676   emit_int16(0x44, (0xC0 | encode));
 2677 }
 2678 
 2679 void Assembler::knotql(KRegister dst, KRegister src) {
 2680   assert(VM_Version::supports_avx512bw(), "");
 2681   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2682   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2683   emit_int16(0x44, (0xC0 | encode));
 2684 }
 2685 
 2686 // This instruction produces ZF or CF flags
 2687 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2688   assert(VM_Version::supports_avx512dq(), "");
 2689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2690   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2691   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2692 }
 2693 
 2694 // This instruction produces ZF or CF flags
 2695 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2696   assert(VM_Version::supports_evex(), "");
 2697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2698   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2699   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2700 }
 2701 
 2702 // This instruction produces ZF or CF flags
 2703 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2704   assert(VM_Version::supports_avx512bw(), "");
 2705   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2706   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2707   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2708 }
 2709 
 2710 // This instruction produces ZF or CF flags
 2711 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2712   assert(VM_Version::supports_avx512bw(), "");
 2713   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2714   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2715   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2716 }
 2717 
 2718 // This instruction produces ZF or CF flags
 2719 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2720   assert(VM_Version::supports_avx512bw(), "");
 2721   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2722   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2723   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2724 }
 2725 
 2726 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2727   assert(VM_Version::supports_avx512bw(), "");
 2728   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2729   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2730   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2731 }
 2732 
 2733 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2734   assert(VM_Version::supports_avx512dq(), "");
 2735   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2736   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2737   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2738 }
 2739 
 2740 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2741   assert(VM_Version::supports_avx512dq(), "");
 2742   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2743   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2744   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2745 }
 2746 
 2747 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2748   assert(VM_Version::supports_avx512bw(), "");
 2749   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2750   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2751   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2752 }
 2753 
 2754 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2755   assert(VM_Version::supports_avx512bw(), "");
 2756   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2757   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2758   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2759 }
 2760 
 2761 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2762   assert(VM_Version::supports_avx512dq(), "");
 2763   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2764   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2765   emit_int16(0x46, (0xC0 | encode));
 2766 }
 2767 
 2768 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2769   assert(VM_Version::supports_avx512dq(), "");
 2770   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2771   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2772   emit_int16(0x32, (0xC0 | encode));
 2773   emit_int8(imm8);
 2774 }
 2775 
 2776 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 2777   assert(VM_Version::supports_avx512dq(), "");
 2778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2779   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2780   emit_int16(0x30, (0xC0 | encode));
 2781 }
 2782 
 2783 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 2784   assert(VM_Version::supports_evex(), "");
 2785   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2786   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2787   emit_int16(0x30, (0xC0 | encode));
 2788   emit_int8(imm8);
 2789 }
 2790 
 2791 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 2792   assert(VM_Version::supports_avx512bw(), "");
 2793   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2794   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2795   emit_int16(0x31, (0xC0 | encode));
 2796   emit_int8(imm8);
 2797 }
 2798 
 2799 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 2800   assert(VM_Version::supports_avx512bw(), "");
 2801   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2802   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2803   emit_int16(0x31, (0xC0 | encode));
 2804   emit_int8(imm8);
 2805 }
 2806 
 2807 void Assembler::movb(Address dst, int imm8) {
 2808   InstructionMark im(this);
 2809    prefix(dst);
 2810   emit_int8((unsigned char)0xC6);
 2811   emit_operand(rax, dst, 1);
 2812   emit_int8(imm8);
 2813 }
 2814 
 2815 
 2816 void Assembler::movb(Address dst, Register src) {
 2817   assert(src->has_byte_register(), "must have byte register");
 2818   InstructionMark im(this);
 2819   prefix(dst, src, true);
 2820   emit_int8((unsigned char)0x88);
 2821   emit_operand(src, dst);
 2822 }
 2823 
 2824 void Assembler::movdl(XMMRegister dst, Register src) {
 2825   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2826   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2827   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2828   emit_int16(0x6E, (0xC0 | encode));
 2829 }
 2830 
 2831 void Assembler::movdl(Register dst, XMMRegister src) {
 2832   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2833   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2834   // swap src/dst to get correct prefix
 2835   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2836   emit_int16(0x7E, (0xC0 | encode));
 2837 }
 2838 
 2839 void Assembler::movdl(XMMRegister dst, Address src) {
 2840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2841   InstructionMark im(this);
 2842   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2843   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2844   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2845   emit_int8(0x6E);
 2846   emit_operand(dst, src);
 2847 }
 2848 
 2849 void Assembler::movdl(Address dst, XMMRegister src) {
 2850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2851   InstructionMark im(this);
 2852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2853   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2854   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2855   emit_int8(0x7E);
 2856   emit_operand(src, dst);
 2857 }
 2858 
 2859 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2861   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2862   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2863   emit_int16(0x6F, (0xC0 | encode));
 2864 }
 2865 
 2866 void Assembler::movdqa(XMMRegister dst, Address src) {
 2867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2868   InstructionMark im(this);
 2869   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2870   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2871   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2872   emit_int8(0x6F);
 2873   emit_operand(dst, src);
 2874 }
 2875 
 2876 void Assembler::movdqu(XMMRegister dst, Address src) {
 2877   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2878   InstructionMark im(this);
 2879   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2880   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2881   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2882   emit_int8(0x6F);
 2883   emit_operand(dst, src);
 2884 }
 2885 
 2886 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2887   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2888   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2889   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2890   emit_int16(0x6F, (0xC0 | encode));
 2891 }
 2892 
 2893 void Assembler::movdqu(Address dst, XMMRegister src) {
 2894   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2895   InstructionMark im(this);
 2896   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2897   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2898   attributes.reset_is_clear_context();
 2899   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2900   emit_int8(0x7F);
 2901   emit_operand(src, dst);
 2902 }
 2903 
 2904 // Move Unaligned 256bit Vector
 2905 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2906   assert(UseAVX > 0, "");
 2907   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2908   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2909   emit_int16(0x6F, (0xC0 | encode));
 2910 }
 2911 
 2912 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 2913   assert(UseAVX > 0, "");
 2914   InstructionMark im(this);
 2915   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2916   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2917   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2918   emit_int8(0x6F);
 2919   emit_operand(dst, src);
 2920 }
 2921 
 2922 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 2923   assert(UseAVX > 0, "");
 2924   InstructionMark im(this);
 2925   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2926   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2927   attributes.reset_is_clear_context();
 2928   // swap src<->dst for encoding
 2929   assert(src != xnoreg, "sanity");
 2930   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2931   emit_int8(0x7F);
 2932   emit_operand(src, dst);
 2933 }
 2934 

























 2935 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 2936 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 2937   assert(VM_Version::supports_evex(), "");
 2938   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2939   attributes.set_is_evex_instruction();
 2940   if (merge) {
 2941     attributes.reset_is_clear_context();
 2942   }
 2943   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2944   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2945   emit_int16(0x6F, (0xC0 | encode));
 2946 }
 2947 
 2948 void Assembler::evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) {
 2949   assert(VM_Version::supports_evex(), "");
 2950   InstructionMark im(this);
 2951   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2952   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2953   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2954   attributes.set_is_evex_instruction();
 2955   if (merge) {
 2956     attributes.reset_is_clear_context();
 2957   }
 2958   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2959   emit_int8(0x6F);
 2960   emit_operand(dst, src);
 2961 }
 2962 
 2963 void Assembler::evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) {
 2964   assert(VM_Version::supports_evex(), "");
 2965   assert(src != xnoreg, "sanity");
 2966   InstructionMark im(this);
 2967   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2968   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2969   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2970   attributes.set_is_evex_instruction();
 2971   if (merge) {
 2972     attributes.reset_is_clear_context();
 2973   }
 2974   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2975   emit_int8(0x7F);
 2976   emit_operand(src, dst);
 2977 }
 2978 
 2979 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2980   assert(VM_Version::supports_avx512vlbw(), "");
 2981   InstructionMark im(this);
 2982   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2983   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2984   attributes.set_embedded_opmask_register_specifier(mask);
 2985   attributes.set_is_evex_instruction();
 2986   if (merge) {
 2987     attributes.reset_is_clear_context();
 2988   }
 2989   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2990   emit_int8(0x6F);
 2991   emit_operand(dst, src);
 2992 }
 2993 
 2994 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2995   assert(VM_Version::supports_avx512vlbw(), "");
 2996   assert(src != xnoreg, "sanity");
 2997   InstructionMark im(this);
 2998   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2999   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3000   attributes.set_embedded_opmask_register_specifier(mask);
 3001   attributes.set_is_evex_instruction();
 3002   if (merge) {
 3003     attributes.reset_is_clear_context();
 3004   }
 3005   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3006   emit_int8(0x7F);
 3007   emit_operand(src, dst);
 3008 }
 3009 
 3010 void Assembler::evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) {
 3011   assert(VM_Version::supports_evex(), "");
 3012   InstructionMark im(this);
 3013   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3014   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3015   attributes.set_is_evex_instruction();
 3016   if (merge) {
 3017     attributes.reset_is_clear_context();
 3018   }
 3019   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3020   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3021   emit_int8(0x6F);
 3022   emit_operand(dst, src);
 3023 }
 3024 
 3025 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3026   assert(VM_Version::supports_avx512vlbw(), "");
 3027   InstructionMark im(this);
 3028   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3029   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3030   attributes.set_embedded_opmask_register_specifier(mask);
 3031   attributes.set_is_evex_instruction();
 3032   if (merge) {
 3033     attributes.reset_is_clear_context();
 3034   }
 3035   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3036   emit_int8(0x6F);
 3037   emit_operand(dst, src);
 3038 }
 3039 
 3040 void Assembler::evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) {
 3041   assert(VM_Version::supports_evex(), "");
 3042   assert(src != xnoreg, "sanity");
 3043   InstructionMark im(this);
 3044   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3045   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3046   attributes.set_is_evex_instruction();
 3047   if (merge) {
 3048     attributes.reset_is_clear_context();
 3049   }
 3050   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 3051   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 3052   emit_int8(0x7F);
 3053   emit_operand(src, dst);
 3054 }
 3055 
 3056 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3057   assert(VM_Version::supports_avx512vlbw(), "");
 3058   assert(src != xnoreg, "sanity");
 3059   InstructionMark im(this);
 3060   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3061   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3062   attributes.set_embedded_opmask_register_specifier(mask);
 3063   attributes.set_is_evex_instruction();
 3064   if (merge) {
 3065     attributes.reset_is_clear_context();
 3066   }
 3067   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3068   emit_int8(0x7F);
 3069   emit_operand(src, dst);
 3070 }
 3071 
 3072 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3073   // Unmasked instruction
 3074   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3075 }
 3076 
 3077 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3078   assert(VM_Version::supports_evex(), "");
 3079   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3080   attributes.set_embedded_opmask_register_specifier(mask);
 3081   attributes.set_is_evex_instruction();
 3082   if (merge) {
 3083     attributes.reset_is_clear_context();
 3084   }
 3085   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3086   emit_int16(0x6F, (0xC0 | encode));
 3087 }
 3088 
 3089 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3090   // Unmasked instruction
 3091   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3092 }
 3093 
 3094 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3095   assert(VM_Version::supports_evex(), "");
 3096   InstructionMark im(this);
 3097   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3098   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3099   attributes.set_embedded_opmask_register_specifier(mask);
 3100   attributes.set_is_evex_instruction();
 3101   if (merge) {
 3102     attributes.reset_is_clear_context();
 3103   }
 3104   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3105   emit_int8(0x6F);
 3106   emit_operand(dst, src);
 3107 }
 3108 
 3109 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3110   // Unmasked isntruction
 3111   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3112 }
 3113 
 3114 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3115   assert(VM_Version::supports_evex(), "");
 3116   assert(src != xnoreg, "sanity");
 3117   InstructionMark im(this);
 3118   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3119   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3120   attributes.set_embedded_opmask_register_specifier(mask);
 3121   attributes.set_is_evex_instruction();
 3122   if (merge) {
 3123     attributes.reset_is_clear_context();
 3124   }
 3125   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3126   emit_int8(0x7F);
 3127   emit_operand(src, dst);
 3128 }
 3129 
 3130 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3131   // Unmasked instruction
 3132   if (dst->encoding() == src->encoding()) return;
 3133   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3134 }
 3135 
 3136 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3137   assert(VM_Version::supports_evex(), "");
 3138   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3139   attributes.set_embedded_opmask_register_specifier(mask);
 3140   attributes.set_is_evex_instruction();
 3141   if (merge) {
 3142     attributes.reset_is_clear_context();
 3143   }
 3144   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3145   emit_int16(0x6F, (0xC0 | encode));
 3146 }
 3147 
 3148 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3149   // Unmasked instruction
 3150   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3151 }
 3152 
 3153 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3154   assert(VM_Version::supports_evex(), "");
 3155   InstructionMark im(this);
 3156   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3157   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3158   attributes.set_embedded_opmask_register_specifier(mask);
 3159   attributes.set_is_evex_instruction();
 3160   if (merge) {
 3161     attributes.reset_is_clear_context();
 3162   }
 3163   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3164   emit_int8(0x6F);
 3165   emit_operand(dst, src);
 3166 }
 3167 
 3168 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3169   // Unmasked instruction
 3170   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3171 }
 3172 
 3173 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3174   assert(VM_Version::supports_evex(), "");
 3175   assert(src != xnoreg, "sanity");
 3176   InstructionMark im(this);
 3177   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3178   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3179   attributes.set_embedded_opmask_register_specifier(mask);
 3180   if (merge) {
 3181     attributes.reset_is_clear_context();
 3182   }
 3183   attributes.set_is_evex_instruction();
 3184   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3185   emit_int8(0x7F);
 3186   emit_operand(src, dst);
 3187 }
 3188 





































































































































 3189 // Uses zero extension on 64bit
 3190 
 3191 void Assembler::movl(Register dst, int32_t imm32) {
 3192   int encode = prefix_and_encode(dst->encoding());
 3193   emit_int8(0xB8 | encode);
 3194   emit_int32(imm32);
 3195 }
 3196 
 3197 void Assembler::movl(Register dst, Register src) {
 3198   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3199   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3200 }
 3201 
 3202 void Assembler::movl(Register dst, Address src) {
 3203   InstructionMark im(this);
 3204   prefix(src, dst);
 3205   emit_int8((unsigned char)0x8B);
 3206   emit_operand(dst, src);
 3207 }
 3208 
 3209 void Assembler::movl(Address dst, int32_t imm32) {
 3210   InstructionMark im(this);
 3211   prefix(dst);
 3212   emit_int8((unsigned char)0xC7);
 3213   emit_operand(rax, dst, 4);
 3214   emit_int32(imm32);
 3215 }
 3216 
 3217 void Assembler::movl(Address dst, Register src) {
 3218   InstructionMark im(this);
 3219   prefix(dst, src);
 3220   emit_int8((unsigned char)0x89);
 3221   emit_operand(src, dst);
 3222 }
 3223 
 3224 // New cpus require to use movsd and movss to avoid partial register stall
 3225 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3226 // The selection is done in MacroAssembler::movdbl() and movflt().
 3227 void Assembler::movlpd(XMMRegister dst, Address src) {
 3228   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3229   InstructionMark im(this);
 3230   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3231   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3232   attributes.set_rex_vex_w_reverted();
 3233   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3234   emit_int8(0x12);
 3235   emit_operand(dst, src);
 3236 }
 3237 

















































































 3238 void Assembler::movq(XMMRegister dst, Address src) {
 3239   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3240   InstructionMark im(this);
 3241   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3242   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3243   attributes.set_rex_vex_w_reverted();
 3244   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3245   emit_int8(0x7E);
 3246   emit_operand(dst, src);
 3247 }
 3248 
 3249 void Assembler::movq(Address dst, XMMRegister src) {
 3250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3251   InstructionMark im(this);
 3252   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3253   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3254   attributes.set_rex_vex_w_reverted();
 3255   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3256   emit_int8((unsigned char)0xD6);
 3257   emit_operand(src, dst);
 3258 }
 3259 
 3260 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3261   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3262   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3263   attributes.set_rex_vex_w_reverted();
 3264   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3265   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3266 }
 3267 
 3268 void Assembler::movq(Register dst, XMMRegister src) {
 3269   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3270   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3271   // swap src/dst to get correct prefix
 3272   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3273   emit_int16(0x7E, (0xC0 | encode));
 3274 }
 3275 
 3276 void Assembler::movq(XMMRegister dst, Register src) {
 3277   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3278   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3279   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3280   emit_int16(0x6E, (0xC0 | encode));
 3281 }
 3282 
 3283 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3284   InstructionMark im(this);
 3285   prefix(src, dst);
 3286   emit_int16(0x0F, (unsigned char)0xBE);
 3287   emit_operand(dst, src);
 3288 }
 3289 
 3290 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3291   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3292   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3293   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3294 }
 3295 
 3296 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3297   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3298   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3299   attributes.set_rex_vex_w_reverted();
 3300   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3301   emit_int16(0x10, (0xC0 | encode));
 3302 }
 3303 
 3304 void Assembler::movsd(XMMRegister dst, Address src) {
 3305   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3306   InstructionMark im(this);
 3307   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3308   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3309   attributes.set_rex_vex_w_reverted();
 3310   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3311   emit_int8(0x10);
 3312   emit_operand(dst, src);
 3313 }
 3314 
 3315 void Assembler::movsd(Address dst, XMMRegister src) {
 3316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3317   InstructionMark im(this);
 3318   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3319   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3320   attributes.reset_is_clear_context();
 3321   attributes.set_rex_vex_w_reverted();
 3322   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3323   emit_int8(0x11);
 3324   emit_operand(src, dst);
 3325 }
 3326 
 3327 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3328   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3329   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3330   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3331   emit_int16(0x10, (0xC0 | encode));
 3332 }
 3333 
 3334 void Assembler::movss(XMMRegister dst, Address src) {
 3335   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3336   InstructionMark im(this);
 3337   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3338   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3339   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3340   emit_int8(0x10);
 3341   emit_operand(dst, src);
 3342 }
 3343 
 3344 void Assembler::movss(Address dst, XMMRegister src) {
 3345   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3346   InstructionMark im(this);
 3347   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3348   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3349   attributes.reset_is_clear_context();
 3350   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3351   emit_int8(0x11);
 3352   emit_operand(src, dst);
 3353 }
 3354 
 3355 void Assembler::movswl(Register dst, Address src) { // movsxw
 3356   InstructionMark im(this);
 3357   prefix(src, dst);
 3358   emit_int16(0x0F, (unsigned char)0xBF);
 3359   emit_operand(dst, src);
 3360 }
 3361 
 3362 void Assembler::movswl(Register dst, Register src) { // movsxw
 3363   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3364   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3365 }
 3366 
 3367 void Assembler::movw(Address dst, int imm16) {
 3368   InstructionMark im(this);
 3369 
 3370   emit_int8(0x66); // switch to 16-bit mode
 3371   prefix(dst);
 3372   emit_int8((unsigned char)0xC7);
 3373   emit_operand(rax, dst, 2);
 3374   emit_int16(imm16);
 3375 }
 3376 
 3377 void Assembler::movw(Register dst, Address src) {
 3378   InstructionMark im(this);
 3379   emit_int8(0x66);
 3380   prefix(src, dst);
 3381   emit_int8((unsigned char)0x8B);
 3382   emit_operand(dst, src);
 3383 }
 3384 
 3385 void Assembler::movw(Address dst, Register src) {
 3386   InstructionMark im(this);
 3387   emit_int8(0x66);
 3388   prefix(dst, src);
 3389   emit_int8((unsigned char)0x89);
 3390   emit_operand(src, dst);
 3391 }
 3392 
 3393 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3394   InstructionMark im(this);
 3395   prefix(src, dst);
 3396   emit_int16(0x0F, (unsigned char)0xB6);
 3397   emit_operand(dst, src);
 3398 }
 3399 
 3400 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3401   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3402   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3403   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3404 }
 3405 
 3406 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3407   InstructionMark im(this);
 3408   prefix(src, dst);
 3409   emit_int16(0x0F, (unsigned char)0xB7);
 3410   emit_operand(dst, src);
 3411 }
 3412 
 3413 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3414   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3415   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3416 }
 3417 
 3418 void Assembler::mull(Address src) {
 3419   InstructionMark im(this);
 3420   prefix(src);
 3421   emit_int8((unsigned char)0xF7);
 3422   emit_operand(rsp, src);
 3423 }
 3424 
 3425 void Assembler::mull(Register src) {
 3426   int encode = prefix_and_encode(src->encoding());
 3427   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3428 }
 3429 
 3430 void Assembler::mulsd(XMMRegister dst, Address src) {
 3431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3432   InstructionMark im(this);
 3433   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3434   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3435   attributes.set_rex_vex_w_reverted();
 3436   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3437   emit_int8(0x59);
 3438   emit_operand(dst, src);
 3439 }
 3440 
 3441 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3442   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3443   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3444   attributes.set_rex_vex_w_reverted();
 3445   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3446   emit_int16(0x59, (0xC0 | encode));
 3447 }
 3448 
 3449 void Assembler::mulss(XMMRegister dst, Address src) {
 3450   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3451   InstructionMark im(this);
 3452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3453   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3454   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3455   emit_int8(0x59);
 3456   emit_operand(dst, src);
 3457 }
 3458 
 3459 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3460   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3462   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3463   emit_int16(0x59, (0xC0 | encode));
 3464 }
 3465 
 3466 void Assembler::negl(Register dst) {
 3467   int encode = prefix_and_encode(dst->encoding());
 3468   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3469 }
 3470 
 3471 void Assembler::negl(Address dst) {
 3472   InstructionMark im(this);
 3473   prefix(dst);
 3474   emit_int8((unsigned char)0xF7);
 3475   emit_operand(as_Register(3), dst);
 3476 }
 3477 
 3478 void Assembler::nop(int i) {
 3479 #ifdef ASSERT
 3480   assert(i > 0, " ");
 3481   // The fancy nops aren't currently recognized by debuggers making it a
 3482   // pain to disassemble code while debugging. If asserts are on clearly
 3483   // speed is not an issue so simply use the single byte traditional nop
 3484   // to do alignment.
 3485 
 3486   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3487   return;
 3488 
 3489 #endif // ASSERT
 3490 
 3491   if (UseAddressNop && VM_Version::is_intel()) {
 3492     //
 3493     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3494     //  1: 0x90
 3495     //  2: 0x66 0x90
 3496     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3497     //  4: 0x0F 0x1F 0x40 0x00
 3498     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3499     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3500     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3501     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3502     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3503     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3504     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3505 
 3506     // The rest coding is Intel specific - don't use consecutive address nops
 3507 
 3508     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3509     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3510     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3511     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3512 
 3513     while(i >= 15) {
 3514       // For Intel don't generate consecutive addess nops (mix with regular nops)
 3515       i -= 15;
 3516       emit_int24(0x66, 0x66, 0x66);
 3517       addr_nop_8();
 3518       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3519     }
 3520     switch (i) {
 3521       case 14:
 3522         emit_int8(0x66); // size prefix
 3523       case 13:
 3524         emit_int8(0x66); // size prefix
 3525       case 12:
 3526         addr_nop_8();
 3527         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3528         break;
 3529       case 11:
 3530         emit_int8(0x66); // size prefix
 3531       case 10:
 3532         emit_int8(0x66); // size prefix
 3533       case 9:
 3534         emit_int8(0x66); // size prefix
 3535       case 8:
 3536         addr_nop_8();
 3537         break;
 3538       case 7:
 3539         addr_nop_7();
 3540         break;
 3541       case 6:
 3542         emit_int8(0x66); // size prefix
 3543       case 5:
 3544         addr_nop_5();
 3545         break;
 3546       case 4:
 3547         addr_nop_4();
 3548         break;
 3549       case 3:
 3550         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3551         emit_int8(0x66); // size prefix
 3552       case 2:
 3553         emit_int8(0x66); // size prefix
 3554       case 1:
 3555         emit_int8((unsigned char)0x90);
 3556                          // nop
 3557         break;
 3558       default:
 3559         assert(i == 0, " ");
 3560     }
 3561     return;
 3562   }
 3563   if (UseAddressNop && VM_Version::is_amd_family()) {
 3564     //
 3565     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3566     //  1: 0x90
 3567     //  2: 0x66 0x90
 3568     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3569     //  4: 0x0F 0x1F 0x40 0x00
 3570     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3571     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3572     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3573     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3574     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3575     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3576     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3577 
 3578     // The rest coding is AMD specific - use consecutive address nops
 3579 
 3580     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3581     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3582     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3583     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3584     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3585     //     Size prefixes (0x66) are added for larger sizes
 3586 
 3587     while(i >= 22) {
 3588       i -= 11;
 3589       emit_int24(0x66, 0x66, 0x66);
 3590       addr_nop_8();
 3591     }
 3592     // Generate first nop for size between 21-12
 3593     switch (i) {
 3594       case 21:
 3595         i -= 1;
 3596         emit_int8(0x66); // size prefix
 3597       case 20:
 3598       case 19:
 3599         i -= 1;
 3600         emit_int8(0x66); // size prefix
 3601       case 18:
 3602       case 17:
 3603         i -= 1;
 3604         emit_int8(0x66); // size prefix
 3605       case 16:
 3606       case 15:
 3607         i -= 8;
 3608         addr_nop_8();
 3609         break;
 3610       case 14:
 3611       case 13:
 3612         i -= 7;
 3613         addr_nop_7();
 3614         break;
 3615       case 12:
 3616         i -= 6;
 3617         emit_int8(0x66); // size prefix
 3618         addr_nop_5();
 3619         break;
 3620       default:
 3621         assert(i < 12, " ");
 3622     }
 3623 
 3624     // Generate second nop for size between 11-1
 3625     switch (i) {
 3626       case 11:
 3627         emit_int8(0x66); // size prefix
 3628       case 10:
 3629         emit_int8(0x66); // size prefix
 3630       case 9:
 3631         emit_int8(0x66); // size prefix
 3632       case 8:
 3633         addr_nop_8();
 3634         break;
 3635       case 7:
 3636         addr_nop_7();
 3637         break;
 3638       case 6:
 3639         emit_int8(0x66); // size prefix
 3640       case 5:
 3641         addr_nop_5();
 3642         break;
 3643       case 4:
 3644         addr_nop_4();
 3645         break;
 3646       case 3:
 3647         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3648         emit_int8(0x66); // size prefix
 3649       case 2:
 3650         emit_int8(0x66); // size prefix
 3651       case 1:
 3652         emit_int8((unsigned char)0x90);
 3653                          // nop
 3654         break;
 3655       default:
 3656         assert(i == 0, " ");
 3657     }
 3658     return;
 3659   }
 3660 
 3661   if (UseAddressNop && VM_Version::is_zx()) {
 3662     //
 3663     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3664     //  1: 0x90
 3665     //  2: 0x66 0x90
 3666     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3667     //  4: 0x0F 0x1F 0x40 0x00
 3668     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3669     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3670     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3671     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3672     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3673     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3674     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3675 
 3676     // The rest coding is ZX specific - don't use consecutive address nops
 3677 
 3678     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3679     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3680     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3681     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3682 
 3683     while (i >= 15) {
 3684       // For ZX don't generate consecutive addess nops (mix with regular nops)
 3685       i -= 15;
 3686       emit_int24(0x66, 0x66, 0x66);
 3687       addr_nop_8();
 3688       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3689     }
 3690     switch (i) {
 3691       case 14:
 3692         emit_int8(0x66); // size prefix
 3693       case 13:
 3694         emit_int8(0x66); // size prefix
 3695       case 12:
 3696         addr_nop_8();
 3697         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3698         break;
 3699       case 11:
 3700         emit_int8(0x66); // size prefix
 3701       case 10:
 3702         emit_int8(0x66); // size prefix
 3703       case 9:
 3704         emit_int8(0x66); // size prefix
 3705       case 8:
 3706         addr_nop_8();
 3707         break;
 3708       case 7:
 3709         addr_nop_7();
 3710         break;
 3711       case 6:
 3712         emit_int8(0x66); // size prefix
 3713       case 5:
 3714         addr_nop_5();
 3715         break;
 3716       case 4:
 3717         addr_nop_4();
 3718         break;
 3719       case 3:
 3720         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3721         emit_int8(0x66); // size prefix
 3722       case 2:
 3723         emit_int8(0x66); // size prefix
 3724       case 1:
 3725         emit_int8((unsigned char)0x90);
 3726                          // nop
 3727         break;
 3728       default:
 3729         assert(i == 0, " ");
 3730     }
 3731     return;
 3732   }
 3733 
 3734   // Using nops with size prefixes "0x66 0x90".
 3735   // From AMD Optimization Guide:
 3736   //  1: 0x90
 3737   //  2: 0x66 0x90
 3738   //  3: 0x66 0x66 0x90
 3739   //  4: 0x66 0x66 0x66 0x90
 3740   //  5: 0x66 0x66 0x90 0x66 0x90
 3741   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3742   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3743   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3744   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3745   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3746   //
 3747   while (i > 12) {
 3748     i -= 4;
 3749     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3750   }
 3751   // 1 - 12 nops
 3752   if (i > 8) {
 3753     if (i > 9) {
 3754       i -= 1;
 3755       emit_int8(0x66);
 3756     }
 3757     i -= 3;
 3758     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3759   }
 3760   // 1 - 8 nops
 3761   if (i > 4) {
 3762     if (i > 6) {
 3763       i -= 1;
 3764       emit_int8(0x66);
 3765     }
 3766     i -= 3;
 3767     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3768   }
 3769   switch (i) {
 3770     case 4:
 3771       emit_int8(0x66);
 3772     case 3:
 3773       emit_int8(0x66);
 3774     case 2:
 3775       emit_int8(0x66);
 3776     case 1:
 3777       emit_int8((unsigned char)0x90);
 3778       break;
 3779     default:
 3780       assert(i == 0, " ");
 3781   }
 3782 }
 3783 
 3784 void Assembler::notl(Register dst) {
 3785   int encode = prefix_and_encode(dst->encoding());
 3786   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 3787 }
 3788 
 3789 void Assembler::orw(Register dst, Register src) {
 3790   (void)prefix_and_encode(dst->encoding(), src->encoding());
 3791   emit_arith(0x0B, 0xC0, dst, src);
 3792 }
 3793 
 3794 void Assembler::orl(Address dst, int32_t imm32) {
 3795   InstructionMark im(this);
 3796   prefix(dst);
 3797   emit_arith_operand(0x81, rcx, dst, imm32);
 3798 }
 3799 
 3800 void Assembler::orl(Register dst, int32_t imm32) {
 3801   prefix(dst);
 3802   emit_arith(0x81, 0xC8, dst, imm32);
 3803 }
 3804 
 3805 void Assembler::orl(Register dst, Address src) {
 3806   InstructionMark im(this);
 3807   prefix(src, dst);
 3808   emit_int8(0x0B);
 3809   emit_operand(dst, src);
 3810 }
 3811 
 3812 void Assembler::orl(Register dst, Register src) {
 3813   (void) prefix_and_encode(dst->encoding(), src->encoding());
 3814   emit_arith(0x0B, 0xC0, dst, src);
 3815 }
 3816 
 3817 void Assembler::orl(Address dst, Register src) {
 3818   InstructionMark im(this);
 3819   prefix(dst, src);
 3820   emit_int8(0x09);
 3821   emit_operand(src, dst);
 3822 }
 3823 
 3824 void Assembler::orb(Address dst, int imm8) {
 3825   InstructionMark im(this);
 3826   prefix(dst);
 3827   emit_int8((unsigned char)0x80);
 3828   emit_operand(rcx, dst, 1);
 3829   emit_int8(imm8);
 3830 }
 3831 
 3832 void Assembler::orb(Address dst, Register src) {
 3833   InstructionMark im(this);
 3834   prefix(dst, src, true);
 3835   emit_int8(0x08);
 3836   emit_operand(src, dst);
 3837 }
 3838 
 3839 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 3840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3841   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3842   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3843   emit_int16(0x63, (0xC0 | encode));
 3844 }
 3845 
 3846 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3847   assert(UseAVX > 0, "some form of AVX must be enabled");
 3848   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3849   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3850   emit_int16(0x63, (0xC0 | encode));
 3851 }
 3852 
 3853 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 3854   assert(VM_Version::supports_sse2(), "");
 3855   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3856   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3857   emit_int16(0x6B, (0xC0 | encode));
 3858 }
 3859 
 3860 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3861   assert(UseAVX > 0, "some form of AVX must be enabled");
 3862   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3863   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3864   emit_int16(0x6B, (0xC0 | encode));
 3865 }
 3866 
 3867 void Assembler::packuswb(XMMRegister dst, Address src) {
 3868   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3869   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 3870   InstructionMark im(this);
 3871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3872   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 3873   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3874   emit_int8(0x67);
 3875   emit_operand(dst, src);
 3876 }
 3877 
 3878 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 3879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3881   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3882   emit_int16(0x67, (0xC0 | encode));
 3883 }
 3884 
 3885 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3886   assert(UseAVX > 0, "some form of AVX must be enabled");
 3887   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3888   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3889   emit_int16(0x67, (0xC0 | encode));
 3890 }
 3891 
 3892 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 3893   assert(VM_Version::supports_sse4_1(), "");
 3894   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3895   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3896   emit_int16(0x2B, (0xC0 | encode));
 3897 }
 3898 
 3899 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3900   assert(UseAVX > 0, "some form of AVX must be enabled");
 3901   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3902   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3903   emit_int16(0x2B, (0xC0 | encode));
 3904 }
 3905 
 3906 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3907   assert(VM_Version::supports_avx2(), "");
 3908   assert(vector_len != AVX_128bit, "");
 3909   // VEX.256.66.0F3A.W1 00 /r ib
 3910   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3911   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3912   emit_int24(0x00, (0xC0 | encode), imm8);
 3913 }
 3914 
 3915 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3916   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 3917          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 3918   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3919   attributes.set_is_evex_instruction();
 3920   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3921   emit_int16(0x36, (0xC0 | encode));
 3922 }
 3923 
 3924 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3925   assert(VM_Version::supports_avx512_vbmi(), "");
 3926   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3927   attributes.set_is_evex_instruction();
 3928   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3929   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 3930 }
 3931 
 3932 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3933   assert(VM_Version::supports_avx512_vbmi(), "");
 3934   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3935   attributes.set_is_evex_instruction();
 3936   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3937   emit_int8((unsigned char)0x8D);
 3938   emit_operand(dst, src);
 3939 }
 3940 
 3941 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3942   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 3943          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 3944          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 3945   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3946   attributes.set_is_evex_instruction();
 3947   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3948   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 3949 }
 3950 
 3951 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3952   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 3953   // VEX.NDS.256.66.0F38.W0 36 /r
 3954   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3955   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3956   emit_int16(0x36, (0xC0 | encode));
 3957 }
 3958 
 3959 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3960   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 3961   // VEX.NDS.256.66.0F38.W0 36 /r
 3962   InstructionMark im(this);
 3963   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3964   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3965   emit_int8(0x36);
 3966   emit_operand(dst, src);
 3967 }
 3968 
 3969 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 3970   assert(VM_Version::supports_avx2(), "");
 3971   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3972   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3973   emit_int24(0x46, (0xC0 | encode), imm8);
 3974 }
 3975 
 3976 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 3977   assert(VM_Version::supports_avx(), "");
 3978   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3979   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3980   emit_int24(0x06, (0xC0 | encode), imm8);
 3981 }
 3982 
 3983 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3984   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 3985   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3986   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3987   emit_int24(0x04, (0xC0 | encode), imm8);
 3988 }
 3989 
 3990 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3991   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 3992   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 3993   attributes.set_rex_vex_w_reverted();
 3994   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3995   emit_int24(0x05, (0xC0 | encode), imm8);
 3996 }
 3997 
 3998 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3999   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4000   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4001   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4002   emit_int24(0x01, (0xC0 | encode), imm8);
 4003 }
 4004 
 4005 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4006   assert(VM_Version::supports_evex(), "");
 4007   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4008   attributes.set_is_evex_instruction();
 4009   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4010   emit_int16(0x76, (0xC0 | encode));
 4011 }
 4012 
 4013 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4014   assert(VM_Version::supports_avx512_vbmi(), "");
 4015   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4016   attributes.set_is_evex_instruction();
 4017   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4018   emit_int16(0x7D, (0xC0 | encode));
 4019 }
 4020 
 4021 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4022   assert(VM_Version::supports_avx512_vbmi(), "");
 4023   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4024   attributes.set_is_evex_instruction();
 4025   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4026   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4027 }
 4028 
 4029 void Assembler::pause() {
 4030   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4031 }
 4032 
 4033 void Assembler::ud2() {
 4034   emit_int16(0x0F, 0x0B);
 4035 }
 4036 
 4037 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4038   assert(VM_Version::supports_sse4_2(), "");
 4039   InstructionMark im(this);
 4040   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4041   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4042   emit_int8(0x61);
 4043   emit_operand(dst, src);
 4044   emit_int8(imm8);
 4045 }
 4046 
 4047 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4048   assert(VM_Version::supports_sse4_2(), "");
 4049   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4050   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4051   emit_int24(0x61, (0xC0 | encode), imm8);
 4052 }
 4053 
 4054 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4055 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4056   assert(VM_Version::supports_sse2(), "");
 4057   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4058   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4059   emit_int16(0x74, (0xC0 | encode));
 4060 }
 4061 
 4062 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4063   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4064   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4065   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4066   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4067   emit_int16(cond_encoding, (0xC0 | encode));
 4068 }
 4069 
 4070 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4071 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4072   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4073   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4074   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4075   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4076   emit_int16(0x74, (0xC0 | encode));
 4077 }
 4078 
 4079 // In this context, kdst is written the mask used to process the equal components
 4080 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4081   assert(VM_Version::supports_avx512bw(), "");
 4082   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4083   attributes.set_is_evex_instruction();
 4084   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4085   emit_int16(0x74, (0xC0 | encode));
 4086 }
 4087 
 4088 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4089   assert(VM_Version::supports_avx512vlbw(), "");
 4090   InstructionMark im(this);
 4091   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4092   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4093   attributes.set_is_evex_instruction();
 4094   int dst_enc = kdst->encoding();
 4095   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4096   emit_int8(0x64);
 4097   emit_operand(as_Register(dst_enc), src);
 4098 }
 4099 
 4100 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4101   assert(VM_Version::supports_avx512vlbw(), "");
 4102   InstructionMark im(this);
 4103   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4104   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4105   attributes.reset_is_clear_context();
 4106   attributes.set_embedded_opmask_register_specifier(mask);
 4107   attributes.set_is_evex_instruction();
 4108   int dst_enc = kdst->encoding();
 4109   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4110   emit_int8(0x64);
 4111   emit_operand(as_Register(dst_enc), src);
 4112 }
 4113 
 4114 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4115   assert(VM_Version::supports_avx512vlbw(), "");
 4116   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4117   attributes.set_is_evex_instruction();
 4118   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4119   emit_int24(0x3E, (0xC0 | encode), vcc);
 4120 }
 4121 
 4122 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4123   assert(VM_Version::supports_avx512vlbw(), "");
 4124   InstructionMark im(this);
 4125   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4126   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4127   attributes.set_is_evex_instruction();
 4128   int dst_enc = kdst->encoding();
 4129   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4130   emit_int8(0x3E);
 4131   emit_operand(as_Register(dst_enc), src);
 4132   emit_int8(vcc);
 4133 }
 4134 
 4135 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4136   assert(VM_Version::supports_avx512bw(), "");
 4137   InstructionMark im(this);
 4138   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4139   attributes.set_is_evex_instruction();
 4140   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4141   int dst_enc = kdst->encoding();
 4142   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4143   emit_int8(0x74);
 4144   emit_operand(as_Register(dst_enc), src);
 4145 }
 4146 
 4147 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4148   assert(VM_Version::supports_avx512vlbw(), "");
 4149   InstructionMark im(this);
 4150   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4151   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4152   attributes.reset_is_clear_context();
 4153   attributes.set_embedded_opmask_register_specifier(mask);
 4154   attributes.set_is_evex_instruction();
 4155   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4156   emit_int8(0x74);
 4157   emit_operand(as_Register(kdst->encoding()), src);
 4158 }
 4159 
 4160 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4161 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4162   assert(VM_Version::supports_sse2(), "");
 4163   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4164   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4165   emit_int16(0x75, (0xC0 | encode));
 4166 }
 4167 
 4168 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4169 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4170   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4171   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4172   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4173   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4174   emit_int16(0x75, (0xC0 | encode));
 4175 }
 4176 
 4177 // In this context, kdst is written the mask used to process the equal components
 4178 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4179   assert(VM_Version::supports_avx512bw(), "");
 4180   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4181   attributes.set_is_evex_instruction();
 4182   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4183   emit_int16(0x75, (0xC0 | encode));
 4184 }
 4185 
 4186 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4187   assert(VM_Version::supports_avx512bw(), "");
 4188   InstructionMark im(this);
 4189   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4190   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4191   attributes.set_is_evex_instruction();
 4192   int dst_enc = kdst->encoding();
 4193   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4194   emit_int8(0x75);
 4195   emit_operand(as_Register(dst_enc), src);
 4196 }
 4197 
 4198 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4199 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4200   assert(VM_Version::supports_sse2(), "");
 4201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4202   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4203   emit_int16(0x76, (0xC0 | encode));
 4204 }
 4205 
 4206 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4207 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4208   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4209   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4210   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4211   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4212   emit_int16(0x76, (0xC0 | encode));
 4213 }
 4214 
 4215 // In this context, kdst is written the mask used to process the equal components
 4216 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4217   assert(VM_Version::supports_evex(), "");
 4218   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4219   attributes.set_is_evex_instruction();
 4220   attributes.reset_is_clear_context();
 4221   attributes.set_embedded_opmask_register_specifier(mask);
 4222   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4223   emit_int16(0x76, (0xC0 | encode));
 4224 }
 4225 
 4226 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4227   assert(VM_Version::supports_evex(), "");
 4228   InstructionMark im(this);
 4229   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4230   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4231   attributes.set_is_evex_instruction();
 4232   attributes.reset_is_clear_context();
 4233   attributes.set_embedded_opmask_register_specifier(mask);
 4234   int dst_enc = kdst->encoding();
 4235   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4236   emit_int8(0x76);
 4237   emit_operand(as_Register(dst_enc), src);
 4238 }
 4239 
 4240 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4241 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4242   assert(VM_Version::supports_sse4_1(), "");
 4243   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4244   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4245   emit_int16(0x29, (0xC0 | encode));
 4246 }
 4247 
 4248 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4249   assert(VM_Version::supports_avx(), "");
 4250   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4251   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4252   emit_int16(cond_encoding, (0xC0 | encode));
 4253 }
 4254 
 4255 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4256 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4257   assert(VM_Version::supports_avx(), "");
 4258   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4260   emit_int16(0x29, (0xC0 | encode));
 4261 }
 4262 
 4263 // In this context, kdst is written the mask used to process the equal components
 4264 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4265   assert(VM_Version::supports_evex(), "");
 4266   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4267   attributes.reset_is_clear_context();
 4268   attributes.set_is_evex_instruction();
 4269   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4270   emit_int16(0x29, (0xC0 | encode));
 4271 }
 4272 
 4273 // In this context, kdst is written the mask used to process the equal components
 4274 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4275   assert(VM_Version::supports_evex(), "");
 4276   InstructionMark im(this);
 4277   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4278   attributes.reset_is_clear_context();
 4279   attributes.set_is_evex_instruction();
 4280   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4281   int dst_enc = kdst->encoding();
 4282   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4283   emit_int8(0x29);
 4284   emit_operand(as_Register(dst_enc), src);
 4285 }
 4286 
 4287 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4288   assert(VM_Version::supports_sse4_1(), "");
 4289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4291   emit_int16(0x37, (0xC0 | encode));
 4292 }
 4293 
 4294 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4295   assert(VM_Version::supports_sse2(), "");
 4296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4297   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4298   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4299 }
 4300 
 4301 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4302   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4303          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4304   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4305   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4306   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4307 }
 4308 
 4309 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4310   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 4311   InstructionMark im(this);
 4312   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 4313   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4314   emit_int8((unsigned char)0x8C);
 4315   emit_operand(dst, src);
 4316 }
 4317 
 4318 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4319   assert(VM_Version::supports_sse4_1(), "");
 4320   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4321   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4322   emit_int24(0x16, (0xC0 | encode), imm8);
 4323 }
 4324 
 4325 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4326   assert(VM_Version::supports_sse4_1(), "");
 4327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4328   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4329   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4330   emit_int8(0x16);
 4331   emit_operand(src, dst);
 4332   emit_int8(imm8);
 4333 }
 4334 
 4335 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4336   assert(VM_Version::supports_sse4_1(), "");
 4337   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4338   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4339   emit_int24(0x16, (0xC0 | encode), imm8);
 4340 }
 4341 
 4342 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4343   assert(VM_Version::supports_sse4_1(), "");
 4344   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4345   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4346   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4347   emit_int8(0x16);
 4348   emit_operand(src, dst);
 4349   emit_int8(imm8);
 4350 }
 4351 
 4352 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4353   assert(VM_Version::supports_sse2(), "");
 4354   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4355   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4356   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4357 }
 4358 
 4359 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4360   assert(VM_Version::supports_sse4_1(), "");
 4361   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4362   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4363   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4364   emit_int8(0x15);
 4365   emit_operand(src, dst);
 4366   emit_int8(imm8);
 4367 }
 4368 
 4369 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4370   assert(VM_Version::supports_sse4_1(), "");
 4371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4372   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4373   emit_int24(0x14, (0xC0 | encode), imm8);
 4374 }
 4375 
 4376 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4377   assert(VM_Version::supports_sse4_1(), "");
 4378   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4379   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4380   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4381   emit_int8(0x14);
 4382   emit_operand(src, dst);
 4383   emit_int8(imm8);
 4384 }
 4385 
 4386 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4387   assert(VM_Version::supports_sse4_1(), "");
 4388   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4389   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4390   emit_int24(0x22, (0xC0 | encode), imm8);
 4391 }
 4392 
 4393 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4394   assert(VM_Version::supports_sse4_1(), "");
 4395   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4396   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4397   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4398   emit_int8(0x22);
 4399   emit_operand(dst,src);
 4400   emit_int8(imm8);
 4401 }
 4402 
 4403 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4404   assert(VM_Version::supports_avx(), "");
 4405   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4406   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4407   emit_int24(0x22, (0xC0 | encode), imm8);
 4408 }
 4409 
 4410 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4411   assert(VM_Version::supports_sse4_1(), "");
 4412   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4413   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4414   emit_int24(0x22, (0xC0 | encode), imm8);
 4415 }
 4416 
 4417 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4418   assert(VM_Version::supports_sse4_1(), "");
 4419   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4420   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4421   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4422   emit_int8(0x22);
 4423   emit_operand(dst, src);
 4424   emit_int8(imm8);
 4425 }
 4426 
 4427 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4428   assert(VM_Version::supports_avx(), "");
 4429   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4430   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4431   emit_int24(0x22, (0xC0 | encode), imm8);
 4432 }
 4433 
 4434 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4435   assert(VM_Version::supports_sse2(), "");
 4436   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4437   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4438   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4439 }
 4440 
 4441 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4442   assert(VM_Version::supports_sse2(), "");
 4443   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4444   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4445   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4446   emit_int8((unsigned char)0xC4);
 4447   emit_operand(dst, src);
 4448   emit_int8(imm8);
 4449 }
 4450 
 4451 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4452   assert(VM_Version::supports_avx(), "");
 4453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4454   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4455   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4456 }
 4457 
 4458 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4459   assert(VM_Version::supports_sse4_1(), "");
 4460   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4461   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4462   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4463   emit_int8(0x20);
 4464   emit_operand(dst, src);
 4465   emit_int8(imm8);
 4466 }
 4467 
 4468 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4469   assert(VM_Version::supports_sse4_1(), "");
 4470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4471   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4472   emit_int24(0x20, (0xC0 | encode), imm8);
 4473 }
 4474 
 4475 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4476   assert(VM_Version::supports_avx(), "");
 4477   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4478   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4479   emit_int24(0x20, (0xC0 | encode), imm8);
 4480 }
 4481 
 4482 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4483   assert(VM_Version::supports_sse4_1(), "");
 4484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4485   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4486   emit_int24(0x21, (0xC0 | encode), imm8);
 4487 }
 4488 
 4489 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4490   assert(VM_Version::supports_avx(), "");
 4491   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4492   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4493   emit_int24(0x21, (0xC0 | encode), imm8);
 4494 }
 4495 
 4496 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4497   assert(VM_Version::supports_sse4_1(), "");
 4498   InstructionMark im(this);
 4499   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4500   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4501   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4502   emit_int8(0x30);
 4503   emit_operand(dst, src);
 4504 }
 4505 
 4506 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4507   assert(VM_Version::supports_sse4_1(), "");
 4508   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4509   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4510   emit_int16(0x30, (0xC0 | encode));
 4511 }
 4512 
 4513 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4514   assert(VM_Version::supports_sse4_1(), "");
 4515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4516   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4517   emit_int16(0x20, (0xC0 | encode));
 4518 }
 4519 
 4520 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4521   assert(VM_Version::supports_sse4_1(), "");
 4522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4523   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4524   emit_int16(0x35, (0xC0 | encode));
 4525 }
 4526 
 4527 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4528   assert(VM_Version::supports_sse4_1(), "");
 4529   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4530   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4531   emit_int16(0x21, (0xC0 | encode));
 4532 }
 4533 
 4534 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4535   assert(VM_Version::supports_sse4_1(), "");
 4536   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4537   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4538   emit_int16(0x31, (0xC0 | encode));
 4539 }
 4540 
 4541 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4542   assert(VM_Version::supports_sse4_1(), "");
 4543   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4544   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4545   emit_int16(0x22, (0xC0 | encode));
 4546 }
 4547 
 4548 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4549   assert(VM_Version::supports_sse4_1(), "");
 4550   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4551   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4552   emit_int16(0x23, (0xC0 | encode));
 4553 }
 4554 
 4555 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4556   assert(VM_Version::supports_avx(), "");
 4557   InstructionMark im(this);
 4558   assert(dst != xnoreg, "sanity");
 4559   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4560   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4561   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4562   emit_int8(0x30);
 4563   emit_operand(dst, src);
 4564 }
 4565 
 4566 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4567   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4568   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4569   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4570   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4571   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4572   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4573 }
 4574 
 4575 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4576   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4577   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4578   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4579   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4580   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4581   emit_int16(0x20, (0xC0 | encode));
 4582 }
 4583 
 4584 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4585   assert(VM_Version::supports_avx512vlbw(), "");
 4586   assert(dst != xnoreg, "sanity");
 4587   InstructionMark im(this);
 4588   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4589   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4590   attributes.set_embedded_opmask_register_specifier(mask);
 4591   attributes.set_is_evex_instruction();
 4592   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4593   emit_int8(0x30);
 4594   emit_operand(dst, src);
 4595 }
 4596 
 4597 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4598   assert(VM_Version::supports_evex(), "");
 4599   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4600   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4601   attributes.set_is_evex_instruction();
 4602   attributes.set_embedded_opmask_register_specifier(mask);
 4603   if (merge) {
 4604     attributes.reset_is_clear_context();
 4605   }
 4606   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4607   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4608 }
 4609 
 4610 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4611   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4612   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4613   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4614   emit_int16(0x35, (0xC0 | encode));
 4615 }
 4616 
 4617 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4618   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4619   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4620   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4621   emit_int16(0x31, (0xC0 | encode));
 4622 }
 4623 
 4624 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4625   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4626   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4627   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4628   emit_int16(0x32, (0xC0 | encode));
 4629 }
 4630 
 4631 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4632   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4633          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4634              VM_Version::supports_evex(), "");
 4635   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4636   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4637   emit_int16(0x21, (0xC0 | encode));
 4638 }
 4639 
 4640 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4641   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4642          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4643              VM_Version::supports_evex(), "");
 4644   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4645   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4646   emit_int16(0x22, (0xC0 | encode));
 4647 }
 4648 
 4649 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4650   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4651          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4652              VM_Version::supports_evex(), "");
 4653   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4654   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4655   emit_int16(0x23, (0xC0 | encode));
 4656 }
 4657 
 4658 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4659   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4660          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4661              VM_Version::supports_evex(), "");
 4662   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4663   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4664   emit_int16(0x24, (0xC0 | encode));
 4665 }
 4666 
 4667 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4668   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4669          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4670              VM_Version::supports_evex(), "");
 4671   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4672   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4673   emit_int16(0x25, (0xC0 | encode));
 4674 }
 4675 
 4676 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4677   assert(VM_Version::supports_avx512vlbw(), "");
 4678   assert(src != xnoreg, "sanity");
 4679   InstructionMark im(this);
 4680   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4681   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4682   attributes.set_is_evex_instruction();
 4683   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4684   emit_int8(0x30);
 4685   emit_operand(src, dst);
 4686 }
 4687 
 4688 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4689   assert(VM_Version::supports_avx512vlbw(), "");
 4690   assert(src != xnoreg, "sanity");
 4691   InstructionMark im(this);
 4692   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4693   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4694   attributes.reset_is_clear_context();
 4695   attributes.set_embedded_opmask_register_specifier(mask);
 4696   attributes.set_is_evex_instruction();
 4697   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4698   emit_int8(0x30);
 4699   emit_operand(src, dst);
 4700 }
 4701 
 4702 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4703   assert(VM_Version::supports_evex(), "");
 4704   assert(src != xnoreg, "sanity");
 4705   InstructionMark im(this);
 4706   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4707   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4708   attributes.set_is_evex_instruction();
 4709   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4710   emit_int8(0x31);
 4711   emit_operand(src, dst);
 4712 }
 4713 
 4714 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4715   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4716   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4717   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4718   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4719   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4720   emit_int16(0x33, (0xC0 | encode));
 4721 }
 4722 
 4723 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4724   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4725   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4726   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4727   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4728 }
 4729 
 4730 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4731   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4732     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4733     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4734   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4735   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4736   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4737 }
 4738 
 4739 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4740 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4741        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4742        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4743   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4744   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4745   emit_int16(0x04, (0xC0 | encode));
 4746 }
 4747 
 4748 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4749   assert(VM_Version::supports_evex(), "");
 4750   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4751   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4752   attributes.set_is_evex_instruction();
 4753   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4754   emit_int16(0x52, (0xC0 | encode));
 4755 }
 4756 
 4757 // generic
 4758 void Assembler::pop(Register dst) {
 4759   int encode = prefix_and_encode(dst->encoding());
 4760   emit_int8(0x58 | encode);
 4761 }
 4762 
 4763 void Assembler::popcntl(Register dst, Address src) {
 4764   assert(VM_Version::supports_popcnt(), "must support");
 4765   InstructionMark im(this);
 4766   emit_int8((unsigned char)0xF3);
 4767   prefix(src, dst);
 4768   emit_int16(0x0F, (unsigned char)0xB8);
 4769   emit_operand(dst, src);
 4770 }
 4771 
 4772 void Assembler::popcntl(Register dst, Register src) {
 4773   assert(VM_Version::supports_popcnt(), "must support");
 4774   emit_int8((unsigned char)0xF3);
 4775   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 4776   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 4777 }
 4778 
 4779 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
 4780   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4781   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4782   attributes.set_is_evex_instruction();
 4783   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4784   emit_int16(0x55, (0xC0 | encode));
 4785 }
 4786 
 4787 void Assembler::popf() {
 4788   emit_int8((unsigned char)0x9D);
 4789 }
 4790 
 4791 #ifndef _LP64 // no 32bit push/pop on amd64
 4792 void Assembler::popl(Address dst) {
 4793   // NOTE: this will adjust stack by 8byte on 64bits
 4794   InstructionMark im(this);
 4795   prefix(dst);
 4796   emit_int8((unsigned char)0x8F);
 4797   emit_operand(rax, dst);
 4798 }
 4799 #endif
 4800 
 4801 void Assembler::prefetchnta(Address src) {
 4802   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4803   InstructionMark im(this);
 4804   prefix(src);
 4805   emit_int16(0x0F, 0x18);
 4806   emit_operand(rax, src); // 0, src
 4807 }
 4808 
 4809 void Assembler::prefetchr(Address src) {
 4810   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4811   InstructionMark im(this);
 4812   prefix(src);
 4813   emit_int16(0x0F, 0x0D);
 4814   emit_operand(rax, src); // 0, src
 4815 }
 4816 
 4817 void Assembler::prefetcht0(Address src) {
 4818   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4819   InstructionMark im(this);
 4820   prefix(src);
 4821   emit_int16(0x0F, 0x18);
 4822   emit_operand(rcx, src); // 1, src
 4823 }
 4824 
 4825 void Assembler::prefetcht1(Address src) {
 4826   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4827   InstructionMark im(this);
 4828   prefix(src);
 4829   emit_int16(0x0F, 0x18);
 4830   emit_operand(rdx, src); // 2, src
 4831 }
 4832 
 4833 void Assembler::prefetcht2(Address src) {
 4834   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4835   InstructionMark im(this);
 4836   prefix(src);
 4837   emit_int16(0x0F, 0x18);
 4838   emit_operand(rbx, src); // 3, src
 4839 }
 4840 
 4841 void Assembler::prefetchw(Address src) {
 4842   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4843   InstructionMark im(this);
 4844   prefix(src);
 4845   emit_int16(0x0F, 0x0D);
 4846   emit_operand(rcx, src); // 1, src
 4847 }
 4848 
 4849 void Assembler::prefix(Prefix p) {
 4850   emit_int8(p);
 4851 }
 4852 
 4853 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 4854   assert(VM_Version::supports_ssse3(), "");
 4855   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4856   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4857   emit_int16(0x00, (0xC0 | encode));
 4858 }
 4859 
 4860 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4861   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4862          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4863          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4864   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4865   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4866   emit_int16(0x00, (0xC0 | encode));
 4867 }
 4868 
 4869 void Assembler::pshufb(XMMRegister dst, Address src) {
 4870   assert(VM_Version::supports_ssse3(), "");
 4871   InstructionMark im(this);
 4872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4873   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4874   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4875   emit_int8(0x00);
 4876   emit_operand(dst, src);
 4877 }
 4878 
 4879 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 4880   assert(isByte(mode), "invalid value");
 4881   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4882   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 4883   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4884   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4885   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4886 }
 4887 
 4888 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 4889   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4890          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4891          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 4892   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4893   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4894   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4895   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4896 }
 4897 
 4898 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 4899   assert(isByte(mode), "invalid value");
 4900   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4901   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4902   InstructionMark im(this);
 4903   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4904   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4905   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4906   emit_int8(0x70);
 4907   emit_operand(dst, src);
 4908   emit_int8(mode & 0xFF);
 4909 }
 4910 
 4911 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 4912   assert(isByte(mode), "invalid value");
 4913   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4914   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4915   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 4916   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4917 }
 4918 
 4919 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 4920   assert(isByte(mode), "invalid value");
 4921   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4922   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4923   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 4924   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4925 }
 4926 
 4927 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 4928   assert(isByte(mode), "invalid value");
 4929   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4930   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4931   InstructionMark im(this);
 4932   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4933   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4934   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 4935   emit_int8(0x70);
 4936   emit_operand(dst, src);
 4937   emit_int8(mode & 0xFF);
 4938 }
 4939 
 4940 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4941   assert(VM_Version::supports_evex(), "requires EVEX support");
 4942   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 4943   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4944   attributes.set_is_evex_instruction();
 4945   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4946   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 4947 }
 4948 
 4949 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 4950   assert(isByte(imm8), "invalid value");
 4951   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4952   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4953   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4954   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4955 }
 4956 
 4957 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4958   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4959   attributes.set_rex_vex_w_reverted();
 4960   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4961   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4962 }
 4963 
 4964 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 4965   assert(isByte(imm8), "invalid value");
 4966   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4967   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4968   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4969   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4970 }
 4971 
 4972 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4973   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4974   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4975   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4976 }
 4977 
 4978 void Assembler::psrldq(XMMRegister dst, int shift) {
 4979   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 4980   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4981   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4982   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4983   emit_int24(0x73, (0xC0 | encode), shift);
 4984 }
 4985 
 4986 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 4987   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4988          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4989          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 4990   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4991   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4992   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 4993 }
 4994 
 4995 void Assembler::pslldq(XMMRegister dst, int shift) {
 4996   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 4997   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4998   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4999   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5000   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5001   emit_int24(0x73, (0xC0 | encode), shift);
 5002 }
 5003 
 5004 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5005   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5006          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5007          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5008   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5009   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5010   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5011 }
 5012 
 5013 void Assembler::ptest(XMMRegister dst, Address src) {
 5014   assert(VM_Version::supports_sse4_1(), "");
 5015   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5016   InstructionMark im(this);
 5017   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5018   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5019   emit_int8(0x17);
 5020   emit_operand(dst, src);
 5021 }
 5022 
 5023 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5024   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5025   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5026   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5027   emit_int8(0x17);
 5028   emit_int8((0xC0 | encode));
 5029 }
 5030 
 5031 void Assembler::vptest(XMMRegister dst, Address src) {
 5032   assert(VM_Version::supports_avx(), "");
 5033   InstructionMark im(this);
 5034   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5035   assert(dst != xnoreg, "sanity");
 5036   // swap src<->dst for encoding
 5037   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5038   emit_int8(0x17);
 5039   emit_operand(dst, src);
 5040 }
 5041 
 5042 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5043   assert(VM_Version::supports_avx(), "");
 5044   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5045   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5046   emit_int16(0x17, (0xC0 | encode));
 5047 }
 5048 
 5049 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5050   assert(VM_Version::supports_avx(), "");
 5051   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5052   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5053   emit_int16(0x17, (0xC0 | encode));
 5054 }
 5055 
 5056 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5057   assert(VM_Version::supports_avx512vlbw(), "");
 5058   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 5059   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5060   attributes.set_is_evex_instruction();
 5061   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5062   emit_int16((unsigned char)0x26, (0xC0 | encode));
 5063 }
 5064 
 5065 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5066   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5067   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5068   InstructionMark im(this);
 5069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5070   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5071   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5072   emit_int8(0x60);
 5073   emit_operand(dst, src);
 5074 }
 5075 
 5076 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5077   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5078   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5079   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5080   emit_int16(0x60, (0xC0 | encode));
 5081 }
 5082 
 5083 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5084   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5085   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5086   InstructionMark im(this);
 5087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5088   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5089   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5090   emit_int8(0x62);
 5091   emit_operand(dst, src);
 5092 }
 5093 
 5094 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5095   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5096   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5097   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5098   emit_int16(0x62, (0xC0 | encode));
 5099 }
 5100 
 5101 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5102   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5103   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5104   attributes.set_rex_vex_w_reverted();
 5105   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5106   emit_int16(0x6C, (0xC0 | encode));
 5107 }
 5108 
 5109 void Assembler::push(int32_t imm32) {
 5110   // in 64bits we push 64bits onto the stack but only
 5111   // take a 32bit immediate
 5112   emit_int8(0x68);
 5113   emit_int32(imm32);
 5114 }
 5115 
 5116 void Assembler::push(Register src) {
 5117   int encode = prefix_and_encode(src->encoding());
 5118   emit_int8(0x50 | encode);
 5119 }
 5120 
 5121 void Assembler::pushf() {
 5122   emit_int8((unsigned char)0x9C);
 5123 }
 5124 
 5125 #ifndef _LP64 // no 32bit push/pop on amd64
 5126 void Assembler::pushl(Address src) {
 5127   // Note this will push 64bit on 64bit
 5128   InstructionMark im(this);
 5129   prefix(src);
 5130   emit_int8((unsigned char)0xFF);
 5131   emit_operand(rsi, src);
 5132 }
 5133 #endif
 5134 
 5135 void Assembler::rcll(Register dst, int imm8) {
 5136   assert(isShiftCount(imm8), "illegal shift count");
 5137   int encode = prefix_and_encode(dst->encoding());
 5138   if (imm8 == 1) {
 5139     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5140   } else {
 5141     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5142   }
 5143 }
 5144 
 5145 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5146   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5147   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5148   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5149   emit_int16(0x53, (0xC0 | encode));
 5150 }
 5151 
 5152 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5153   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5154   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5155   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5156   emit_int16(0x53, (0xC0 | encode));
 5157 }
 5158 
 5159 void Assembler::rdtsc() {
 5160   emit_int16(0x0F, 0x31);
 5161 }
 5162 
 5163 // copies data from [esi] to [edi] using rcx pointer sized words
 5164 // generic
 5165 void Assembler::rep_mov() {
 5166   // REP
 5167   // MOVSQ
 5168   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5169   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5170 }
 5171 
 5172 // sets rcx bytes with rax, value at [edi]
 5173 void Assembler::rep_stosb() {
 5174   // REP
 5175   // STOSB
 5176   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5177   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5178 }
 5179 
 5180 // sets rcx pointer sized words with rax, value at [edi]
 5181 // generic
 5182 void Assembler::rep_stos() {
 5183   // REP
 5184   // LP64:STOSQ, LP32:STOSD
 5185   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5186   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5187 }
 5188 
 5189 // scans rcx pointer sized words at [edi] for occurance of rax,
 5190 // generic
 5191 void Assembler::repne_scan() { // repne_scan
 5192   // SCASQ
 5193   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5194   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5195 }
 5196 
 5197 #ifdef _LP64
 5198 // scans rcx 4 byte words at [edi] for occurance of rax,
 5199 // generic
 5200 void Assembler::repne_scanl() { // repne_scan
 5201   // SCASL
 5202   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5203 }
 5204 #endif
 5205 
 5206 void Assembler::ret(int imm16) {
 5207   if (imm16 == 0) {
 5208     emit_int8((unsigned char)0xC3);
 5209   } else {
 5210     emit_int8((unsigned char)0xC2);
 5211     emit_int16(imm16);
 5212   }
 5213 }
 5214 
 5215 void Assembler::roll(Register dst, int imm8) {
 5216   assert(isShiftCount(imm8), "illegal shift count");
 5217   int encode = prefix_and_encode(dst->encoding());
 5218   if (imm8 == 1) {
 5219     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5220   } else {
 5221     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5222   }
 5223 }
 5224 
 5225 void Assembler::roll(Register dst) {
 5226   int encode = prefix_and_encode(dst->encoding());
 5227   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5228 }
 5229 
 5230 void Assembler::rorl(Register dst, int imm8) {
 5231   assert(isShiftCount(imm8), "illegal shift count");
 5232   int encode = prefix_and_encode(dst->encoding());
 5233   if (imm8 == 1) {
 5234     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5235   } else {
 5236     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5237   }
 5238 }
 5239 
 5240 void Assembler::rorl(Register dst) {
 5241   int encode = prefix_and_encode(dst->encoding());
 5242   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5243 }
 5244 
 5245 #ifdef _LP64
 5246 void Assembler::rorq(Register dst) {
 5247   int encode = prefixq_and_encode(dst->encoding());
 5248   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5249 }
 5250 
 5251 void Assembler::rorq(Register dst, int imm8) {
 5252   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5253   int encode = prefixq_and_encode(dst->encoding());
 5254   if (imm8 == 1) {
 5255     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5256   } else {
 5257     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5258   }
 5259 }
 5260 
 5261 void Assembler::rolq(Register dst) {
 5262   int encode = prefixq_and_encode(dst->encoding());
 5263   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5264 }
 5265 
 5266 void Assembler::rolq(Register dst, int imm8) {
 5267   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5268   int encode = prefixq_and_encode(dst->encoding());
 5269   if (imm8 == 1) {
 5270     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5271   } else {
 5272     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5273   }
 5274 }
 5275 #endif
 5276 
 5277 void Assembler::sahf() {
 5278 #ifdef _LP64
 5279   // Not supported in 64bit mode
 5280   ShouldNotReachHere();
 5281 #endif
 5282   emit_int8((unsigned char)0x9E);
 5283 }
 5284 
 5285 void Assembler::sall(Address dst, int imm8) {
 5286   InstructionMark im(this);
 5287   assert(isShiftCount(imm8), "illegal shift count");
 5288   prefix(dst);
 5289   if (imm8 == 1) {
 5290     emit_int8((unsigned char)0xD1);
 5291     emit_operand(as_Register(4), dst);
 5292   }
 5293   else {
 5294     emit_int8((unsigned char)0xC1);
 5295     emit_operand(as_Register(4), dst);
 5296     emit_int8(imm8);
 5297   }
 5298 }
 5299 
 5300 void Assembler::sall(Address dst) {
 5301   InstructionMark im(this);
 5302   prefix(dst);
 5303   emit_int8((unsigned char)0xD3);
 5304   emit_operand(as_Register(4), dst);
 5305 }
 5306 
 5307 void Assembler::sall(Register dst, int imm8) {
 5308   assert(isShiftCount(imm8), "illegal shift count");
 5309   int encode = prefix_and_encode(dst->encoding());
 5310   if (imm8 == 1) {
 5311     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5312   } else {
 5313     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5314   }
 5315 }
 5316 
 5317 void Assembler::sall(Register dst) {
 5318   int encode = prefix_and_encode(dst->encoding());
 5319   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5320 }
 5321 
 5322 void Assembler::sarl(Address dst, int imm8) {
 5323   assert(isShiftCount(imm8), "illegal shift count");
 5324   InstructionMark im(this);
 5325   prefix(dst);
 5326   if (imm8 == 1) {
 5327     emit_int8((unsigned char)0xD1);
 5328     emit_operand(as_Register(7), dst);
 5329   }
 5330   else {
 5331     emit_int8((unsigned char)0xC1);
 5332     emit_operand(as_Register(7), dst);
 5333     emit_int8(imm8);
 5334   }
 5335 }
 5336 
 5337 void Assembler::sarl(Address dst) {
 5338   InstructionMark im(this);
 5339   prefix(dst);
 5340   emit_int8((unsigned char)0xD3);
 5341   emit_operand(as_Register(7), dst);
 5342 }
 5343 
 5344 void Assembler::sarl(Register dst, int imm8) {
 5345   int encode = prefix_and_encode(dst->encoding());
 5346   assert(isShiftCount(imm8), "illegal shift count");
 5347   if (imm8 == 1) {
 5348     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5349   } else {
 5350     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5351   }
 5352 }
 5353 
 5354 void Assembler::sarl(Register dst) {
 5355   int encode = prefix_and_encode(dst->encoding());
 5356   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5357 }
 5358 
 5359 void Assembler::sbbl(Address dst, int32_t imm32) {
 5360   InstructionMark im(this);
 5361   prefix(dst);
 5362   emit_arith_operand(0x81, rbx, dst, imm32);
 5363 }
 5364 
 5365 void Assembler::sbbl(Register dst, int32_t imm32) {
 5366   prefix(dst);
 5367   emit_arith(0x81, 0xD8, dst, imm32);
 5368 }
 5369 
 5370 
 5371 void Assembler::sbbl(Register dst, Address src) {
 5372   InstructionMark im(this);
 5373   prefix(src, dst);
 5374   emit_int8(0x1B);
 5375   emit_operand(dst, src);
 5376 }
 5377 
 5378 void Assembler::sbbl(Register dst, Register src) {
 5379   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5380   emit_arith(0x1B, 0xC0, dst, src);
 5381 }
 5382 
 5383 void Assembler::setb(Condition cc, Register dst) {
 5384   assert(0 <= cc && cc < 16, "illegal cc");
 5385   int encode = prefix_and_encode(dst->encoding(), true);
 5386   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5387 }
 5388 
 5389 void Assembler::sete(Register dst) {
 5390   int encode = prefix_and_encode(dst->encoding(), true);
 5391   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5392 }
 5393 
 5394 void Assembler::setl(Register dst) {
 5395   int encode = prefix_and_encode(dst->encoding(), true);
 5396   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5397 }
 5398 
 5399 void Assembler::setne(Register dst) {
 5400   int encode = prefix_and_encode(dst->encoding(), true);
 5401   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5402 }
 5403 
 5404 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5405   assert(VM_Version::supports_ssse3(), "");
 5406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5407   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5408   emit_int24(0x0F, (0xC0 | encode), imm8);
 5409 }
 5410 
 5411 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5412   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5413          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5414          0, "");
 5415   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5416   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5417   emit_int24(0x0F, (0xC0 | encode), imm8);
 5418 }
 5419 
 5420 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5421   assert(VM_Version::supports_evex(), "");
 5422   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5423   attributes.set_is_evex_instruction();
 5424   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5425   emit_int24(0x3, (0xC0 | encode), imm8);
 5426 }
 5427 
 5428 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5429   assert(VM_Version::supports_sse4_1(), "");
 5430   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5431   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5432   emit_int24(0x0E, (0xC0 | encode), imm8);
 5433 }
 5434 
 5435 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5436   assert(VM_Version::supports_sha(), "");
 5437   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5438   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5439 }
 5440 
 5441 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5442   assert(VM_Version::supports_sha(), "");
 5443   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5444   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5445 }
 5446 
 5447 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5448   assert(VM_Version::supports_sha(), "");
 5449   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5450   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5451 }
 5452 
 5453 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5454   assert(VM_Version::supports_sha(), "");
 5455   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5456   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5457 }
 5458 
 5459 // xmm0 is implicit additional source to this instruction.
 5460 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5461   assert(VM_Version::supports_sha(), "");
 5462   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5463   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5464 }
 5465 
 5466 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5467   assert(VM_Version::supports_sha(), "");
 5468   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5469   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5470 }
 5471 
 5472 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5473   assert(VM_Version::supports_sha(), "");
 5474   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5475   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5476 }
 5477 
 5478 
 5479 void Assembler::shll(Register dst, int imm8) {
 5480   assert(isShiftCount(imm8), "illegal shift count");
 5481   int encode = prefix_and_encode(dst->encoding());
 5482   if (imm8 == 1 ) {
 5483     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5484   } else {
 5485     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5486   }
 5487 }
 5488 
 5489 void Assembler::shll(Register dst) {
 5490   int encode = prefix_and_encode(dst->encoding());
 5491   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5492 }
 5493 
 5494 void Assembler::shrl(Register dst, int imm8) {
 5495   assert(isShiftCount(imm8), "illegal shift count");
 5496   int encode = prefix_and_encode(dst->encoding());
 5497   if (imm8 == 1) {
 5498     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5499   }
 5500   else {
 5501     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5502   }
 5503 }
 5504 
 5505 void Assembler::shrl(Register dst) {
 5506   int encode = prefix_and_encode(dst->encoding());
 5507   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5508 }
 5509 
 5510 void Assembler::shrl(Address dst) {
 5511   InstructionMark im(this);
 5512   prefix(dst);
 5513   emit_int8((unsigned char)0xD3);
 5514   emit_operand(as_Register(5), dst);
 5515 }
 5516 
 5517 void Assembler::shrl(Address dst, int imm8) {
 5518   InstructionMark im(this);
 5519   assert(isShiftCount(imm8), "illegal shift count");
 5520   prefix(dst);
 5521   if (imm8 == 1) {
 5522     emit_int8((unsigned char)0xD1);
 5523     emit_operand(as_Register(5), dst);
 5524   }
 5525   else {
 5526     emit_int8((unsigned char)0xC1);
 5527     emit_operand(as_Register(5), dst);
 5528     emit_int8(imm8);
 5529   }
 5530 }
 5531 
 5532 
 5533 void Assembler::shldl(Register dst, Register src) {
 5534   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5535   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5536 }
 5537 
 5538 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5539   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5540   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5541 }
 5542 
 5543 void Assembler::shrdl(Register dst, Register src) {
 5544   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5545   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5546 }
 5547 
 5548 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5549   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5550   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5551 }
 5552 
 5553 // copies a single word from [esi] to [edi]
 5554 void Assembler::smovl() {
 5555   emit_int8((unsigned char)0xA5);
 5556 }
 5557 
 5558 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5559   assert(VM_Version::supports_sse4_1(), "");
 5560   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5561   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5562   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5563 }
 5564 
 5565 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5566   assert(VM_Version::supports_sse4_1(), "");
 5567   InstructionMark im(this);
 5568   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5569   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5570   emit_int8(0x0B);
 5571   emit_operand(dst, src);
 5572   emit_int8((unsigned char)rmode);
 5573 }
 5574 
 5575 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5576   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5577   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5578   attributes.set_rex_vex_w_reverted();
 5579   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5580   emit_int16(0x51, (0xC0 | encode));
 5581 }
 5582 
 5583 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5584   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5585   InstructionMark im(this);
 5586   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5587   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5588   attributes.set_rex_vex_w_reverted();
 5589   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5590   emit_int8(0x51);
 5591   emit_operand(dst, src);
 5592 }
 5593 
 5594 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5595   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5596   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5597   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5598   emit_int16(0x51, (0xC0 | encode));
 5599 }
 5600 
 5601 void Assembler::std() {
 5602   emit_int8((unsigned char)0xFD);
 5603 }
 5604 
 5605 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5606   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5607   InstructionMark im(this);
 5608   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5609   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5610   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5611   emit_int8(0x51);
 5612   emit_operand(dst, src);
 5613 }
 5614 
 5615 void Assembler::stmxcsr( Address dst) {
 5616   if (UseAVX > 0 ) {
 5617     assert(VM_Version::supports_avx(), "");
 5618     InstructionMark im(this);
 5619     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5620     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5621     emit_int8((unsigned char)0xAE);
 5622     emit_operand(as_Register(3), dst);
 5623   } else {
 5624     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5625     InstructionMark im(this);
 5626     prefix(dst);
 5627     emit_int16(0x0F, (unsigned char)0xAE);
 5628     emit_operand(as_Register(3), dst);
 5629   }
 5630 }
 5631 
 5632 void Assembler::subl(Address dst, int32_t imm32) {
 5633   InstructionMark im(this);
 5634   prefix(dst);
 5635   emit_arith_operand(0x81, rbp, dst, imm32);
 5636 }
 5637 
 5638 void Assembler::subl(Address dst, Register src) {
 5639   InstructionMark im(this);
 5640   prefix(dst, src);
 5641   emit_int8(0x29);
 5642   emit_operand(src, dst);
 5643 }
 5644 
 5645 void Assembler::subl(Register dst, int32_t imm32) {
 5646   prefix(dst);
 5647   emit_arith(0x81, 0xE8, dst, imm32);
 5648 }
 5649 
 5650 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5651 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5652   prefix(dst);
 5653   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5654 }
 5655 
 5656 void Assembler::subl(Register dst, Address src) {
 5657   InstructionMark im(this);
 5658   prefix(src, dst);
 5659   emit_int8(0x2B);
 5660   emit_operand(dst, src);
 5661 }
 5662 
 5663 void Assembler::subl(Register dst, Register src) {
 5664   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5665   emit_arith(0x2B, 0xC0, dst, src);
 5666 }
 5667 
 5668 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5669   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5670   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5671   attributes.set_rex_vex_w_reverted();
 5672   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5673   emit_int16(0x5C, (0xC0 | encode));
 5674 }
 5675 
 5676 void Assembler::subsd(XMMRegister dst, Address src) {
 5677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5678   InstructionMark im(this);
 5679   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5680   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5681   attributes.set_rex_vex_w_reverted();
 5682   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5683   emit_int8(0x5C);
 5684   emit_operand(dst, src);
 5685 }
 5686 
 5687 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5688   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5690   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5691   emit_int16(0x5C, (0xC0 | encode));
 5692 }
 5693 
 5694 void Assembler::subss(XMMRegister dst, Address src) {
 5695   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5696   InstructionMark im(this);
 5697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5698   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5699   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5700   emit_int8(0x5C);
 5701   emit_operand(dst, src);
 5702 }
 5703 
 5704 void Assembler::testb(Register dst, int imm8) {
 5705   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5706   (void) prefix_and_encode(dst->encoding(), true);
 5707   emit_arith_b(0xF6, 0xC0, dst, imm8);
 5708 }
 5709 
 5710 void Assembler::testb(Address dst, int imm8) {
 5711   InstructionMark im(this);
 5712   prefix(dst);
 5713   emit_int8((unsigned char)0xF6);
 5714   emit_operand(rax, dst, 1);
 5715   emit_int8(imm8);
 5716 }
 5717 
 5718 void Assembler::testl(Register dst, int32_t imm32) {
 5719   // not using emit_arith because test
 5720   // doesn't support sign-extension of
 5721   // 8bit operands
 5722   int encode = dst->encoding();
 5723   encode = prefix_and_encode(encode);
 5724   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5725   emit_int32(imm32);
 5726 }
 5727 
 5728 void Assembler::testl(Register dst, Register src) {
 5729   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5730   emit_arith(0x85, 0xC0, dst, src);
 5731 }
 5732 
 5733 void Assembler::testl(Register dst, Address src) {
 5734   InstructionMark im(this);
 5735   prefix(src, dst);
 5736   emit_int8((unsigned char)0x85);
 5737   emit_operand(dst, src);
 5738 }
 5739 
 5740 void Assembler::tzcntl(Register dst, Register src) {
 5741   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5742   emit_int8((unsigned char)0xF3);
 5743   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5744   emit_int24(0x0F,
 5745              (unsigned char)0xBC,
 5746              0xC0 | encode);
 5747 }
 5748 
 5749 void Assembler::tzcntq(Register dst, Register src) {
 5750   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5751   emit_int8((unsigned char)0xF3);
 5752   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5753   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5754 }
 5755 
 5756 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5757   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5758   InstructionMark im(this);
 5759   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5760   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5761   attributes.set_rex_vex_w_reverted();
 5762   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5763   emit_int8(0x2E);
 5764   emit_operand(dst, src);
 5765 }
 5766 
 5767 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 5768   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5769   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5770   attributes.set_rex_vex_w_reverted();
 5771   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5772   emit_int16(0x2E, (0xC0 | encode));
 5773 }
 5774 
 5775 void Assembler::ucomiss(XMMRegister dst, Address src) {
 5776   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5777   InstructionMark im(this);
 5778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5779   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5780   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5781   emit_int8(0x2E);
 5782   emit_operand(dst, src);
 5783 }
 5784 
 5785 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 5786   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5787   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5788   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5789   emit_int16(0x2E, (0xC0 | encode));
 5790 }
 5791 
 5792 void Assembler::xabort(int8_t imm8) {
 5793   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 5794 }
 5795 
 5796 void Assembler::xaddb(Address dst, Register src) {
 5797   InstructionMark im(this);
 5798   prefix(dst, src, true);
 5799   emit_int16(0x0F, (unsigned char)0xC0);
 5800   emit_operand(src, dst);
 5801 }
 5802 
 5803 void Assembler::xaddw(Address dst, Register src) {
 5804   InstructionMark im(this);
 5805   emit_int8(0x66);
 5806   prefix(dst, src);
 5807   emit_int16(0x0F, (unsigned char)0xC1);
 5808   emit_operand(src, dst);
 5809 }
 5810 
 5811 void Assembler::xaddl(Address dst, Register src) {
 5812   InstructionMark im(this);
 5813   prefix(dst, src);
 5814   emit_int16(0x0F, (unsigned char)0xC1);
 5815   emit_operand(src, dst);
 5816 }
 5817 
 5818 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 5819   InstructionMark im(this);
 5820   relocate(rtype);
 5821   if (abort.is_bound()) {
 5822     address entry = target(abort);
 5823     assert(entry != NULL, "abort entry NULL");
 5824     intptr_t offset = entry - pc();
 5825     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5826     emit_int32(offset - 6); // 2 opcode + 4 address
 5827   } else {
 5828     abort.add_patch_at(code(), locator());
 5829     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5830     emit_int32(0);
 5831   }
 5832 }
 5833 
 5834 void Assembler::xchgb(Register dst, Address src) { // xchg
 5835   InstructionMark im(this);
 5836   prefix(src, dst, true);
 5837   emit_int8((unsigned char)0x86);
 5838   emit_operand(dst, src);
 5839 }
 5840 
 5841 void Assembler::xchgw(Register dst, Address src) { // xchg
 5842   InstructionMark im(this);
 5843   emit_int8(0x66);
 5844   prefix(src, dst);
 5845   emit_int8((unsigned char)0x87);
 5846   emit_operand(dst, src);
 5847 }
 5848 
 5849 void Assembler::xchgl(Register dst, Address src) { // xchg
 5850   InstructionMark im(this);
 5851   prefix(src, dst);
 5852   emit_int8((unsigned char)0x87);
 5853   emit_operand(dst, src);
 5854 }
 5855 
 5856 void Assembler::xchgl(Register dst, Register src) {
 5857   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5858   emit_int16((unsigned char)0x87, (0xC0 | encode));
 5859 }
 5860 
 5861 void Assembler::xend() {
 5862   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 5863 }
 5864 
 5865 void Assembler::xgetbv() {
 5866   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 5867 }
 5868 
 5869 void Assembler::xorl(Address dst, int32_t imm32) {
 5870   InstructionMark im(this);
 5871   prefix(dst);
 5872   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 5873 }
 5874 
 5875 void Assembler::xorl(Register dst, int32_t imm32) {
 5876   prefix(dst);
 5877   emit_arith(0x81, 0xF0, dst, imm32);
 5878 }
 5879 
 5880 void Assembler::xorl(Register dst, Address src) {
 5881   InstructionMark im(this);
 5882   prefix(src, dst);
 5883   emit_int8(0x33);
 5884   emit_operand(dst, src);
 5885 }
 5886 
 5887 void Assembler::xorl(Register dst, Register src) {
 5888   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5889   emit_arith(0x33, 0xC0, dst, src);
 5890 }
 5891 
 5892 void Assembler::xorl(Address dst, Register src) {
 5893   InstructionMark im(this);
 5894   prefix(dst, src);
 5895   emit_int8(0x31);
 5896   emit_operand(src, dst);
 5897 }
 5898 
 5899 void Assembler::xorb(Register dst, Address src) {
 5900   InstructionMark im(this);
 5901   prefix(src, dst);
 5902   emit_int8(0x32);
 5903   emit_operand(dst, src);
 5904 }
 5905 
 5906 void Assembler::xorb(Address dst, Register src) {
 5907   InstructionMark im(this);
 5908   prefix(dst, src, true);
 5909   emit_int8(0x30);
 5910   emit_operand(src, dst);
 5911 }
 5912 
 5913 void Assembler::xorw(Register dst, Register src) {
 5914   (void)prefix_and_encode(dst->encoding(), src->encoding());
 5915   emit_arith(0x33, 0xC0, dst, src);
 5916 }
 5917 
 5918 // AVX 3-operands scalar float-point arithmetic instructions
 5919 
 5920 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 5921   assert(VM_Version::supports_avx(), "");
 5922   InstructionMark im(this);
 5923   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5924   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5925   attributes.set_rex_vex_w_reverted();
 5926   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5927   emit_int8(0x58);
 5928   emit_operand(dst, src);
 5929 }
 5930 
 5931 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5932   assert(VM_Version::supports_avx(), "");
 5933   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5934   attributes.set_rex_vex_w_reverted();
 5935   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5936   emit_int16(0x58, (0xC0 | encode));
 5937 }
 5938 
 5939 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 5940   assert(VM_Version::supports_avx(), "");
 5941   InstructionMark im(this);
 5942   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5943   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5944   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5945   emit_int8(0x58);
 5946   emit_operand(dst, src);
 5947 }
 5948 
 5949 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5950   assert(VM_Version::supports_avx(), "");
 5951   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5952   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5953   emit_int16(0x58, (0xC0 | encode));
 5954 }
 5955 
 5956 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 5957   assert(VM_Version::supports_avx(), "");
 5958   InstructionMark im(this);
 5959   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5960   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5961   attributes.set_rex_vex_w_reverted();
 5962   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5963   emit_int8(0x5E);
 5964   emit_operand(dst, src);
 5965 }
 5966 
 5967 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5968   assert(VM_Version::supports_avx(), "");
 5969   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5970   attributes.set_rex_vex_w_reverted();
 5971   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5972   emit_int16(0x5E, (0xC0 | encode));
 5973 }
 5974 
 5975 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 5976   assert(VM_Version::supports_avx(), "");
 5977   InstructionMark im(this);
 5978   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5979   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5980   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5981   emit_int8(0x5E);
 5982   emit_operand(dst, src);
 5983 }
 5984 
 5985 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5986   assert(VM_Version::supports_avx(), "");
 5987   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5988   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5989   emit_int16(0x5E, (0xC0 | encode));
 5990 }
 5991 
 5992 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 5993   assert(VM_Version::supports_fma(), "");
 5994   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5995   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5996   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 5997 }
 5998 
 5999 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6000   assert(VM_Version::supports_fma(), "");
 6001   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6002   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6003   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6004 }
 6005 
 6006 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6007   assert(VM_Version::supports_avx(), "");
 6008   InstructionMark im(this);
 6009   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6010   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6011   attributes.set_rex_vex_w_reverted();
 6012   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6013   emit_int8(0x59);
 6014   emit_operand(dst, src);
 6015 }
 6016 
 6017 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6018   assert(VM_Version::supports_avx(), "");
 6019   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6020   attributes.set_rex_vex_w_reverted();
 6021   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6022   emit_int16(0x59, (0xC0 | encode));
 6023 }
 6024 
 6025 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6026   assert(VM_Version::supports_avx(), "");
 6027   InstructionMark im(this);
 6028   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6029   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6030   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6031   emit_int8(0x59);
 6032   emit_operand(dst, src);
 6033 }
 6034 
 6035 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6036   assert(VM_Version::supports_avx(), "");
 6037   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6038   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6039   emit_int16(0x59, (0xC0 | encode));
 6040 }
 6041 
 6042 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6043   assert(VM_Version::supports_avx(), "");
 6044   InstructionMark im(this);
 6045   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6046   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6047   attributes.set_rex_vex_w_reverted();
 6048   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6049   emit_int8(0x5C);
 6050   emit_operand(dst, src);
 6051 }
 6052 
 6053 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6054   assert(VM_Version::supports_avx(), "");
 6055   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6056   attributes.set_rex_vex_w_reverted();
 6057   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6058   emit_int16(0x5C, (0xC0 | encode));
 6059 }
 6060 
 6061 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6062   assert(VM_Version::supports_avx(), "");
 6063   InstructionMark im(this);
 6064   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6065   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6066   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6067   emit_int8(0x5C);
 6068   emit_operand(dst, src);
 6069 }
 6070 
 6071 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6072   assert(VM_Version::supports_avx(), "");
 6073   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6074   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6075   emit_int16(0x5C, (0xC0 | encode));
 6076 }
 6077 
 6078 //====================VECTOR ARITHMETIC=====================================
 6079 
 6080 // Float-point vector arithmetic
 6081 
 6082 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6083   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6084   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6085   attributes.set_rex_vex_w_reverted();
 6086   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6087   emit_int16(0x58, (0xC0 | encode));
 6088 }
 6089 
 6090 void Assembler::addpd(XMMRegister dst, Address src) {
 6091   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6092   InstructionMark im(this);
 6093   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6094   attributes.set_rex_vex_w_reverted();
 6095   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6096   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6097   emit_int8(0x58);
 6098   emit_operand(dst, src);
 6099 }
 6100 
 6101 
 6102 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6103   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6104   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6105   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6106   emit_int16(0x58, (0xC0 | encode));
 6107 }
 6108 
 6109 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6110   assert(VM_Version::supports_avx(), "");
 6111   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6112   attributes.set_rex_vex_w_reverted();
 6113   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6114   emit_int16(0x58, (0xC0 | encode));
 6115 }
 6116 
 6117 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6118   assert(VM_Version::supports_avx(), "");
 6119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6120   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6121   emit_int16(0x58, (0xC0 | encode));
 6122 }
 6123 
 6124 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6125   assert(VM_Version::supports_avx(), "");
 6126   InstructionMark im(this);
 6127   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6128   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6129   attributes.set_rex_vex_w_reverted();
 6130   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6131   emit_int8(0x58);
 6132   emit_operand(dst, src);
 6133 }
 6134 
 6135 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6136   assert(VM_Version::supports_avx(), "");
 6137   InstructionMark im(this);
 6138   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6139   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6140   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6141   emit_int8(0x58);
 6142   emit_operand(dst, src);
 6143 }
 6144 
 6145 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6146   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6147   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6148   attributes.set_rex_vex_w_reverted();
 6149   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6150   emit_int16(0x5C, (0xC0 | encode));
 6151 }
 6152 
 6153 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6155   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6156   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6157   emit_int16(0x5C, (0xC0 | encode));
 6158 }
 6159 
 6160 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6161   assert(VM_Version::supports_avx(), "");
 6162   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6163   attributes.set_rex_vex_w_reverted();
 6164   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6165   emit_int16(0x5C, (0xC0 | encode));
 6166 }
 6167 
 6168 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6169   assert(VM_Version::supports_avx(), "");
 6170   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6171   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6172   emit_int16(0x5C, (0xC0 | encode));
 6173 }
 6174 
 6175 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6176   assert(VM_Version::supports_avx(), "");
 6177   InstructionMark im(this);
 6178   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6179   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6180   attributes.set_rex_vex_w_reverted();
 6181   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6182   emit_int8(0x5C);
 6183   emit_operand(dst, src);
 6184 }
 6185 
 6186 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6187   assert(VM_Version::supports_avx(), "");
 6188   InstructionMark im(this);
 6189   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6190   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6191   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6192   emit_int8(0x5C);
 6193   emit_operand(dst, src);
 6194 }
 6195 
 6196 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6197   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6198   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6199   attributes.set_rex_vex_w_reverted();
 6200   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6201   emit_int16(0x59, (0xC0 | encode));
 6202 }
 6203 
 6204 void Assembler::mulpd(XMMRegister dst, Address src) {
 6205   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6206   InstructionMark im(this);
 6207   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6208   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6209   attributes.set_rex_vex_w_reverted();
 6210   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6211   emit_int8(0x59);
 6212   emit_operand(dst, src);
 6213 }
 6214 
 6215 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6216   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6217   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6218   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6219   emit_int16(0x59, (0xC0 | encode));
 6220 }
 6221 
 6222 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6223   assert(VM_Version::supports_avx(), "");
 6224   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6225   attributes.set_rex_vex_w_reverted();
 6226   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6227   emit_int16(0x59, (0xC0 | encode));
 6228 }
 6229 
 6230 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6231   assert(VM_Version::supports_avx(), "");
 6232   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6233   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6234   emit_int16(0x59, (0xC0 | encode));
 6235 }
 6236 
 6237 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6238   assert(VM_Version::supports_avx(), "");
 6239   InstructionMark im(this);
 6240   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6241   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6242   attributes.set_rex_vex_w_reverted();
 6243   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6244   emit_int8(0x59);
 6245   emit_operand(dst, src);
 6246 }
 6247 
 6248 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6249   assert(VM_Version::supports_avx(), "");
 6250   InstructionMark im(this);
 6251   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6252   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6253   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6254   emit_int8(0x59);
 6255   emit_operand(dst, src);
 6256 }
 6257 
 6258 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6259   assert(VM_Version::supports_fma(), "");
 6260   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6261   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6262   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6263 }
 6264 
 6265 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6266   assert(VM_Version::supports_fma(), "");
 6267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6268   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6269   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6270 }
 6271 
 6272 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6273   assert(VM_Version::supports_fma(), "");
 6274   InstructionMark im(this);
 6275   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6276   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6277   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6278   emit_int8((unsigned char)0xB8);
 6279   emit_operand(dst, src2);
 6280 }
 6281 
 6282 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6283   assert(VM_Version::supports_fma(), "");
 6284   InstructionMark im(this);
 6285   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6286   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6287   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6288   emit_int8((unsigned char)0xB8);
 6289   emit_operand(dst, src2);
 6290 }
 6291 
 6292 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6294   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6295   attributes.set_rex_vex_w_reverted();
 6296   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6297   emit_int16(0x5E, (0xC0 | encode));
 6298 }
 6299 
 6300 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6301   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6303   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6304   emit_int16(0x5E, (0xC0 | encode));
 6305 }
 6306 
 6307 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6308   assert(VM_Version::supports_avx(), "");
 6309   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6310   attributes.set_rex_vex_w_reverted();
 6311   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6312   emit_int16(0x5E, (0xC0 | encode));
 6313 }
 6314 
 6315 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6316   assert(VM_Version::supports_avx(), "");
 6317   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6318   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6319   emit_int16(0x5E, (0xC0 | encode));
 6320 }
 6321 
 6322 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6323   assert(VM_Version::supports_avx(), "");
 6324   InstructionMark im(this);
 6325   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6326   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6327   attributes.set_rex_vex_w_reverted();
 6328   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6329   emit_int8(0x5E);
 6330   emit_operand(dst, src);
 6331 }
 6332 
 6333 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6334   assert(VM_Version::supports_avx(), "");
 6335   InstructionMark im(this);
 6336   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6337   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6338   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6339   emit_int8(0x5E);
 6340   emit_operand(dst, src);
 6341 }
 6342 
 6343 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6344   assert(VM_Version::supports_avx(), "");
 6345   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6346   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6347   emit_int24(0x09, (0xC0 | encode), (rmode));
 6348 }
 6349 
 6350 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6351   assert(VM_Version::supports_avx(), "");
 6352   InstructionMark im(this);
 6353   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6354   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6355   emit_int8(0x09);
 6356   emit_operand(dst, src);
 6357   emit_int8((rmode));
 6358 }
 6359 
 6360 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6361   assert(VM_Version::supports_evex(), "requires EVEX support");
 6362   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6363   attributes.set_is_evex_instruction();
 6364   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6365   emit_int24(0x09, (0xC0 | encode), (rmode));
 6366 }
 6367 
 6368 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6369   assert(VM_Version::supports_evex(), "requires EVEX support");
 6370   assert(dst != xnoreg, "sanity");
 6371   InstructionMark im(this);
 6372   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6373   attributes.set_is_evex_instruction();
 6374   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6375   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6376   emit_int8(0x09);
 6377   emit_operand(dst, src);
 6378   emit_int8((rmode));
 6379 }
 6380 
 6381 
 6382 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6383   assert(VM_Version::supports_avx(), "");
 6384   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6385   attributes.set_rex_vex_w_reverted();
 6386   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6387   emit_int16(0x51, (0xC0 | encode));
 6388 }
 6389 
 6390 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6391   assert(VM_Version::supports_avx(), "");
 6392   InstructionMark im(this);
 6393   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6394   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6395   attributes.set_rex_vex_w_reverted();
 6396   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6397   emit_int8(0x51);
 6398   emit_operand(dst, src);
 6399 }
 6400 
 6401 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6402   assert(VM_Version::supports_avx(), "");
 6403   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6404   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6405   emit_int16(0x51, (0xC0 | encode));
 6406 }
 6407 
 6408 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6409   assert(VM_Version::supports_avx(), "");
 6410   InstructionMark im(this);
 6411   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6412   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6413   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6414   emit_int8(0x51);
 6415   emit_operand(dst, src);
 6416 }
 6417 
 6418 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6419   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6420   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6421   attributes.set_rex_vex_w_reverted();
 6422   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6423   emit_int16(0x54, (0xC0 | encode));
 6424 }
 6425 
 6426 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6427   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6428   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6429   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6430   emit_int16(0x54, (0xC0 | encode));
 6431 }
 6432 
 6433 void Assembler::andps(XMMRegister dst, Address src) {
 6434   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6435   InstructionMark im(this);
 6436   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6437   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6438   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6439   emit_int8(0x54);
 6440   emit_operand(dst, src);
 6441 }
 6442 
 6443 void Assembler::andpd(XMMRegister dst, Address src) {
 6444   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6445   InstructionMark im(this);
 6446   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6447   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6448   attributes.set_rex_vex_w_reverted();
 6449   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6450   emit_int8(0x54);
 6451   emit_operand(dst, src);
 6452 }
 6453 
 6454 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6455   assert(VM_Version::supports_avx(), "");
 6456   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6457   attributes.set_rex_vex_w_reverted();
 6458   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6459   emit_int16(0x54, (0xC0 | encode));
 6460 }
 6461 
 6462 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6463   assert(VM_Version::supports_avx(), "");
 6464   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6465   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6466   emit_int16(0x54, (0xC0 | encode));
 6467 }
 6468 
 6469 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6470   assert(VM_Version::supports_avx(), "");
 6471   InstructionMark im(this);
 6472   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6473   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6474   attributes.set_rex_vex_w_reverted();
 6475   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6476   emit_int8(0x54);
 6477   emit_operand(dst, src);
 6478 }
 6479 
 6480 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6481   assert(VM_Version::supports_avx(), "");
 6482   InstructionMark im(this);
 6483   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6484   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6485   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6486   emit_int8(0x54);
 6487   emit_operand(dst, src);
 6488 }
 6489 
 6490 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6491   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6492   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6493   attributes.set_rex_vex_w_reverted();
 6494   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6495   emit_int8(0x15);
 6496   emit_int8((0xC0 | encode));
 6497 }
 6498 
 6499 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6500   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6501   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6502   attributes.set_rex_vex_w_reverted();
 6503   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6504   emit_int16(0x14, (0xC0 | encode));
 6505 }
 6506 
 6507 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6508   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6509   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6510   attributes.set_rex_vex_w_reverted();
 6511   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6512   emit_int16(0x57, (0xC0 | encode));
 6513 }
 6514 
 6515 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6516   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6517   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6518   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6519   emit_int16(0x57, (0xC0 | encode));
 6520 }
 6521 
 6522 void Assembler::xorpd(XMMRegister dst, Address src) {
 6523   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6524   InstructionMark im(this);
 6525   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6526   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6527   attributes.set_rex_vex_w_reverted();
 6528   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6529   emit_int8(0x57);
 6530   emit_operand(dst, src);
 6531 }
 6532 
 6533 void Assembler::xorps(XMMRegister dst, Address src) {
 6534   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6535   InstructionMark im(this);
 6536   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6537   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6538   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6539   emit_int8(0x57);
 6540   emit_operand(dst, src);
 6541 }
 6542 
 6543 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6544   assert(VM_Version::supports_avx(), "");
 6545   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6546   attributes.set_rex_vex_w_reverted();
 6547   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6548   emit_int16(0x57, (0xC0 | encode));
 6549 }
 6550 
 6551 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6552   assert(VM_Version::supports_avx(), "");
 6553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6554   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6555   emit_int16(0x57, (0xC0 | encode));
 6556 }
 6557 
 6558 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6559   assert(VM_Version::supports_avx(), "");
 6560   InstructionMark im(this);
 6561   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6562   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6563   attributes.set_rex_vex_w_reverted();
 6564   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6565   emit_int8(0x57);
 6566   emit_operand(dst, src);
 6567 }
 6568 
 6569 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6570   assert(VM_Version::supports_avx(), "");
 6571   InstructionMark im(this);
 6572   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6573   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6574   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6575   emit_int8(0x57);
 6576   emit_operand(dst, src);
 6577 }
 6578 
 6579 // Integer vector arithmetic
 6580 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6581   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6582          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6583   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6584   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6585   emit_int16(0x01, (0xC0 | encode));
 6586 }
 6587 
 6588 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6589   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6590          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6591   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6592   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6593   emit_int16(0x02, (0xC0 | encode));
 6594 }
 6595 
 6596 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6597   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6598   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6599   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6600   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6601 }
 6602 
 6603 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6604   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6605   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6606   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6607   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6608 }
 6609 
 6610 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6611   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6612   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6613   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6614   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6615 }
 6616 
 6617 void Assembler::paddd(XMMRegister dst, Address src) {
 6618   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6619   InstructionMark im(this);
 6620   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6621   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6622   emit_int8((unsigned char)0xFE);
 6623   emit_operand(dst, src);
 6624 }
 6625 
 6626 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6627   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6628   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6629   attributes.set_rex_vex_w_reverted();
 6630   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6631   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6632 }
 6633 
 6634 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6635   assert(VM_Version::supports_sse3(), "");
 6636   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6637   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6638   emit_int16(0x01, (0xC0 | encode));
 6639 }
 6640 
 6641 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6642   assert(VM_Version::supports_sse3(), "");
 6643   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6644   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6645   emit_int16(0x02, (0xC0 | encode));
 6646 }
 6647 
 6648 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6649   assert(UseAVX > 0, "requires some form of AVX");
 6650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6651   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6652   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6653 }
 6654 
 6655 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6656   assert(UseAVX > 0, "requires some form of AVX");
 6657   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6658   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6659   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6660 }
 6661 
 6662 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6663   assert(UseAVX > 0, "requires some form of AVX");
 6664   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6665   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6666   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6667 }
 6668 
 6669 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6670   assert(UseAVX > 0, "requires some form of AVX");
 6671   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6672   attributes.set_rex_vex_w_reverted();
 6673   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6674   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6675 }
 6676 
 6677 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6678   assert(UseAVX > 0, "requires some form of AVX");
 6679   InstructionMark im(this);
 6680   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6681   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6682   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6683   emit_int8((unsigned char)0xFC);
 6684   emit_operand(dst, src);
 6685 }
 6686 
 6687 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6688   assert(UseAVX > 0, "requires some form of AVX");
 6689   InstructionMark im(this);
 6690   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6691   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6692   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6693   emit_int8((unsigned char)0xFD);
 6694   emit_operand(dst, src);
 6695 }
 6696 
 6697 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6698   assert(UseAVX > 0, "requires some form of AVX");
 6699   InstructionMark im(this);
 6700   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6701   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6702   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6703   emit_int8((unsigned char)0xFE);
 6704   emit_operand(dst, src);
 6705 }
 6706 
 6707 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6708   assert(UseAVX > 0, "requires some form of AVX");
 6709   InstructionMark im(this);
 6710   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6711   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6712   attributes.set_rex_vex_w_reverted();
 6713   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6714   emit_int8((unsigned char)0xD4);
 6715   emit_operand(dst, src);
 6716 }
 6717 
 6718 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6719   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6720   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6721   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6722   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6723 }
 6724 
 6725 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6726   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6727   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6728   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6729   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6730 }
 6731 
 6732 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6733   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6734   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6735   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6736 }
 6737 
 6738 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6740   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6741   attributes.set_rex_vex_w_reverted();
 6742   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6743   emit_int8((unsigned char)0xFB);
 6744   emit_int8((0xC0 | encode));
 6745 }
 6746 
 6747 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6748   assert(UseAVX > 0, "requires some form of AVX");
 6749   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6750   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6751   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6752 }
 6753 
 6754 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6755   assert(UseAVX > 0, "requires some form of AVX");
 6756   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6757   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6758   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6759 }
 6760 
 6761 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6762   assert(UseAVX > 0, "requires some form of AVX");
 6763   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6764   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6765   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6766 }
 6767 
 6768 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6769   assert(UseAVX > 0, "requires some form of AVX");
 6770   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6771   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6772   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6773 }
 6774 
 6775 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6776   assert(UseAVX > 0, "requires some form of AVX");
 6777   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6778   attributes.set_rex_vex_w_reverted();
 6779   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6780   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 6781 }
 6782 
 6783 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6784   assert(UseAVX > 0, "requires some form of AVX");
 6785   InstructionMark im(this);
 6786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6787   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6788   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6789   emit_int8((unsigned char)0xF8);
 6790   emit_operand(dst, src);
 6791 }
 6792 
 6793 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6794   assert(UseAVX > 0, "requires some form of AVX");
 6795   InstructionMark im(this);
 6796   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6797   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6798   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6799   emit_int8((unsigned char)0xF9);
 6800   emit_operand(dst, src);
 6801 }
 6802 
 6803 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6804   assert(UseAVX > 0, "requires some form of AVX");
 6805   InstructionMark im(this);
 6806   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6807   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6808   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6809   emit_int8((unsigned char)0xFA);
 6810   emit_operand(dst, src);
 6811 }
 6812 
 6813 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6814   assert(UseAVX > 0, "requires some form of AVX");
 6815   InstructionMark im(this);
 6816   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6817   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6818   attributes.set_rex_vex_w_reverted();
 6819   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6820   emit_int8((unsigned char)0xFB);
 6821   emit_operand(dst, src);
 6822 }
 6823 
 6824 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 6825   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6826   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6827   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6828   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6829 }
 6830 
 6831 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 6832   assert(VM_Version::supports_sse4_1(), "");
 6833   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6834   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6835   emit_int16(0x40, (0xC0 | encode));
 6836 }
 6837 
 6838 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 6839   assert(VM_Version::supports_sse2(), "");
 6840   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6841   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6842   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6843 }
 6844 
 6845 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6846   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 6847          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 6848          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 6849   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6850   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6851   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 6852 }
 6853 
 6854 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6855   assert(UseAVX > 0, "requires some form of AVX");
 6856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6857   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6858   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6859 }
 6860 
 6861 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6862   assert(UseAVX > 0, "requires some form of AVX");
 6863   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6864   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6865   emit_int16(0x40, (0xC0 | encode));
 6866 }
 6867 
 6868 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6869   assert(UseAVX > 2, "requires some form of EVEX");
 6870   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6871   attributes.set_is_evex_instruction();
 6872   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6873   emit_int16(0x40, (0xC0 | encode));
 6874 }
 6875 
 6876 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6877   assert(UseAVX > 0, "requires some form of AVX");
 6878   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6879   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6880   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6881 }
 6882 
 6883 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6884   assert(UseAVX > 0, "requires some form of AVX");
 6885   InstructionMark im(this);
 6886   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6887   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6888   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6889   emit_int8((unsigned char)0xD5);
 6890   emit_operand(dst, src);
 6891 }
 6892 
 6893 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6894   assert(UseAVX > 0, "requires some form of AVX");
 6895   InstructionMark im(this);
 6896   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6897   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6898   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6899   emit_int8(0x40);
 6900   emit_operand(dst, src);
 6901 }
 6902 
 6903 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6904   assert(UseAVX > 2, "requires some form of EVEX");
 6905   InstructionMark im(this);
 6906   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6907   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6908   attributes.set_is_evex_instruction();
 6909   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6910   emit_int8(0x40);
 6911   emit_operand(dst, src);
 6912 }
 6913 
 6914 // Min, max
 6915 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 6916   assert(VM_Version::supports_sse4_1(), "");
 6917   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6918   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6919   emit_int16(0x38, (0xC0 | encode));
 6920 }
 6921 
 6922 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6923   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6924         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6926   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6927   emit_int16(0x38, (0xC0 | encode));
 6928 }
 6929 
 6930 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 6931   assert(VM_Version::supports_sse2(), "");
 6932   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6933   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6934   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 6935 }
 6936 
 6937 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6938   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6939         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6940   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6941   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6942   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 6943 }
 6944 
 6945 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 6946   assert(VM_Version::supports_sse4_1(), "");
 6947   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6948   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6949   emit_int16(0x39, (0xC0 | encode));
 6950 }
 6951 
 6952 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6953   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6954         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 6955   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6956   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6957   emit_int16(0x39, (0xC0 | encode));
 6958 }
 6959 
 6960 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6961   assert(UseAVX > 2, "requires AVX512F");
 6962   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6963   attributes.set_is_evex_instruction();
 6964   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6965   emit_int16(0x39, (0xC0 | encode));
 6966 }
 6967 
 6968 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 6969   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6970   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6971   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6972   emit_int16(0x5D, (0xC0 | encode));
 6973 }
 6974 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6975   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6976   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6977   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6978   emit_int16(0x5D, (0xC0 | encode));
 6979 }
 6980 
 6981 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 6982   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6983   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6984   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6985   emit_int16(0x5D, (0xC0 | encode));
 6986 }
 6987 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6988   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6989   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6990   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6991   emit_int16(0x5D, (0xC0 | encode));
 6992 }
 6993 
 6994 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 6995   assert(VM_Version::supports_sse4_1(), "");
 6996   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6997   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6998   emit_int16(0x3C, (0xC0 | encode));
 6999 }
 7000 
 7001 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7002   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7003         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7005   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7006   emit_int16(0x3C, (0xC0 | encode));
 7007 }
 7008 
 7009 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7010   assert(VM_Version::supports_sse2(), "");
 7011   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7012   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7013   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7014 }
 7015 
 7016 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7017   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7018         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7019   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7020   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7021   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7022 }
 7023 
 7024 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7025   assert(VM_Version::supports_sse4_1(), "");
 7026   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7027   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7028   emit_int16(0x3D, (0xC0 | encode));
 7029 }
 7030 
 7031 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7032   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7033         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7034   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7035   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7036   emit_int16(0x3D, (0xC0 | encode));
 7037 }
 7038 
 7039 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7040   assert(UseAVX > 2, "requires AVX512F");
 7041   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7042   attributes.set_is_evex_instruction();
 7043   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7044   emit_int16(0x3D, (0xC0 | encode));
 7045 }
 7046 
 7047 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7048   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7049   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7050   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7051   emit_int16(0x5F, (0xC0 | encode));
 7052 }
 7053 
 7054 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7055   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7056   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7057   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7058   emit_int16(0x5F, (0xC0 | encode));
 7059 }
 7060 
 7061 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7062   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7063   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7064   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7065   emit_int16(0x5F, (0xC0 | encode));
 7066 }
 7067 
 7068 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7069   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7070   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7071   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7072   emit_int16(0x5F, (0xC0 | encode));
 7073 }
 7074 
 7075 // Shift packed integers left by specified number of bits.
 7076 void Assembler::psllw(XMMRegister dst, int shift) {
 7077   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7078   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7079   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7080   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7081   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7082 }
 7083 
 7084 void Assembler::pslld(XMMRegister dst, int shift) {
 7085   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7086   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7087   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7088   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7089   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7090 }
 7091 
 7092 void Assembler::psllq(XMMRegister dst, int shift) {
 7093   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7094   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7095   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7096   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7097   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7098 }
 7099 
 7100 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7101   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7102   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7103   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7104   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7105 }
 7106 
 7107 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7108   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7110   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7111   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7112 }
 7113 
 7114 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7115   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7116   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7117   attributes.set_rex_vex_w_reverted();
 7118   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7119   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7120 }
 7121 
 7122 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7123   assert(UseAVX > 0, "requires some form of AVX");
 7124   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7125   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7126   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7127   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7128 }
 7129 
 7130 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7131   assert(UseAVX > 0, "requires some form of AVX");
 7132   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7133   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7134   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7135   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7136   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7137 }
 7138 
 7139 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7140   assert(UseAVX > 0, "requires some form of AVX");
 7141   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7142   attributes.set_rex_vex_w_reverted();
 7143   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7144   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7145   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7146 }
 7147 
 7148 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7149   assert(UseAVX > 0, "requires some form of AVX");
 7150   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7151   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7152   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7153 }
 7154 
 7155 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7156   assert(UseAVX > 0, "requires some form of AVX");
 7157   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7158   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7159   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7160 }
 7161 
 7162 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7163   assert(UseAVX > 0, "requires some form of AVX");
 7164   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7165   attributes.set_rex_vex_w_reverted();
 7166   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7167   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7168 }
 7169 
 7170 // Shift packed integers logically right by specified number of bits.
 7171 void Assembler::psrlw(XMMRegister dst, int shift) {
 7172   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7173   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7174   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7175   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7176   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7177 }
 7178 
 7179 void Assembler::psrld(XMMRegister dst, int shift) {
 7180   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7181   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7182   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7183   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7184   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7185 }
 7186 
 7187 void Assembler::psrlq(XMMRegister dst, int shift) {
 7188   // Do not confuse it with psrldq SSE2 instruction which
 7189   // shifts 128 bit value in xmm register by number of bytes.
 7190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7191   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7192   attributes.set_rex_vex_w_reverted();
 7193   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7194   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7195   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7196 }
 7197 
 7198 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7199   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7200   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7201   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7202   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7203 }
 7204 
 7205 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7206   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7207   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7208   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7209   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7210 }
 7211 
 7212 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7213   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7214   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7215   attributes.set_rex_vex_w_reverted();
 7216   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7217   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7218 }
 7219 
 7220 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7221   assert(UseAVX > 0, "requires some form of AVX");
 7222   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7223   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7224   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7225   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7226 }
 7227 
 7228 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7229   assert(UseAVX > 0, "requires some form of AVX");
 7230   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7231   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7232   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7233   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7234 }
 7235 
 7236 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7237   assert(UseAVX > 0, "requires some form of AVX");
 7238   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7239   attributes.set_rex_vex_w_reverted();
 7240   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7241   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7242   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7243 }
 7244 
 7245 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7246   assert(UseAVX > 0, "requires some form of AVX");
 7247   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7248   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7249   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7250 }
 7251 
 7252 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7253   assert(UseAVX > 0, "requires some form of AVX");
 7254   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7255   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7256   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7257 }
 7258 
 7259 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7260   assert(UseAVX > 0, "requires some form of AVX");
 7261   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7262   attributes.set_rex_vex_w_reverted();
 7263   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7264   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7265 }
 7266 
 7267 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7268   assert(VM_Version::supports_avx512bw(), "");
 7269   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7270   attributes.set_is_evex_instruction();
 7271   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7272   emit_int16(0x10, (0xC0 | encode));
 7273 }
 7274 
 7275 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7276   assert(VM_Version::supports_avx512bw(), "");
 7277   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7278   attributes.set_is_evex_instruction();
 7279   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7280   emit_int16(0x12, (0xC0 | encode));
 7281 }
 7282 
 7283 // Shift packed integers arithmetically right by specified number of bits.
 7284 void Assembler::psraw(XMMRegister dst, int shift) {
 7285   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7287   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7288   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7289   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7290 }
 7291 
 7292 void Assembler::psrad(XMMRegister dst, int shift) {
 7293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7294   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7295   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7296   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7297   emit_int8(0x72);
 7298   emit_int8((0xC0 | encode));
 7299   emit_int8(shift & 0xFF);
 7300 }
 7301 
 7302 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7303   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7305   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7306   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7307 }
 7308 
 7309 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7310   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7311   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7312   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7313   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7314 }
 7315 
 7316 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7317   assert(UseAVX > 0, "requires some form of AVX");
 7318   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7319   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7320   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7321   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7322 }
 7323 
 7324 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7325   assert(UseAVX > 0, "requires some form of AVX");
 7326   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7327   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7328   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7329   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7330 }
 7331 
 7332 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7333   assert(UseAVX > 0, "requires some form of AVX");
 7334   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7335   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7336   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7337 }
 7338 
 7339 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7340   assert(UseAVX > 0, "requires some form of AVX");
 7341   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7342   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7343   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7344 }
 7345 
 7346 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7347   assert(UseAVX > 2, "requires AVX512");
 7348   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7349   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7350   attributes.set_is_evex_instruction();
 7351   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7352   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7353 }
 7354 
 7355 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7356   assert(UseAVX > 2, "requires AVX512");
 7357   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7358   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7359   attributes.set_is_evex_instruction();
 7360   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7361   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7362 }
 7363 
 7364 // logical operations packed integers
 7365 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7368   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7369   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7370 }
 7371 
 7372 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7373   assert(UseAVX > 0, "requires some form of AVX");
 7374   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7375   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7376   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7377 }
 7378 
 7379 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7380   assert(UseAVX > 0, "requires some form of AVX");
 7381   InstructionMark im(this);
 7382   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7383   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7384   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7385   emit_int8((unsigned char)0xDB);
 7386   emit_operand(dst, src);
 7387 }
 7388 
 7389 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7390   assert(VM_Version::supports_evex(), "");
 7391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7392   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7393   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7394 }
 7395 
 7396 //Variable Shift packed integers logically left.
 7397 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7398   assert(UseAVX > 1, "requires AVX2");
 7399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7400   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7401   emit_int16(0x47, (0xC0 | encode));
 7402 }
 7403 
 7404 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7405   assert(UseAVX > 1, "requires AVX2");
 7406   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7407   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7408   emit_int16(0x47, (0xC0 | encode));
 7409 }
 7410 
 7411 //Variable Shift packed integers logically right.
 7412 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7413   assert(UseAVX > 1, "requires AVX2");
 7414   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7415   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7416   emit_int16(0x45, (0xC0 | encode));
 7417 }
 7418 
 7419 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7420   assert(UseAVX > 1, "requires AVX2");
 7421   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7422   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7423   emit_int16(0x45, (0xC0 | encode));
 7424 }
 7425 
 7426 //Variable right Shift arithmetic packed integers .
 7427 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7428   assert(UseAVX > 1, "requires AVX2");
 7429   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7430   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7431   emit_int16(0x46, (0xC0 | encode));
 7432 }
 7433 
 7434 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7435   assert(VM_Version::supports_avx512bw(), "");
 7436   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7437   attributes.set_is_evex_instruction();
 7438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7439   emit_int16(0x11, (0xC0 | encode));
 7440 }
 7441 
 7442 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7443   assert(UseAVX > 2, "requires AVX512");
 7444   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7445   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7446   attributes.set_is_evex_instruction();
 7447   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7448   emit_int16(0x46, (0xC0 | encode));
 7449 }
 7450 
 7451 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7452   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7453   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7454   attributes.set_is_evex_instruction();
 7455   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7456   emit_int16(0x71, (0xC0 | encode));
 7457 }
 7458 
 7459 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7460   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7461   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7462   attributes.set_is_evex_instruction();
 7463   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7464   emit_int16(0x73, (0xC0 | encode));
 7465 }
 7466 
 7467 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7469   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7470   attributes.set_rex_vex_w_reverted();
 7471   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7472   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7473 }
 7474 
 7475 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7476   assert(UseAVX > 0, "requires some form of AVX");
 7477   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7478   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7479   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7480 }
 7481 
 7482 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7483   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7484   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7485   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7486   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7487 }
 7488 
 7489 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7490   assert(UseAVX > 0, "requires some form of AVX");
 7491   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7492   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7493   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7494 }
 7495 
 7496 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7497   assert(UseAVX > 0, "requires some form of AVX");
 7498   InstructionMark im(this);
 7499   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7500   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7501   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7502   emit_int8((unsigned char)0xEB);
 7503   emit_operand(dst, src);
 7504 }
 7505 
 7506 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7507   assert(VM_Version::supports_evex(), "");
 7508   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7509   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7510   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7511 }
 7512 
 7513 
 7514 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7515   assert(VM_Version::supports_evex(), "");
 7516   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7517   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7518   attributes.set_is_evex_instruction();
 7519   attributes.set_embedded_opmask_register_specifier(mask);
 7520   if (merge) {
 7521     attributes.reset_is_clear_context();
 7522   }
 7523   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7524   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7525 }
 7526 
 7527 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7528   assert(VM_Version::supports_evex(), "");
 7529   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7530   InstructionMark im(this);
 7531   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7532   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7533   attributes.set_is_evex_instruction();
 7534   attributes.set_embedded_opmask_register_specifier(mask);
 7535   if (merge) {
 7536     attributes.reset_is_clear_context();
 7537   }
 7538   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7539   emit_int8((unsigned char)0xEB);
 7540   emit_operand(dst, src);
 7541 }
 7542 
 7543 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7545   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7546   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7547   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7548 }
 7549 
 7550 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7551   assert(UseAVX > 0, "requires some form of AVX");
 7552   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7553   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7554   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7555 }
 7556 
 7557 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7558   assert(UseAVX > 0, "requires some form of AVX");
 7559   InstructionMark im(this);
 7560   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7561   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7562   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7563   emit_int8((unsigned char)0xEF);
 7564   emit_operand(dst, src);
 7565 }
 7566 
 7567 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7568   assert(UseAVX > 2, "requires some form of EVEX");
 7569   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7570   attributes.set_rex_vex_w_reverted();
 7571   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7572   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7573 }
 7574 
 7575 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7576   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7577   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7578   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7579   attributes.set_is_evex_instruction();
 7580   attributes.set_embedded_opmask_register_specifier(mask);
 7581   if (merge) {
 7582     attributes.reset_is_clear_context();
 7583   }
 7584   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7585   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7586 }
 7587 
 7588 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7589   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7590   InstructionMark im(this);
 7591   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7592   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7593   attributes.set_is_evex_instruction();
 7594   attributes.set_embedded_opmask_register_specifier(mask);
 7595   if (merge) {
 7596     attributes.reset_is_clear_context();
 7597   }
 7598   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7599   emit_int8((unsigned char)0xEF);
 7600   emit_operand(dst, src);
 7601 }
 7602 
 7603 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7604   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 7605   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7606   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7607   attributes.set_is_evex_instruction();
 7608   attributes.set_embedded_opmask_register_specifier(mask);
 7609   if (merge) {
 7610     attributes.reset_is_clear_context();
 7611   }
 7612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7613   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7614 }
 7615 
 7616 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7617   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7618   InstructionMark im(this);
 7619   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7620   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7621   attributes.set_is_evex_instruction();
 7622   attributes.set_embedded_opmask_register_specifier(mask);
 7623   if (merge) {
 7624     attributes.reset_is_clear_context();
 7625   }
 7626   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7627   emit_int8((unsigned char)0xEF);
 7628   emit_operand(dst, src);
 7629 }
 7630 
 7631 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7632   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7633   InstructionMark im(this);
 7634   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7635   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7636   attributes.set_is_evex_instruction();
 7637   attributes.set_embedded_opmask_register_specifier(mask);
 7638   if (merge) {
 7639     attributes.reset_is_clear_context();
 7640   }
 7641   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7642   emit_int8((unsigned char)0xDB);
 7643   emit_operand(dst, src);
 7644 }
 7645 
 7646 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7647   assert(VM_Version::supports_evex(), "");
 7648   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7649   attributes.set_is_evex_instruction();
 7650   attributes.set_embedded_opmask_register_specifier(mask);
 7651   if (merge) {
 7652     attributes.reset_is_clear_context();
 7653   }
 7654   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7655   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7656 }
 7657 
 7658 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7659   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7660   InstructionMark im(this);
 7661   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7662   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7663   attributes.set_is_evex_instruction();
 7664   attributes.set_embedded_opmask_register_specifier(mask);
 7665   if (merge) {
 7666     attributes.reset_is_clear_context();
 7667   }
 7668   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7669   emit_int8((unsigned char)0xDB);
 7670   emit_operand(dst, src);
 7671 }
 7672 
 7673 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7674   assert(VM_Version::supports_evex(), "");
 7675   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7676   attributes.set_is_evex_instruction();
 7677   attributes.set_embedded_opmask_register_specifier(mask);
 7678   if (merge) {
 7679     attributes.reset_is_clear_context();
 7680   }
 7681   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7682   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7683 }
 7684 
 7685 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7686   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 7687   InstructionMark im(this);
 7688   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 7689   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 7690   attributes.set_is_evex_instruction();
 7691   attributes.set_embedded_opmask_register_specifier(mask);
 7692   if (merge) {
 7693     attributes.reset_is_clear_context();
 7694   }
 7695   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7696   emit_int8((unsigned char)0xEB);
 7697   emit_operand(dst, src);
 7698 }
 7699 
 7700 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7701   assert(VM_Version::supports_evex(), "requires EVEX support");
 7702   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7703   attributes.set_is_evex_instruction();
 7704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7705   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7706 }
 7707 
 7708 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7709   assert(VM_Version::supports_evex(), "requires EVEX support");
 7710   assert(dst != xnoreg, "sanity");
 7711   InstructionMark im(this);
 7712   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7713   attributes.set_is_evex_instruction();
 7714   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7715   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7716   emit_int8((unsigned char)0xEF);
 7717   emit_operand(dst, src);
 7718 }
 7719 
 7720 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7721   assert(VM_Version::supports_evex(), "requires EVEX support");
 7722   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7723   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7724   attributes.set_is_evex_instruction();
 7725   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7726   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7727 }
 7728 
 7729 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7730   assert(VM_Version::supports_evex(), "requires EVEX support");
 7731   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7732   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7733   attributes.set_is_evex_instruction();
 7734   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7735   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7736 }
 7737 
 7738 // Register is a class, but it would be assigned numerical value.
 7739 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7740 PRAGMA_DIAG_PUSH
 7741 PRAGMA_NONNULL_IGNORED
 7742 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7743   assert(VM_Version::supports_evex(), "requires EVEX support");
 7744   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7745   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7746   attributes.set_is_evex_instruction();
 7747   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7748   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7749 }
 7750 
 7751 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7752   assert(VM_Version::supports_evex(), "requires EVEX support");
 7753   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7754   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7755   attributes.set_is_evex_instruction();
 7756   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7757   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7758 }
 7759 PRAGMA_DIAG_POP
 7760 
 7761 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7762   assert(VM_Version::supports_evex(), "requires EVEX support");
 7763   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7764   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7765   attributes.set_is_evex_instruction();
 7766   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7767   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7768 }
 7769 
 7770 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7771   assert(VM_Version::supports_evex(), "requires EVEX support");
 7772   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7773   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7774   attributes.set_is_evex_instruction();
 7775   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7776   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7777 }
 7778 
 7779 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7780   assert(VM_Version::supports_evex(), "requires EVEX support");
 7781   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7782   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7783   attributes.set_is_evex_instruction();
 7784   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7785   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7786 }
 7787 
 7788 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7789   assert(VM_Version::supports_evex(), "requires EVEX support");
 7790   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7791   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7792   attributes.set_is_evex_instruction();
 7793   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7794   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7795 }
 7796 
 7797 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7798   assert(VM_Version::supports_evex(), "requires EVEX support");
 7799   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7800   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7801   attributes.set_is_evex_instruction();
 7802   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7803   emit_int8(0x25);
 7804   emit_int8((unsigned char)(0xC0 | encode));
 7805   emit_int8(imm8);
 7806 }
 7807 
 7808 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 7809   assert(VM_Version::supports_evex(), "requires EVEX support");
 7810   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7811   assert(dst != xnoreg, "sanity");
 7812   InstructionMark im(this);
 7813   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7814   attributes.set_is_evex_instruction();
 7815   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7816   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7817   emit_int8(0x25);
 7818   emit_operand(dst, src3);
 7819   emit_int8(imm8);
 7820 }
 7821 
 7822 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7823   assert(VM_Version::supports_evex(), "requires EVEX support");
 7824   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7825   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7826   attributes.set_is_evex_instruction();
 7827   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7828   emit_int8(0x25);
 7829   emit_int8((unsigned char)(0xC0 | encode));
 7830   emit_int8(imm8);
 7831 }
 7832 
 7833 // vinserti forms
 7834 
 7835 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7836   assert(VM_Version::supports_avx2(), "");
 7837   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7838   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7839   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7840   // last byte:
 7841   // 0x00 - insert into lower 128 bits
 7842   // 0x01 - insert into upper 128 bits
 7843   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 7844 }
 7845 
 7846 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7847   assert(VM_Version::supports_avx2(), "");
 7848   assert(dst != xnoreg, "sanity");
 7849   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7850   InstructionMark im(this);
 7851   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7852   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7853   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7854   emit_int8(0x38);
 7855   emit_operand(dst, src);
 7856   // 0x00 - insert into lower 128 bits
 7857   // 0x01 - insert into upper 128 bits
 7858   emit_int8(imm8 & 0x01);
 7859 }
 7860 
 7861 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7862   assert(VM_Version::supports_evex(), "");
 7863   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7864   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7865   attributes.set_is_evex_instruction();
 7866   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7867   // imm8:
 7868   // 0x00 - insert into q0 128 bits (0..127)
 7869   // 0x01 - insert into q1 128 bits (128..255)
 7870   // 0x02 - insert into q2 128 bits (256..383)
 7871   // 0x03 - insert into q3 128 bits (384..511)
 7872   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 7873 }
 7874 
 7875 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7876   assert(VM_Version::supports_avx(), "");
 7877   assert(dst != xnoreg, "sanity");
 7878   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7879   InstructionMark im(this);
 7880   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7881   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7882   attributes.set_is_evex_instruction();
 7883   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7884   emit_int8(0x18);
 7885   emit_operand(dst, src);
 7886   // 0x00 - insert into q0 128 bits (0..127)
 7887   // 0x01 - insert into q1 128 bits (128..255)
 7888   // 0x02 - insert into q2 128 bits (256..383)
 7889   // 0x03 - insert into q3 128 bits (384..511)
 7890   emit_int8(imm8 & 0x03);
 7891 }
 7892 
 7893 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7894   assert(VM_Version::supports_evex(), "");
 7895   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7896   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7897   attributes.set_is_evex_instruction();
 7898   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7899   //imm8:
 7900   // 0x00 - insert into lower 256 bits
 7901   // 0x01 - insert into upper 256 bits
 7902   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 7903 }
 7904 
 7905 
 7906 // vinsertf forms
 7907 
 7908 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7909   assert(VM_Version::supports_avx(), "");
 7910   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7911   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7912   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7913   // imm8:
 7914   // 0x00 - insert into lower 128 bits
 7915   // 0x01 - insert into upper 128 bits
 7916   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 7917 }
 7918 
 7919 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7920   assert(VM_Version::supports_avx(), "");
 7921   assert(dst != xnoreg, "sanity");
 7922   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7923   InstructionMark im(this);
 7924   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7925   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7926   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7927   emit_int8(0x18);
 7928   emit_operand(dst, src);
 7929   // 0x00 - insert into lower 128 bits
 7930   // 0x01 - insert into upper 128 bits
 7931   emit_int8(imm8 & 0x01);
 7932 }
 7933 
 7934 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7935   assert(VM_Version::supports_avx2(), "");
 7936   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7937   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7938   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7939   // imm8:
 7940   // 0x00 - insert into q0 128 bits (0..127)
 7941   // 0x01 - insert into q1 128 bits (128..255)
 7942   // 0x02 - insert into q0 128 bits (256..383)
 7943   // 0x03 - insert into q1 128 bits (384..512)
 7944   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 7945 }
 7946 
 7947 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7948   assert(VM_Version::supports_avx(), "");
 7949   assert(dst != xnoreg, "sanity");
 7950   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7951   InstructionMark im(this);
 7952   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7953   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7954   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7955   emit_int8(0x18);
 7956   emit_operand(dst, src);
 7957   // 0x00 - insert into q0 128 bits (0..127)
 7958   // 0x01 - insert into q1 128 bits (128..255)
 7959   // 0x02 - insert into q0 128 bits (256..383)
 7960   // 0x03 - insert into q1 128 bits (384..512)
 7961   emit_int8(imm8 & 0x03);
 7962 }
 7963 
 7964 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7965   assert(VM_Version::supports_evex(), "");
 7966   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7967   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7968   attributes.set_is_evex_instruction();
 7969   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7970   // imm8:
 7971   // 0x00 - insert into lower 256 bits
 7972   // 0x01 - insert into upper 256 bits
 7973   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 7974 }
 7975 
 7976 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7977   assert(VM_Version::supports_evex(), "");
 7978   assert(dst != xnoreg, "sanity");
 7979   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7980   InstructionMark im(this);
 7981   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7982   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 7983   attributes.set_is_evex_instruction();
 7984   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7985   emit_int8(0x1A);
 7986   emit_operand(dst, src);
 7987   // 0x00 - insert into lower 256 bits
 7988   // 0x01 - insert into upper 256 bits
 7989   emit_int8(imm8 & 0x01);
 7990 }
 7991 
 7992 
 7993 // vextracti forms
 7994 
 7995 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7996   assert(VM_Version::supports_avx2(), "");
 7997   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7998   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7999   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8000   // imm8:
 8001   // 0x00 - extract from lower 128 bits
 8002   // 0x01 - extract from upper 128 bits
 8003   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8004 }
 8005 
 8006 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8007   assert(VM_Version::supports_avx2(), "");
 8008   assert(src != xnoreg, "sanity");
 8009   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8010   InstructionMark im(this);
 8011   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8012   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8013   attributes.reset_is_clear_context();
 8014   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8015   emit_int8(0x39);
 8016   emit_operand(src, dst);
 8017   // 0x00 - extract from lower 128 bits
 8018   // 0x01 - extract from upper 128 bits
 8019   emit_int8(imm8 & 0x01);
 8020 }
 8021 
 8022 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8023   assert(VM_Version::supports_evex(), "");
 8024   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8025   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8026   attributes.set_is_evex_instruction();
 8027   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8028   // imm8:
 8029   // 0x00 - extract from bits 127:0
 8030   // 0x01 - extract from bits 255:128
 8031   // 0x02 - extract from bits 383:256
 8032   // 0x03 - extract from bits 511:384
 8033   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8034 }
 8035 
 8036 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8037   assert(VM_Version::supports_evex(), "");
 8038   assert(src != xnoreg, "sanity");
 8039   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8040   InstructionMark im(this);
 8041   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8042   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8043   attributes.reset_is_clear_context();
 8044   attributes.set_is_evex_instruction();
 8045   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8046   emit_int8(0x39);
 8047   emit_operand(src, dst);
 8048   // 0x00 - extract from bits 127:0
 8049   // 0x01 - extract from bits 255:128
 8050   // 0x02 - extract from bits 383:256
 8051   // 0x03 - extract from bits 511:384
 8052   emit_int8(imm8 & 0x03);
 8053 }
 8054 
 8055 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8056   assert(VM_Version::supports_avx512dq(), "");
 8057   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8058   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8059   attributes.set_is_evex_instruction();
 8060   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8061   // imm8:
 8062   // 0x00 - extract from bits 127:0
 8063   // 0x01 - extract from bits 255:128
 8064   // 0x02 - extract from bits 383:256
 8065   // 0x03 - extract from bits 511:384
 8066   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8067 }
 8068 
 8069 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8070   assert(VM_Version::supports_evex(), "");
 8071   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8072   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8073   attributes.set_is_evex_instruction();
 8074   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8075   // imm8:
 8076   // 0x00 - extract from lower 256 bits
 8077   // 0x01 - extract from upper 256 bits
 8078   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8079 }
 8080 
 8081 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8082   assert(VM_Version::supports_evex(), "");
 8083   assert(src != xnoreg, "sanity");
 8084   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8085   InstructionMark im(this);
 8086   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8087   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8088   attributes.reset_is_clear_context();
 8089   attributes.set_is_evex_instruction();
 8090   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8091   emit_int8(0x38);
 8092   emit_operand(src, dst);
 8093   // 0x00 - extract from lower 256 bits
 8094   // 0x01 - extract from upper 256 bits
 8095   emit_int8(imm8 & 0x01);
 8096 }
 8097 // vextractf forms
 8098 
 8099 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8100   assert(VM_Version::supports_avx(), "");
 8101   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8102   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8103   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8104   // imm8:
 8105   // 0x00 - extract from lower 128 bits
 8106   // 0x01 - extract from upper 128 bits
 8107   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8108 }
 8109 
 8110 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8111   assert(VM_Version::supports_avx(), "");
 8112   assert(src != xnoreg, "sanity");
 8113   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8114   InstructionMark im(this);
 8115   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8116   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8117   attributes.reset_is_clear_context();
 8118   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8119   emit_int8(0x19);
 8120   emit_operand(src, dst);
 8121   // 0x00 - extract from lower 128 bits
 8122   // 0x01 - extract from upper 128 bits
 8123   emit_int8(imm8 & 0x01);
 8124 }
 8125 
 8126 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8127   assert(VM_Version::supports_evex(), "");
 8128   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8129   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8130   attributes.set_is_evex_instruction();
 8131   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8132   // imm8:
 8133   // 0x00 - extract from bits 127:0
 8134   // 0x01 - extract from bits 255:128
 8135   // 0x02 - extract from bits 383:256
 8136   // 0x03 - extract from bits 511:384
 8137   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8138 }
 8139 
 8140 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8141   assert(VM_Version::supports_evex(), "");
 8142   assert(src != xnoreg, "sanity");
 8143   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8144   InstructionMark im(this);
 8145   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8146   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8147   attributes.reset_is_clear_context();
 8148   attributes.set_is_evex_instruction();
 8149   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8150   emit_int8(0x19);
 8151   emit_operand(src, dst);
 8152   // 0x00 - extract from bits 127:0
 8153   // 0x01 - extract from bits 255:128
 8154   // 0x02 - extract from bits 383:256
 8155   // 0x03 - extract from bits 511:384
 8156   emit_int8(imm8 & 0x03);
 8157 }
 8158 
 8159 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8160   assert(VM_Version::supports_avx512dq(), "");
 8161   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8162   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8163   attributes.set_is_evex_instruction();
 8164   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8165   // imm8:
 8166   // 0x00 - extract from bits 127:0
 8167   // 0x01 - extract from bits 255:128
 8168   // 0x02 - extract from bits 383:256
 8169   // 0x03 - extract from bits 511:384
 8170   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8171 }
 8172 
 8173 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8174   assert(VM_Version::supports_evex(), "");
 8175   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8176   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8177   attributes.set_is_evex_instruction();
 8178   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8179   // imm8:
 8180   // 0x00 - extract from lower 256 bits
 8181   // 0x01 - extract from upper 256 bits
 8182   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8183 }
 8184 
 8185 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8186   assert(VM_Version::supports_evex(), "");
 8187   assert(src != xnoreg, "sanity");
 8188   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8189   InstructionMark im(this);
 8190   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8191   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8192   attributes.reset_is_clear_context();
 8193   attributes.set_is_evex_instruction();
 8194   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8195   emit_int8(0x1B);
 8196   emit_operand(src, dst);
 8197   // 0x00 - extract from lower 256 bits
 8198   // 0x01 - extract from upper 256 bits
 8199   emit_int8(imm8 & 0x01);
 8200 }
 8201 
 8202 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8203 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8204   assert(VM_Version::supports_avx2(), "");
 8205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8206   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8207   emit_int16(0x78, (0xC0 | encode));
 8208 }
 8209 
 8210 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8211   assert(VM_Version::supports_avx2(), "");
 8212   assert(dst != xnoreg, "sanity");
 8213   InstructionMark im(this);
 8214   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8215   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8216   // swap src<->dst for encoding
 8217   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8218   emit_int8(0x78);
 8219   emit_operand(dst, src);
 8220 }
 8221 
 8222 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8223 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8224   assert(VM_Version::supports_avx2(), "");
 8225   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8226   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8227   emit_int16(0x79, (0xC0 | encode));
 8228 }
 8229 
 8230 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8231   assert(VM_Version::supports_avx2(), "");
 8232   assert(dst != xnoreg, "sanity");
 8233   InstructionMark im(this);
 8234   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8235   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8236   // swap src<->dst for encoding
 8237   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8238   emit_int8(0x79);
 8239   emit_operand(dst, src);
 8240 }
 8241 
 8242 // xmm/mem sourced byte/word/dword/qword replicate
 8243 
 8244 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8245   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8246   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8247   attributes.set_is_evex_instruction();
 8248   attributes.set_embedded_opmask_register_specifier(mask);
 8249   if (merge) {
 8250     attributes.reset_is_clear_context();
 8251   }
 8252   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8253   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 8254 }
 8255 
 8256 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8257   InstructionMark im(this);
 8258   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8259   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8260   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8261   attributes.set_is_evex_instruction();
 8262   attributes.set_embedded_opmask_register_specifier(mask);
 8263   if (merge) {
 8264     attributes.reset_is_clear_context();
 8265   }
 8266   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8267   emit_int8((unsigned char)0xFC);
 8268   emit_operand(dst, src);
 8269 }
 8270 
 8271 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8272   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8273   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8274   attributes.set_is_evex_instruction();
 8275   attributes.set_embedded_opmask_register_specifier(mask);
 8276   if (merge) {
 8277     attributes.reset_is_clear_context();
 8278   }
 8279   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8280   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 8281 }
 8282 
 8283 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8284   InstructionMark im(this);
 8285   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8286   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8287   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8288   attributes.set_is_evex_instruction();
 8289   attributes.set_embedded_opmask_register_specifier(mask);
 8290   if (merge) {
 8291     attributes.reset_is_clear_context();
 8292   }
 8293   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8294   emit_int8((unsigned char)0xFD);
 8295   emit_operand(dst, src);
 8296 }
 8297 
 8298 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8299   assert(VM_Version::supports_evex(), "");
 8300   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8302   attributes.set_is_evex_instruction();
 8303   attributes.set_embedded_opmask_register_specifier(mask);
 8304   if (merge) {
 8305     attributes.reset_is_clear_context();
 8306   }
 8307   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8308   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 8309 }
 8310 
 8311 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8312   InstructionMark im(this);
 8313   assert(VM_Version::supports_evex(), "");
 8314   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8315   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8316   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8317   attributes.set_is_evex_instruction();
 8318   attributes.set_embedded_opmask_register_specifier(mask);
 8319   if (merge) {
 8320     attributes.reset_is_clear_context();
 8321   }
 8322   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8323   emit_int8((unsigned char)0xFE);
 8324   emit_operand(dst, src);
 8325 }
 8326 
 8327 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8328   assert(VM_Version::supports_evex(), "");
 8329   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8330   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8331   attributes.set_is_evex_instruction();
 8332   attributes.set_embedded_opmask_register_specifier(mask);
 8333   if (merge) {
 8334     attributes.reset_is_clear_context();
 8335   }
 8336   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8337   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 8338 }
 8339 
 8340 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8341   InstructionMark im(this);
 8342   assert(VM_Version::supports_evex(), "");
 8343   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8344   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8345   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8346   attributes.set_is_evex_instruction();
 8347   attributes.set_embedded_opmask_register_specifier(mask);
 8348   if (merge) {
 8349     attributes.reset_is_clear_context();
 8350   }
 8351   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8352   emit_int8((unsigned char)0xD4);
 8353   emit_operand(dst, src);
 8354 }
 8355 
 8356 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8357   assert(VM_Version::supports_evex(), "");
 8358   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8359   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8360   attributes.set_is_evex_instruction();
 8361   attributes.set_embedded_opmask_register_specifier(mask);
 8362   if (merge) {
 8363     attributes.reset_is_clear_context();
 8364   }
 8365   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8366   emit_int16(0x58, (0xC0 | encode));
 8367 }
 8368 
 8369 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8370   InstructionMark im(this);
 8371   assert(VM_Version::supports_evex(), "");
 8372   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8373   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8374   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8375   attributes.set_is_evex_instruction();
 8376   attributes.set_embedded_opmask_register_specifier(mask);
 8377   if (merge) {
 8378     attributes.reset_is_clear_context();
 8379   }
 8380   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8381   emit_int8(0x58);
 8382   emit_operand(dst, src);
 8383 }
 8384 
 8385 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8386   assert(VM_Version::supports_evex(), "");
 8387   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8388   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8389   attributes.set_is_evex_instruction();
 8390   attributes.set_embedded_opmask_register_specifier(mask);
 8391   if (merge) {
 8392     attributes.reset_is_clear_context();
 8393   }
 8394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8395   emit_int16(0x58, (0xC0 | encode));
 8396 }
 8397 
 8398 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8399   InstructionMark im(this);
 8400   assert(VM_Version::supports_evex(), "");
 8401   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8402   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8403   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8404   attributes.set_is_evex_instruction();
 8405   attributes.set_embedded_opmask_register_specifier(mask);
 8406   if (merge) {
 8407     attributes.reset_is_clear_context();
 8408   }
 8409   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8410   emit_int8(0x58);
 8411   emit_operand(dst, src);
 8412 }
 8413 
 8414 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8415   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8416   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8417   attributes.set_is_evex_instruction();
 8418   attributes.set_embedded_opmask_register_specifier(mask);
 8419   if (merge) {
 8420     attributes.reset_is_clear_context();
 8421   }
 8422   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8423   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 8424 }
 8425 
 8426 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8427   InstructionMark im(this);
 8428   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8429   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8430   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8431   attributes.set_is_evex_instruction();
 8432   attributes.set_embedded_opmask_register_specifier(mask);
 8433   if (merge) {
 8434     attributes.reset_is_clear_context();
 8435   }
 8436   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8437   emit_int8((unsigned char)0xF8);
 8438   emit_operand(dst, src);
 8439 }
 8440 
 8441 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8442   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8443   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8444   attributes.set_is_evex_instruction();
 8445   attributes.set_embedded_opmask_register_specifier(mask);
 8446   if (merge) {
 8447     attributes.reset_is_clear_context();
 8448   }
 8449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8450   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 8451 }
 8452 
 8453 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8454   InstructionMark im(this);
 8455   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8456   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8457   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8458   attributes.set_is_evex_instruction();
 8459   attributes.set_embedded_opmask_register_specifier(mask);
 8460   if (merge) {
 8461     attributes.reset_is_clear_context();
 8462   }
 8463   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8464   emit_int8((unsigned char)0xF9);
 8465   emit_operand(dst, src);
 8466 }
 8467 
 8468 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8469   assert(VM_Version::supports_evex(), "");
 8470   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8471   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8472   attributes.set_is_evex_instruction();
 8473   attributes.set_embedded_opmask_register_specifier(mask);
 8474   if (merge) {
 8475     attributes.reset_is_clear_context();
 8476   }
 8477   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8478   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 8479 }
 8480 
 8481 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8482   InstructionMark im(this);
 8483   assert(VM_Version::supports_evex(), "");
 8484   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8485   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8486   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8487   attributes.set_is_evex_instruction();
 8488   attributes.set_embedded_opmask_register_specifier(mask);
 8489   if (merge) {
 8490     attributes.reset_is_clear_context();
 8491   }
 8492   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8493   emit_int8((unsigned char)0xFA);
 8494   emit_operand(dst, src);
 8495 }
 8496 
 8497 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8498   assert(VM_Version::supports_evex(), "");
 8499   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8500   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8501   attributes.set_is_evex_instruction();
 8502   attributes.set_embedded_opmask_register_specifier(mask);
 8503   if (merge) {
 8504     attributes.reset_is_clear_context();
 8505   }
 8506   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8507   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 8508 }
 8509 
 8510 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8511   InstructionMark im(this);
 8512   assert(VM_Version::supports_evex(), "");
 8513   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8514   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8515   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8516   attributes.set_is_evex_instruction();
 8517   attributes.set_embedded_opmask_register_specifier(mask);
 8518   if (merge) {
 8519     attributes.reset_is_clear_context();
 8520   }
 8521   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8522   emit_int8((unsigned char)0xFB);
 8523   emit_operand(dst, src);
 8524 }
 8525 
 8526 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8527   assert(VM_Version::supports_evex(), "");
 8528   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8529   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8530   attributes.set_is_evex_instruction();
 8531   attributes.set_embedded_opmask_register_specifier(mask);
 8532   if (merge) {
 8533     attributes.reset_is_clear_context();
 8534   }
 8535   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8536   emit_int16(0x5C, (0xC0 | encode));
 8537 }
 8538 
 8539 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8540   InstructionMark im(this);
 8541   assert(VM_Version::supports_evex(), "");
 8542   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8543   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8544   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8545   attributes.set_is_evex_instruction();
 8546   attributes.set_embedded_opmask_register_specifier(mask);
 8547   if (merge) {
 8548     attributes.reset_is_clear_context();
 8549   }
 8550   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8551   emit_int8(0x5C);
 8552   emit_operand(dst, src);
 8553 }
 8554 
 8555 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8556   assert(VM_Version::supports_evex(), "");
 8557   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8558   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8559   attributes.set_is_evex_instruction();
 8560   attributes.set_embedded_opmask_register_specifier(mask);
 8561   if (merge) {
 8562     attributes.reset_is_clear_context();
 8563   }
 8564   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8565   emit_int16(0x5C, (0xC0 | encode));
 8566 }
 8567 
 8568 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8569   InstructionMark im(this);
 8570   assert(VM_Version::supports_evex(), "");
 8571   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8572   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8573   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8574   attributes.set_is_evex_instruction();
 8575   attributes.set_embedded_opmask_register_specifier(mask);
 8576   if (merge) {
 8577     attributes.reset_is_clear_context();
 8578   }
 8579   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8580   emit_int8(0x5C);
 8581   emit_operand(dst, src);
 8582 }
 8583 
 8584 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8585   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8586   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8587   attributes.set_is_evex_instruction();
 8588   attributes.set_embedded_opmask_register_specifier(mask);
 8589   if (merge) {
 8590     attributes.reset_is_clear_context();
 8591   }
 8592   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8593   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 8594 }
 8595 
 8596 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8597   InstructionMark im(this);
 8598   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8599   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8600   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8601   attributes.set_is_evex_instruction();
 8602   attributes.set_embedded_opmask_register_specifier(mask);
 8603   if (merge) {
 8604     attributes.reset_is_clear_context();
 8605   }
 8606   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8607   emit_int8((unsigned char)0xD5);
 8608   emit_operand(dst, src);
 8609 }
 8610 
 8611 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8612   assert(VM_Version::supports_evex(), "");
 8613   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8614   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8615   attributes.set_is_evex_instruction();
 8616   attributes.set_embedded_opmask_register_specifier(mask);
 8617   if (merge) {
 8618     attributes.reset_is_clear_context();
 8619   }
 8620   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8621   emit_int16(0x40, (0xC0 | encode));
 8622 }
 8623 
 8624 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8625   InstructionMark im(this);
 8626   assert(VM_Version::supports_evex(), "");
 8627   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8628   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8629   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8630   attributes.set_is_evex_instruction();
 8631   attributes.set_embedded_opmask_register_specifier(mask);
 8632   if (merge) {
 8633     attributes.reset_is_clear_context();
 8634   }
 8635   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8636   emit_int8(0x40);
 8637   emit_operand(dst, src);
 8638 }
 8639 
 8640 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8641   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8642   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8643   attributes.set_is_evex_instruction();
 8644   attributes.set_embedded_opmask_register_specifier(mask);
 8645   if (merge) {
 8646     attributes.reset_is_clear_context();
 8647   }
 8648   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8649   emit_int16(0x40, (0xC0 | encode));
 8650 }
 8651 
 8652 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8653   InstructionMark im(this);
 8654   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8655   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8656   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8657   attributes.set_is_evex_instruction();
 8658   attributes.set_embedded_opmask_register_specifier(mask);
 8659   if (merge) {
 8660     attributes.reset_is_clear_context();
 8661   }
 8662   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8663   emit_int8(0x40);
 8664   emit_operand(dst, src);
 8665 }
 8666 
 8667 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8668   assert(VM_Version::supports_evex(), "");
 8669   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8670   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8671   attributes.set_is_evex_instruction();
 8672   attributes.set_embedded_opmask_register_specifier(mask);
 8673   if (merge) {
 8674     attributes.reset_is_clear_context();
 8675   }
 8676   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8677   emit_int16(0x59, (0xC0 | encode));
 8678 }
 8679 
 8680 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8681   InstructionMark im(this);
 8682   assert(VM_Version::supports_evex(), "");
 8683   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8684   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8685   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8686   attributes.set_is_evex_instruction();
 8687   attributes.set_embedded_opmask_register_specifier(mask);
 8688   if (merge) {
 8689     attributes.reset_is_clear_context();
 8690   }
 8691   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8692   emit_int8(0x59);
 8693   emit_operand(dst, src);
 8694 }
 8695 
 8696 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8697   assert(VM_Version::supports_evex(), "");
 8698   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8699   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8700   attributes.set_is_evex_instruction();
 8701   attributes.set_embedded_opmask_register_specifier(mask);
 8702   if (merge) {
 8703     attributes.reset_is_clear_context();
 8704   }
 8705   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8706   emit_int16(0x59, (0xC0 | encode));
 8707 }
 8708 
 8709 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8710   InstructionMark im(this);
 8711   assert(VM_Version::supports_evex(), "");
 8712   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8713   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8714   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8715   attributes.set_is_evex_instruction();
 8716   attributes.set_embedded_opmask_register_specifier(mask);
 8717   if (merge) {
 8718     attributes.reset_is_clear_context();
 8719   }
 8720   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8721   emit_int8(0x59);
 8722   emit_operand(dst, src);
 8723 }
 8724 
 8725 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8726   assert(VM_Version::supports_evex(), "");
 8727   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8728   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8729   attributes.set_is_evex_instruction();
 8730   attributes.set_embedded_opmask_register_specifier(mask);
 8731   if (merge) {
 8732     attributes.reset_is_clear_context();
 8733   }
 8734   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8735   emit_int16(0x51, (0xC0 | encode));
 8736 }
 8737 
 8738 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8739   InstructionMark im(this);
 8740   assert(VM_Version::supports_evex(), "");
 8741   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8742   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8743   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8744   attributes.set_is_evex_instruction();
 8745   attributes.set_embedded_opmask_register_specifier(mask);
 8746   if (merge) {
 8747     attributes.reset_is_clear_context();
 8748   }
 8749   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8750   emit_int8(0x51);
 8751   emit_operand(dst, src);
 8752 }
 8753 
 8754 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8755   assert(VM_Version::supports_evex(), "");
 8756   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8757   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8758   attributes.set_is_evex_instruction();
 8759   attributes.set_embedded_opmask_register_specifier(mask);
 8760   if (merge) {
 8761     attributes.reset_is_clear_context();
 8762   }
 8763   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8764   emit_int16(0x51, (0xC0 | encode));
 8765 }
 8766 
 8767 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8768   InstructionMark im(this);
 8769   assert(VM_Version::supports_evex(), "");
 8770   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8771   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8772   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8773   attributes.set_is_evex_instruction();
 8774   attributes.set_embedded_opmask_register_specifier(mask);
 8775   if (merge) {
 8776     attributes.reset_is_clear_context();
 8777   }
 8778   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8779   emit_int8(0x51);
 8780   emit_operand(dst, src);
 8781 }
 8782 
 8783 
 8784 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8785   assert(VM_Version::supports_evex(), "");
 8786   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8787   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8788   attributes.set_is_evex_instruction();
 8789   attributes.set_embedded_opmask_register_specifier(mask);
 8790   if (merge) {
 8791     attributes.reset_is_clear_context();
 8792   }
 8793   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8794   emit_int16(0x5E, (0xC0 | encode));
 8795 }
 8796 
 8797 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8798   InstructionMark im(this);
 8799   assert(VM_Version::supports_evex(), "");
 8800   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8801   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8802   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8803   attributes.set_is_evex_instruction();
 8804   attributes.set_embedded_opmask_register_specifier(mask);
 8805   if (merge) {
 8806     attributes.reset_is_clear_context();
 8807   }
 8808   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8809   emit_int8(0x5E);
 8810   emit_operand(dst, src);
 8811 }
 8812 
 8813 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8814   assert(VM_Version::supports_evex(), "");
 8815   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8816   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8817   attributes.set_is_evex_instruction();
 8818   attributes.set_embedded_opmask_register_specifier(mask);
 8819   if (merge) {
 8820     attributes.reset_is_clear_context();
 8821   }
 8822   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8823   emit_int16(0x5E, (0xC0 | encode));
 8824 }
 8825 
 8826 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8827   InstructionMark im(this);
 8828   assert(VM_Version::supports_evex(), "");
 8829   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8830   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8832   attributes.set_is_evex_instruction();
 8833   attributes.set_embedded_opmask_register_specifier(mask);
 8834   if (merge) {
 8835     attributes.reset_is_clear_context();
 8836   }
 8837   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8838   emit_int8(0x5E);
 8839   emit_operand(dst, src);
 8840 }
 8841 
 8842 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8843   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8844   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8845   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8846   attributes.set_is_evex_instruction();
 8847   attributes.set_embedded_opmask_register_specifier(mask);
 8848   if (merge) {
 8849     attributes.reset_is_clear_context();
 8850   }
 8851   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8852   emit_int16(0x1C, (0xC0 | encode));
 8853 }
 8854 
 8855 
 8856 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 8857   InstructionMark im(this);
 8858   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8859   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8860   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8861   attributes.set_is_evex_instruction();
 8862   attributes.set_embedded_opmask_register_specifier(mask);
 8863   if (merge) {
 8864     attributes.reset_is_clear_context();
 8865   }
 8866   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8867   emit_int8(0x1C);
 8868   emit_operand(dst, src);
 8869 }
 8870 
 8871 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8872   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8873   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8874   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8875   attributes.set_is_evex_instruction();
 8876   attributes.set_embedded_opmask_register_specifier(mask);
 8877   if (merge) {
 8878     attributes.reset_is_clear_context();
 8879   }
 8880   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8881   emit_int16(0x1D, (0xC0 | encode));
 8882 }
 8883 
 8884 
 8885 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 8886   InstructionMark im(this);
 8887   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 8888   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8889   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8890   attributes.set_is_evex_instruction();
 8891   attributes.set_embedded_opmask_register_specifier(mask);
 8892   if (merge) {
 8893     attributes.reset_is_clear_context();
 8894   }
 8895   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8896   emit_int8(0x1D);
 8897   emit_operand(dst, src);
 8898 }
 8899 
 8900 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8901   assert(VM_Version::supports_evex(), "");
 8902   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8903   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8904   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8905   attributes.set_is_evex_instruction();
 8906   attributes.set_embedded_opmask_register_specifier(mask);
 8907   if (merge) {
 8908     attributes.reset_is_clear_context();
 8909   }
 8910   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8911   emit_int16(0x1E, (0xC0 | encode));
 8912 }
 8913 
 8914 
 8915 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 8916   InstructionMark im(this);
 8917   assert(VM_Version::supports_evex(), "");
 8918   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8919   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8920   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8921   attributes.set_is_evex_instruction();
 8922   attributes.set_embedded_opmask_register_specifier(mask);
 8923   if (merge) {
 8924     attributes.reset_is_clear_context();
 8925   }
 8926   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8927   emit_int8(0x1E);
 8928   emit_operand(dst, src);
 8929 }
 8930 
 8931 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8932   assert(VM_Version::supports_evex(), "");
 8933   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8934   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8935   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8936   attributes.set_is_evex_instruction();
 8937   attributes.set_embedded_opmask_register_specifier(mask);
 8938   if (merge) {
 8939     attributes.reset_is_clear_context();
 8940   }
 8941   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8942   emit_int16(0x1F, (0xC0 | encode));
 8943 }
 8944 
 8945 
 8946 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 8947   InstructionMark im(this);
 8948   assert(VM_Version::supports_evex(), "");
 8949   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8950   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8951   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8952   attributes.set_is_evex_instruction();
 8953   attributes.set_embedded_opmask_register_specifier(mask);
 8954   if (merge) {
 8955     attributes.reset_is_clear_context();
 8956   }
 8957   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8958   emit_int8(0x1F);
 8959   emit_operand(dst, src);
 8960 }
 8961 
 8962 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8963   assert(VM_Version::supports_evex(), "");
 8964   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8965   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8966   attributes.set_is_evex_instruction();
 8967   attributes.set_embedded_opmask_register_specifier(mask);
 8968   if (merge) {
 8969     attributes.reset_is_clear_context();
 8970   }
 8971   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8972   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 8973 }
 8974 
 8975 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8976   InstructionMark im(this);
 8977   assert(VM_Version::supports_evex(), "");
 8978   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8979   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8981   attributes.set_is_evex_instruction();
 8982   attributes.set_embedded_opmask_register_specifier(mask);
 8983   if (merge) {
 8984     attributes.reset_is_clear_context();
 8985   }
 8986   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8987   emit_int8((unsigned char)0xA8);
 8988   emit_operand(dst, src);
 8989 }
 8990 
 8991 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8992   assert(VM_Version::supports_evex(), "");
 8993   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8994   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8995   attributes.set_is_evex_instruction();
 8996   attributes.set_embedded_opmask_register_specifier(mask);
 8997   if (merge) {
 8998     attributes.reset_is_clear_context();
 8999   }
 9000   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9001   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9002 }
 9003 
 9004 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9005   InstructionMark im(this);
 9006   assert(VM_Version::supports_evex(), "");
 9007   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9008   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9009   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9010   attributes.set_is_evex_instruction();
 9011   attributes.set_embedded_opmask_register_specifier(mask);
 9012   if (merge) {
 9013     attributes.reset_is_clear_context();
 9014   }
 9015   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9016   emit_int8((unsigned char)0xA8);
 9017   emit_operand(dst, src);
 9018 }
 9019 
 9020 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9021   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9022   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9023   attributes.set_is_evex_instruction();
 9024   attributes.set_embedded_opmask_register_specifier(mask);
 9025   if (merge) {
 9026     attributes.reset_is_clear_context();
 9027   }
 9028   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9029   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9030 }
 9031 
 9032 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9033   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9034   InstructionMark im(this);
 9035   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9036   attributes.set_is_evex_instruction();
 9037   attributes.set_embedded_opmask_register_specifier(mask);
 9038   if (merge) {
 9039     attributes.reset_is_clear_context();
 9040   }
 9041   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9042   emit_int8((unsigned char)0x8D);
 9043   emit_operand(dst, src);
 9044 }
 9045 
 9046 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9047   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9048   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9049   attributes.set_is_evex_instruction();
 9050   attributes.set_embedded_opmask_register_specifier(mask);
 9051   if (merge) {
 9052     attributes.reset_is_clear_context();
 9053   }
 9054   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9055   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9056 }
 9057 
 9058 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9059   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9060   InstructionMark im(this);
 9061   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9062   attributes.set_is_evex_instruction();
 9063   attributes.set_embedded_opmask_register_specifier(mask);
 9064   if (merge) {
 9065     attributes.reset_is_clear_context();
 9066   }
 9067   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9068   emit_int8((unsigned char)0x8D);
 9069   emit_operand(dst, src);
 9070 }
 9071 
 9072 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9073   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9074   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9075   attributes.set_is_evex_instruction();
 9076   attributes.set_embedded_opmask_register_specifier(mask);
 9077   if (merge) {
 9078     attributes.reset_is_clear_context();
 9079   }
 9080   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9081   emit_int16(0x36, (0xC0 | encode));
 9082 }
 9083 
 9084 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9085   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9086   InstructionMark im(this);
 9087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9088   attributes.set_is_evex_instruction();
 9089   attributes.set_embedded_opmask_register_specifier(mask);
 9090   if (merge) {
 9091     attributes.reset_is_clear_context();
 9092   }
 9093   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9094   emit_int8(0x36);
 9095   emit_operand(dst, src);
 9096 }
 9097 
 9098 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9099   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9100   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9101   attributes.set_is_evex_instruction();
 9102   attributes.set_embedded_opmask_register_specifier(mask);
 9103   if (merge) {
 9104     attributes.reset_is_clear_context();
 9105   }
 9106   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9107   emit_int16(0x36, (0xC0 | encode));
 9108 }
 9109 
 9110 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9111   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9112   InstructionMark im(this);
 9113   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9114   attributes.set_is_evex_instruction();
 9115   attributes.set_embedded_opmask_register_specifier(mask);
 9116   if (merge) {
 9117     attributes.reset_is_clear_context();
 9118   }
 9119   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9120   emit_int8(0x36);
 9121   emit_operand(dst, src);
 9122 }
 9123 
 9124 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9125   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9126   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9127   attributes.set_is_evex_instruction();
 9128   attributes.set_embedded_opmask_register_specifier(mask);
 9129   if (merge) {
 9130     attributes.reset_is_clear_context();
 9131   }
 9132   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9133   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9134 }
 9135 
 9136 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9137   assert(VM_Version::supports_evex(), "");
 9138   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9140   attributes.set_is_evex_instruction();
 9141   attributes.set_embedded_opmask_register_specifier(mask);
 9142   if (merge) {
 9143     attributes.reset_is_clear_context();
 9144   }
 9145   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9146   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9147 }
 9148 
 9149 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9150   assert(VM_Version::supports_evex(), "");
 9151   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9152   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9153   attributes.set_is_evex_instruction();
 9154   attributes.set_embedded_opmask_register_specifier(mask);
 9155   if (merge) {
 9156     attributes.reset_is_clear_context();
 9157   }
 9158   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9159   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9160 }
 9161 
 9162 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9163   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9164   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9165   attributes.set_is_evex_instruction();
 9166   attributes.set_embedded_opmask_register_specifier(mask);
 9167   if (merge) {
 9168     attributes.reset_is_clear_context();
 9169   }
 9170   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9171   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9172 }
 9173 
 9174 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9175   assert(VM_Version::supports_evex(), "");
 9176   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9177   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9178   attributes.set_is_evex_instruction();
 9179   attributes.set_embedded_opmask_register_specifier(mask);
 9180   if (merge) {
 9181     attributes.reset_is_clear_context();
 9182   }
 9183   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9184   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9185 }
 9186 
 9187 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9188   assert(VM_Version::supports_evex(), "");
 9189   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9190   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9191   attributes.set_is_evex_instruction();
 9192   attributes.set_embedded_opmask_register_specifier(mask);
 9193   if (merge) {
 9194     attributes.reset_is_clear_context();
 9195   }
 9196   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9197   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9198 }
 9199 
 9200 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9201   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9202   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9203   attributes.set_is_evex_instruction();
 9204   attributes.set_embedded_opmask_register_specifier(mask);
 9205   if (merge) {
 9206     attributes.reset_is_clear_context();
 9207   }
 9208   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9209   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9210 }
 9211 
 9212 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9213   assert(VM_Version::supports_evex(), "");
 9214   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9216   attributes.set_is_evex_instruction();
 9217   attributes.set_embedded_opmask_register_specifier(mask);
 9218   if (merge) {
 9219     attributes.reset_is_clear_context();
 9220   }
 9221   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9222   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9223 }
 9224 
 9225 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9226   assert(VM_Version::supports_evex(), "");
 9227   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9228   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9229   attributes.set_is_evex_instruction();
 9230   attributes.set_embedded_opmask_register_specifier(mask);
 9231   if (merge) {
 9232     attributes.reset_is_clear_context();
 9233   }
 9234   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9235   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9236 }
 9237 
 9238 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9239   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9240   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9241   attributes.set_is_evex_instruction();
 9242   attributes.set_embedded_opmask_register_specifier(mask);
 9243   if (merge) {
 9244     attributes.reset_is_clear_context();
 9245   }
 9246   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9247   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 9248 }
 9249 
 9250 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9251   assert(VM_Version::supports_evex(), "");
 9252   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9253   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9254   attributes.set_is_evex_instruction();
 9255   attributes.set_embedded_opmask_register_specifier(mask);
 9256   if (merge) {
 9257     attributes.reset_is_clear_context();
 9258   }
 9259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9260   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 9261 }
 9262 
 9263 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9264   assert(VM_Version::supports_evex(), "");
 9265   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9266   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9267   attributes.set_is_evex_instruction();
 9268   attributes.set_embedded_opmask_register_specifier(mask);
 9269   if (merge) {
 9270     attributes.reset_is_clear_context();
 9271   }
 9272   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9273   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 9274 }
 9275 
 9276 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9277   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9278   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9279   attributes.set_is_evex_instruction();
 9280   attributes.set_embedded_opmask_register_specifier(mask);
 9281   if (merge) {
 9282     attributes.reset_is_clear_context();
 9283   }
 9284   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9285   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 9286 }
 9287 
 9288 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9289   assert(VM_Version::supports_evex(), "");
 9290   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9291   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9292   attributes.set_is_evex_instruction();
 9293   attributes.set_embedded_opmask_register_specifier(mask);
 9294   if (merge) {
 9295     attributes.reset_is_clear_context();
 9296   }
 9297   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9298   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 9299 }
 9300 
 9301 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9302   assert(VM_Version::supports_evex(), "");
 9303   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9304   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9305   attributes.set_is_evex_instruction();
 9306   attributes.set_embedded_opmask_register_specifier(mask);
 9307   if (merge) {
 9308     attributes.reset_is_clear_context();
 9309   }
 9310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9311   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 9312 }
 9313 
 9314 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9315   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9316   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9317   attributes.set_is_evex_instruction();
 9318   attributes.set_embedded_opmask_register_specifier(mask);
 9319   if (merge) {
 9320     attributes.reset_is_clear_context();
 9321   }
 9322   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9323   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 9324 }
 9325 
 9326 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9327   assert(VM_Version::supports_evex(), "");
 9328   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9329   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9330   attributes.set_is_evex_instruction();
 9331   attributes.set_embedded_opmask_register_specifier(mask);
 9332   if (merge) {
 9333     attributes.reset_is_clear_context();
 9334   }
 9335   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9336   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9337 }
 9338 
 9339 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9340   assert(VM_Version::supports_evex(), "");
 9341   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9342   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9343   attributes.set_is_evex_instruction();
 9344   attributes.set_embedded_opmask_register_specifier(mask);
 9345   if (merge) {
 9346     attributes.reset_is_clear_context();
 9347   }
 9348   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9349   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 9350 }
 9351 
 9352 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9353   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9355   attributes.set_is_evex_instruction();
 9356   attributes.set_embedded_opmask_register_specifier(mask);
 9357   if (merge) {
 9358     attributes.reset_is_clear_context();
 9359   }
 9360   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9361   emit_int16(0x12, (0xC0 | encode));
 9362 }
 9363 
 9364 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9365   assert(VM_Version::supports_evex(), "");
 9366   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9367   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9368   attributes.set_is_evex_instruction();
 9369   attributes.set_embedded_opmask_register_specifier(mask);
 9370   if (merge) {
 9371     attributes.reset_is_clear_context();
 9372   }
 9373   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9374   emit_int16(0x47, (0xC0 | encode));
 9375 }
 9376 
 9377 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9378   assert(VM_Version::supports_evex(), "");
 9379   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9380   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9381   attributes.set_is_evex_instruction();
 9382   attributes.set_embedded_opmask_register_specifier(mask);
 9383   if (merge) {
 9384     attributes.reset_is_clear_context();
 9385   }
 9386   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9387   emit_int16(0x47, (0xC0 | encode));
 9388 }
 9389 
 9390 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9391   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9392   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9393   attributes.set_is_evex_instruction();
 9394   attributes.set_embedded_opmask_register_specifier(mask);
 9395   if (merge) {
 9396     attributes.reset_is_clear_context();
 9397   }
 9398   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9399   emit_int16(0x10, (0xC0 | encode));
 9400 }
 9401 
 9402 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9403   assert(VM_Version::supports_evex(), "");
 9404   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9405   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9406   attributes.set_is_evex_instruction();
 9407   attributes.set_embedded_opmask_register_specifier(mask);
 9408   if (merge) {
 9409     attributes.reset_is_clear_context();
 9410   }
 9411   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9412   emit_int16(0x45, (0xC0 | encode));
 9413 }
 9414 
 9415 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9416   assert(VM_Version::supports_evex(), "");
 9417   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9418   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9419   attributes.set_is_evex_instruction();
 9420   attributes.set_embedded_opmask_register_specifier(mask);
 9421   if (merge) {
 9422     attributes.reset_is_clear_context();
 9423   }
 9424   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9425   emit_int16(0x45, (0xC0 | encode));
 9426 }
 9427 
 9428 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9429   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9430   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9431   attributes.set_is_evex_instruction();
 9432   attributes.set_embedded_opmask_register_specifier(mask);
 9433   if (merge) {
 9434     attributes.reset_is_clear_context();
 9435   }
 9436   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9437   emit_int16(0x11, (0xC0 | encode));
 9438 }
 9439 
 9440 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9441   assert(VM_Version::supports_evex(), "");
 9442   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9443   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9444   attributes.set_is_evex_instruction();
 9445   attributes.set_embedded_opmask_register_specifier(mask);
 9446   if (merge) {
 9447     attributes.reset_is_clear_context();
 9448   }
 9449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9450   emit_int16(0x46, (0xC0 | encode));
 9451 }
 9452 
 9453 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9454   assert(VM_Version::supports_evex(), "");
 9455   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9456   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9457   attributes.set_is_evex_instruction();
 9458   attributes.set_embedded_opmask_register_specifier(mask);
 9459   if (merge) {
 9460     attributes.reset_is_clear_context();
 9461   }
 9462   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9463   emit_int16(0x46, (0xC0 | encode));
 9464 }
 9465 
 9466 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9467   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9469   attributes.set_is_evex_instruction();
 9470   attributes.set_embedded_opmask_register_specifier(mask);
 9471   if (merge) {
 9472     attributes.reset_is_clear_context();
 9473   }
 9474   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9475   emit_int16(0x38, (0xC0 | encode));
 9476 }
 9477 
 9478 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9479   assert(VM_Version::supports_avx512bw(), "");
 9480   InstructionMark im(this);
 9481   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9482   attributes.set_is_evex_instruction();
 9483   attributes.set_embedded_opmask_register_specifier(mask);
 9484   if (merge) {
 9485     attributes.reset_is_clear_context();
 9486   }
 9487   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9488   emit_int8(0x38);
 9489   emit_operand(dst, src);
 9490 }
 9491 
 9492 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9493   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9494   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9495   attributes.set_is_evex_instruction();
 9496   attributes.set_embedded_opmask_register_specifier(mask);
 9497   if (merge) {
 9498     attributes.reset_is_clear_context();
 9499   }
 9500   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9501   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 9502 }
 9503 
 9504 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9505   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9506   InstructionMark im(this);
 9507   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9508   attributes.set_is_evex_instruction();
 9509   attributes.set_embedded_opmask_register_specifier(mask);
 9510   if (merge) {
 9511     attributes.reset_is_clear_context();
 9512   }
 9513   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9514   emit_int8((unsigned char)0xEA);
 9515   emit_operand(dst, src);
 9516 }
 9517 
 9518 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9519   assert(VM_Version::supports_evex(), "");
 9520   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9521   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9522   attributes.set_is_evex_instruction();
 9523   attributes.set_embedded_opmask_register_specifier(mask);
 9524   if (merge) {
 9525     attributes.reset_is_clear_context();
 9526   }
 9527   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9528   emit_int16(0x39, (0xC0 | encode));
 9529 }
 9530 
 9531 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9532   assert(VM_Version::supports_evex(), "");
 9533   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9534   InstructionMark im(this);
 9535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9536   attributes.set_is_evex_instruction();
 9537   attributes.set_embedded_opmask_register_specifier(mask);
 9538   if (merge) {
 9539     attributes.reset_is_clear_context();
 9540   }
 9541   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9542   emit_int8(0x39);
 9543   emit_operand(dst, src);
 9544 }
 9545 
 9546 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9547   assert(VM_Version::supports_evex(), "");
 9548   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9549   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9550   attributes.set_is_evex_instruction();
 9551   attributes.set_embedded_opmask_register_specifier(mask);
 9552   if (merge) {
 9553     attributes.reset_is_clear_context();
 9554   }
 9555   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9556   emit_int16(0x39, (0xC0 | encode));
 9557 }
 9558 
 9559 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9560   assert(VM_Version::supports_evex(), "");
 9561   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9562   InstructionMark im(this);
 9563   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9564   attributes.set_is_evex_instruction();
 9565   attributes.set_embedded_opmask_register_specifier(mask);
 9566   if (merge) {
 9567     attributes.reset_is_clear_context();
 9568   }
 9569   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9570   emit_int8(0x39);
 9571   emit_operand(dst, src);
 9572 }
 9573 
 9574 
 9575 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9576   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9577   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9578   attributes.set_is_evex_instruction();
 9579   attributes.set_embedded_opmask_register_specifier(mask);
 9580   if (merge) {
 9581     attributes.reset_is_clear_context();
 9582   }
 9583   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9584   emit_int16(0x3C, (0xC0 | encode));
 9585 }
 9586 
 9587 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9588   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9589   InstructionMark im(this);
 9590   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9591   attributes.set_is_evex_instruction();
 9592   attributes.set_embedded_opmask_register_specifier(mask);
 9593   if (merge) {
 9594     attributes.reset_is_clear_context();
 9595   }
 9596   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9597   emit_int8(0x3C);
 9598   emit_operand(dst, src);
 9599 }
 9600 
 9601 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9602   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9603   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9604   attributes.set_is_evex_instruction();
 9605   attributes.set_embedded_opmask_register_specifier(mask);
 9606   if (merge) {
 9607     attributes.reset_is_clear_context();
 9608   }
 9609   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9610   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 9611 }
 9612 
 9613 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9614   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9615   InstructionMark im(this);
 9616   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9617   attributes.set_is_evex_instruction();
 9618   attributes.set_embedded_opmask_register_specifier(mask);
 9619   if (merge) {
 9620     attributes.reset_is_clear_context();
 9621   }
 9622   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9623   emit_int8((unsigned char)0xEE);
 9624   emit_operand(dst, src);
 9625 }
 9626 
 9627 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9628   assert(VM_Version::supports_evex(), "");
 9629   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9630   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9631   attributes.set_is_evex_instruction();
 9632   attributes.set_embedded_opmask_register_specifier(mask);
 9633   if (merge) {
 9634     attributes.reset_is_clear_context();
 9635   }
 9636   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9637   emit_int16(0x3D, (0xC0 | encode));
 9638 }
 9639 
 9640 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9641   assert(VM_Version::supports_evex(), "");
 9642   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9643   InstructionMark im(this);
 9644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9645   attributes.set_is_evex_instruction();
 9646   attributes.set_embedded_opmask_register_specifier(mask);
 9647   if (merge) {
 9648     attributes.reset_is_clear_context();
 9649   }
 9650   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9651   emit_int8(0x3D);
 9652   emit_operand(dst, src);
 9653 }
 9654 
 9655 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9656   assert(VM_Version::supports_evex(), "");
 9657   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9658   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9659   attributes.set_is_evex_instruction();
 9660   attributes.set_embedded_opmask_register_specifier(mask);
 9661   if (merge) {
 9662     attributes.reset_is_clear_context();
 9663   }
 9664   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9665   emit_int16(0x3D, (0xC0 | encode));
 9666 }
 9667 
 9668 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9669   assert(VM_Version::supports_evex(), "");
 9670   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9671   InstructionMark im(this);
 9672   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9673   attributes.set_is_evex_instruction();
 9674   attributes.set_embedded_opmask_register_specifier(mask);
 9675   if (merge) {
 9676     attributes.reset_is_clear_context();
 9677   }
 9678   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9679   emit_int8(0x3D);
 9680   emit_operand(dst, src);
 9681 }
 9682 
 9683 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 9684 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
 9685   assert(UseAVX >= 2, "");
 9686   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9687   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9688   emit_int16(0x58, (0xC0 | encode));
 9689 }
 9690 
 9691 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
 9692   assert(VM_Version::supports_avx2(), "");
 9693   assert(dst != xnoreg, "sanity");
 9694   InstructionMark im(this);
 9695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9696   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9697   // swap src<->dst for encoding
 9698   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9699   emit_int8(0x58);
 9700   emit_operand(dst, src);
 9701 }
 9702 
 9703 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 9704 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
 9705   assert(VM_Version::supports_avx2(), "");
 9706   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9707   attributes.set_rex_vex_w_reverted();
 9708   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9709   emit_int16(0x59, (0xC0 | encode));
 9710 }
 9711 
 9712 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
 9713   assert(VM_Version::supports_avx2(), "");
 9714   assert(dst != xnoreg, "sanity");
 9715   InstructionMark im(this);
 9716   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9717   attributes.set_rex_vex_w_reverted();
 9718   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 9719   // swap src<->dst for encoding
 9720   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9721   emit_int8(0x59);
 9722   emit_operand(dst, src);
 9723 }
 9724 
 9725 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
 9726   assert(vector_len != Assembler::AVX_128bit, "");
 9727   assert(VM_Version::supports_avx512dq(), "");
 9728   assert(dst != xnoreg, "sanity");
 9729   InstructionMark im(this);
 9730   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9731   attributes.set_rex_vex_w_reverted();
 9732   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 9733   // swap src<->dst for encoding
 9734   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9735   emit_int8(0x5A);
 9736   emit_operand(dst, src);
 9737 }
 9738 
 9739 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
 9740   assert(vector_len != Assembler::AVX_128bit, "");
 9741   assert(VM_Version::supports_avx512dq(), "");
 9742   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9743   attributes.set_rex_vex_w_reverted();
 9744   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9745   emit_int16(0x5A, (0xC0 | encode));
 9746 }
 9747 
 9748 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
 9749   assert(vector_len != Assembler::AVX_128bit, "");
 9750   assert(VM_Version::supports_avx512dq(), "");
 9751   assert(dst != xnoreg, "sanity");
 9752   InstructionMark im(this);
 9753   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9754   attributes.set_rex_vex_w_reverted();
 9755   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 9756   // swap src<->dst for encoding
 9757   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9758   emit_int8(0x5A);
 9759   emit_operand(dst, src);
 9760 }
 9761 
 9762 // scalar single/double precision replicate
 9763 
 9764 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
 9765 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
 9766   assert(VM_Version::supports_avx2(), "");
 9767   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9768   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9769   emit_int16(0x18, (0xC0 | encode));
 9770 }
 9771 
 9772 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
 9773   assert(VM_Version::supports_avx(), "");
 9774   assert(dst != xnoreg, "sanity");
 9775   InstructionMark im(this);
 9776   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9777   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9778   // swap src<->dst for encoding
 9779   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9780   emit_int8(0x18);
 9781   emit_operand(dst, src);
 9782 }
 9783 
 9784 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
 9785 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
 9786   assert(VM_Version::supports_avx2(), "");
 9787   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 9788   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9789   attributes.set_rex_vex_w_reverted();
 9790   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9791   emit_int16(0x19, (0xC0 | encode));
 9792 }
 9793 
 9794 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
 9795   assert(VM_Version::supports_avx(), "");
 9796   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 9797   assert(dst != xnoreg, "sanity");
 9798   InstructionMark im(this);
 9799   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9800   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 9801   attributes.set_rex_vex_w_reverted();
 9802   // swap src<->dst for encoding
 9803   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9804   emit_int8(0x19);
 9805   emit_operand(dst, src);
 9806 }
 9807 
 9808 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
 9809   assert(VM_Version::supports_avx(), "");
 9810   assert(vector_len == AVX_256bit, "");
 9811   assert(dst != xnoreg, "sanity");
 9812   InstructionMark im(this);
 9813   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9814   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 9815   // swap src<->dst for encoding
 9816   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9817   emit_int8(0x1A);
 9818   emit_operand(dst, src);
 9819 }
 9820 
 9821 // gpr source broadcast forms
 9822 
 9823 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 9824 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
 9825   assert(VM_Version::supports_avx512bw(), "");
 9826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 9827   attributes.set_is_evex_instruction();
 9828   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9829   emit_int16(0x7A, (0xC0 | encode));
 9830 }
 9831 
 9832 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 9833 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
 9834   assert(VM_Version::supports_avx512bw(), "");
 9835   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 9836   attributes.set_is_evex_instruction();
 9837   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9838   emit_int16(0x7B, (0xC0 | encode));
 9839 }
 9840 
 9841 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 9842 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
 9843   assert(VM_Version::supports_evex(), "");
 9844   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9845   attributes.set_is_evex_instruction();
 9846   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9847   emit_int16(0x7C, (0xC0 | encode));
 9848 }
 9849 
 9850 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 9851 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
 9852   assert(VM_Version::supports_evex(), "");
 9853   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9854   attributes.set_is_evex_instruction();
 9855   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9856   emit_int16(0x7C, (0xC0 | encode));
 9857 }
 9858 
 9859 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 9860   assert(VM_Version::supports_avx2(), "");
 9861   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 9862   assert(dst != xnoreg, "sanity");
 9863   assert(src.isxmmindex(),"expected to be xmm index");
 9864   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9865   InstructionMark im(this);
 9866   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9867   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9868   emit_int8((unsigned char)0x90);
 9869   emit_operand(dst, src);
 9870 }
 9871 
 9872 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 9873   assert(VM_Version::supports_avx2(), "");
 9874   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 9875   assert(dst != xnoreg, "sanity");
 9876   assert(src.isxmmindex(),"expected to be xmm index");
 9877   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9878   InstructionMark im(this);
 9879   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9880   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9881   emit_int8((unsigned char)0x90);
 9882   emit_operand(dst, src);
 9883 }
 9884 
 9885 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 9886   assert(VM_Version::supports_avx2(), "");
 9887   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 9888   assert(dst != xnoreg, "sanity");
 9889   assert(src.isxmmindex(),"expected to be xmm index");
 9890   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9891   InstructionMark im(this);
 9892   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9893   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9894   emit_int8((unsigned char)0x92);
 9895   emit_operand(dst, src);
 9896 }
 9897 
 9898 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 9899   assert(VM_Version::supports_avx2(), "");
 9900   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 9901   assert(dst != xnoreg, "sanity");
 9902   assert(src.isxmmindex(),"expected to be xmm index");
 9903   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9904   InstructionMark im(this);
 9905   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 9906   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9907   emit_int8((unsigned char)0x92);
 9908   emit_operand(dst, src);
 9909 }
 9910 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 9911   assert(VM_Version::supports_evex(), "");
 9912   assert(dst != xnoreg, "sanity");
 9913   assert(src.isxmmindex(),"expected to be xmm index");
 9914   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9915   assert(mask != k0, "instruction will #UD if mask is in k0");
 9916   InstructionMark im(this);
 9917   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9918   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9919   attributes.reset_is_clear_context();
 9920   attributes.set_embedded_opmask_register_specifier(mask);
 9921   attributes.set_is_evex_instruction();
 9922   // swap src<->dst for encoding
 9923   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9924   emit_int8((unsigned char)0x90);
 9925   emit_operand(dst, src);
 9926 }
 9927 
 9928 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 9929   assert(VM_Version::supports_evex(), "");
 9930   assert(dst != xnoreg, "sanity");
 9931   assert(src.isxmmindex(),"expected to be xmm index");
 9932   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9933   assert(mask != k0, "instruction will #UD if mask is in k0");
 9934   InstructionMark im(this);
 9935   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9936   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9937   attributes.reset_is_clear_context();
 9938   attributes.set_embedded_opmask_register_specifier(mask);
 9939   attributes.set_is_evex_instruction();
 9940   // swap src<->dst for encoding
 9941   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9942   emit_int8((unsigned char)0x90);
 9943   emit_operand(dst, src);
 9944 }
 9945 
 9946 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 9947   assert(VM_Version::supports_evex(), "");
 9948   assert(dst != xnoreg, "sanity");
 9949   assert(src.isxmmindex(),"expected to be xmm index");
 9950   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9951   assert(mask != k0, "instruction will #UD if mask is in k0");
 9952   InstructionMark im(this);
 9953   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9954   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9955   attributes.reset_is_clear_context();
 9956   attributes.set_embedded_opmask_register_specifier(mask);
 9957   attributes.set_is_evex_instruction();
 9958   // swap src<->dst for encoding
 9959   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9960   emit_int8((unsigned char)0x92);
 9961   emit_operand(dst, src);
 9962 }
 9963 
 9964 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 9965   assert(VM_Version::supports_evex(), "");
 9966   assert(dst != xnoreg, "sanity");
 9967   assert(src.isxmmindex(),"expected to be xmm index");
 9968   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 9969   assert(mask != k0, "instruction will #UD if mask is in k0");
 9970   InstructionMark im(this);
 9971   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9972   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9973   attributes.reset_is_clear_context();
 9974   attributes.set_embedded_opmask_register_specifier(mask);
 9975   attributes.set_is_evex_instruction();
 9976   // swap src<->dst for encoding
 9977   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9978   emit_int8((unsigned char)0x92);
 9979   emit_operand(dst, src);
 9980 }
 9981 
 9982 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 9983   assert(VM_Version::supports_evex(), "");
 9984   assert(mask != k0, "instruction will #UD if mask is in k0");
 9985   InstructionMark im(this);
 9986   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9987   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 9988   attributes.reset_is_clear_context();
 9989   attributes.set_embedded_opmask_register_specifier(mask);
 9990   attributes.set_is_evex_instruction();
 9991   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9992   emit_int8((unsigned char)0xA0);
 9993   emit_operand(src, dst);
 9994 }
 9995 
 9996 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 9997   assert(VM_Version::supports_evex(), "");
 9998   assert(mask != k0, "instruction will #UD if mask is in k0");
 9999   InstructionMark im(this);
10000   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10001   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10002   attributes.reset_is_clear_context();
10003   attributes.set_embedded_opmask_register_specifier(mask);
10004   attributes.set_is_evex_instruction();
10005   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10006   emit_int8((unsigned char)0xA0);
10007   emit_operand(src, dst);
10008 }
10009 
10010 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10011   assert(VM_Version::supports_evex(), "");
10012   assert(mask != k0, "instruction will #UD if mask is in k0");
10013   InstructionMark im(this);
10014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10015   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10016   attributes.reset_is_clear_context();
10017   attributes.set_embedded_opmask_register_specifier(mask);
10018   attributes.set_is_evex_instruction();
10019   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10020   emit_int8((unsigned char)0xA2);
10021   emit_operand(src, dst);
10022 }
10023 
10024 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10025   assert(VM_Version::supports_evex(), "");
10026   assert(mask != k0, "instruction will #UD if mask is in k0");
10027   InstructionMark im(this);
10028   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10029   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10030   attributes.reset_is_clear_context();
10031   attributes.set_embedded_opmask_register_specifier(mask);
10032   attributes.set_is_evex_instruction();
10033   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10034   emit_int8((unsigned char)0xA2);
10035   emit_operand(src, dst);
10036 }
10037 // Carry-Less Multiplication Quadword
10038 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10039   assert(VM_Version::supports_clmul(), "");
10040   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10041   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10042   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10043 }
10044 
10045 // Carry-Less Multiplication Quadword
10046 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10047   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10048   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10049   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10050   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10051 }
10052 
10053 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10054   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10055   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10056   attributes.set_is_evex_instruction();
10057   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10058   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10059 }
10060 
10061 void Assembler::vzeroupper_uncached() {
10062   if (VM_Version::supports_vzeroupper()) {
10063     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10064     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10065     emit_int8(0x77);
10066   }
10067 }
10068 
10069 void Assembler::fld_x(Address adr) {
10070   InstructionMark im(this);
10071   emit_int8((unsigned char)0xDB);
10072   emit_operand32(rbp, adr);
10073 }
10074 
10075 void Assembler::fstp_x(Address adr) {
10076   InstructionMark im(this);
10077   emit_int8((unsigned char)0xDB);
10078   emit_operand32(rdi, adr);
10079 }
10080 
10081 void Assembler::emit_operand32(Register reg, Address adr) {
10082   assert(reg->encoding() < 8, "no extended registers");
10083   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10084   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
10085                adr._rspec);
10086 }
10087 
10088 #ifndef _LP64
10089 // 32bit only pieces of the assembler
10090 
10091 void Assembler::emms() {
10092   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10093   emit_int16(0x0F, 0x77);
10094 }
10095 
10096 void Assembler::vzeroupper() {
10097   vzeroupper_uncached();
10098 }
10099 
10100 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10101   // NO PREFIX AS NEVER 64BIT
10102   InstructionMark im(this);
10103   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10104   emit_data(imm32, rspec, 0);
10105 }
10106 
10107 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10108   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10109   InstructionMark im(this);
10110   emit_int8((unsigned char)0x81);
10111   emit_operand(rdi, src1);
10112   emit_data(imm32, rspec, 0);
10113 }
10114 
10115 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10116 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10117 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
10118 void Assembler::cmpxchg8(Address adr) {
10119   InstructionMark im(this);
10120   emit_int16(0x0F, (unsigned char)0xC7);
10121   emit_operand(rcx, adr);
10122 }
10123 
10124 void Assembler::decl(Register dst) {
10125   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10126  emit_int8(0x48 | dst->encoding());
10127 }
10128 
10129 // 64bit doesn't use the x87
10130 
10131 void Assembler::emit_farith(int b1, int b2, int i) {
10132   assert(isByte(b1) && isByte(b2), "wrong opcode");
10133   assert(0 <= i &&  i < 8, "illegal stack offset");
10134   emit_int16(b1, b2 + i);
10135 }
10136 
10137 void Assembler::fabs() {
10138   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10139 }
10140 
10141 void Assembler::fadd(int i) {
10142   emit_farith(0xD8, 0xC0, i);
10143 }
10144 
10145 void Assembler::fadd_d(Address src) {
10146   InstructionMark im(this);
10147   emit_int8((unsigned char)0xDC);
10148   emit_operand32(rax, src);
10149 }
10150 
10151 void Assembler::fadd_s(Address src) {
10152   InstructionMark im(this);
10153   emit_int8((unsigned char)0xD8);
10154   emit_operand32(rax, src);
10155 }
10156 
10157 void Assembler::fadda(int i) {
10158   emit_farith(0xDC, 0xC0, i);
10159 }
10160 
10161 void Assembler::faddp(int i) {
10162   emit_farith(0xDE, 0xC0, i);
10163 }
10164 
10165 void Assembler::fchs() {
10166   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10167 }
10168 
10169 void Assembler::fcom(int i) {
10170   emit_farith(0xD8, 0xD0, i);
10171 }
10172 
10173 void Assembler::fcomp(int i) {
10174   emit_farith(0xD8, 0xD8, i);
10175 }
10176 
10177 void Assembler::fcomp_d(Address src) {
10178   InstructionMark im(this);
10179   emit_int8((unsigned char)0xDC);
10180   emit_operand32(rbx, src);
10181 }
10182 
10183 void Assembler::fcomp_s(Address src) {
10184   InstructionMark im(this);
10185   emit_int8((unsigned char)0xD8);
10186   emit_operand32(rbx, src);
10187 }
10188 
10189 void Assembler::fcompp() {
10190   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10191 }
10192 
10193 void Assembler::fcos() {
10194   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10195 }
10196 
10197 void Assembler::fdecstp() {
10198   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10199 }
10200 
10201 void Assembler::fdiv(int i) {
10202   emit_farith(0xD8, 0xF0, i);
10203 }
10204 
10205 void Assembler::fdiv_d(Address src) {
10206   InstructionMark im(this);
10207   emit_int8((unsigned char)0xDC);
10208   emit_operand32(rsi, src);
10209 }
10210 
10211 void Assembler::fdiv_s(Address src) {
10212   InstructionMark im(this);
10213   emit_int8((unsigned char)0xD8);
10214   emit_operand32(rsi, src);
10215 }
10216 
10217 void Assembler::fdiva(int i) {
10218   emit_farith(0xDC, 0xF8, i);
10219 }
10220 
10221 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10222 //       is erroneous for some of the floating-point instructions below.
10223 
10224 void Assembler::fdivp(int i) {
10225   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10226 }
10227 
10228 void Assembler::fdivr(int i) {
10229   emit_farith(0xD8, 0xF8, i);
10230 }
10231 
10232 void Assembler::fdivr_d(Address src) {
10233   InstructionMark im(this);
10234   emit_int8((unsigned char)0xDC);
10235   emit_operand32(rdi, src);
10236 }
10237 
10238 void Assembler::fdivr_s(Address src) {
10239   InstructionMark im(this);
10240   emit_int8((unsigned char)0xD8);
10241   emit_operand32(rdi, src);
10242 }
10243 
10244 void Assembler::fdivra(int i) {
10245   emit_farith(0xDC, 0xF0, i);
10246 }
10247 
10248 void Assembler::fdivrp(int i) {
10249   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10250 }
10251 
10252 void Assembler::ffree(int i) {
10253   emit_farith(0xDD, 0xC0, i);
10254 }
10255 
10256 void Assembler::fild_d(Address adr) {
10257   InstructionMark im(this);
10258   emit_int8((unsigned char)0xDF);
10259   emit_operand32(rbp, adr);
10260 }
10261 
10262 void Assembler::fild_s(Address adr) {
10263   InstructionMark im(this);
10264   emit_int8((unsigned char)0xDB);
10265   emit_operand32(rax, adr);
10266 }
10267 
10268 void Assembler::fincstp() {
10269   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10270 }
10271 
10272 void Assembler::finit() {
10273   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10274 }
10275 
10276 void Assembler::fist_s(Address adr) {
10277   InstructionMark im(this);
10278   emit_int8((unsigned char)0xDB);
10279   emit_operand32(rdx, adr);
10280 }
10281 
10282 void Assembler::fistp_d(Address adr) {
10283   InstructionMark im(this);
10284   emit_int8((unsigned char)0xDF);
10285   emit_operand32(rdi, adr);
10286 }
10287 
10288 void Assembler::fistp_s(Address adr) {
10289   InstructionMark im(this);
10290   emit_int8((unsigned char)0xDB);
10291   emit_operand32(rbx, adr);
10292 }
10293 
10294 void Assembler::fld1() {
10295   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10296 }
10297 
10298 void Assembler::fld_d(Address adr) {
10299   InstructionMark im(this);
10300   emit_int8((unsigned char)0xDD);
10301   emit_operand32(rax, adr);
10302 }
10303 
10304 void Assembler::fld_s(Address adr) {
10305   InstructionMark im(this);
10306   emit_int8((unsigned char)0xD9);
10307   emit_operand32(rax, adr);
10308 }
10309 
10310 
10311 void Assembler::fld_s(int index) {
10312   emit_farith(0xD9, 0xC0, index);
10313 }
10314 
10315 void Assembler::fldcw(Address src) {
10316   InstructionMark im(this);
10317   emit_int8((unsigned char)0xD9);
10318   emit_operand32(rbp, src);
10319 }
10320 
10321 void Assembler::fldenv(Address src) {
10322   InstructionMark im(this);
10323   emit_int8((unsigned char)0xD9);
10324   emit_operand32(rsp, src);
10325 }
10326 
10327 void Assembler::fldlg2() {
10328   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10329 }
10330 
10331 void Assembler::fldln2() {
10332   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10333 }
10334 
10335 void Assembler::fldz() {
10336   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10337 }
10338 
10339 void Assembler::flog() {
10340   fldln2();
10341   fxch();
10342   fyl2x();
10343 }
10344 
10345 void Assembler::flog10() {
10346   fldlg2();
10347   fxch();
10348   fyl2x();
10349 }
10350 
10351 void Assembler::fmul(int i) {
10352   emit_farith(0xD8, 0xC8, i);
10353 }
10354 
10355 void Assembler::fmul_d(Address src) {
10356   InstructionMark im(this);
10357   emit_int8((unsigned char)0xDC);
10358   emit_operand32(rcx, src);
10359 }
10360 
10361 void Assembler::fmul_s(Address src) {
10362   InstructionMark im(this);
10363   emit_int8((unsigned char)0xD8);
10364   emit_operand32(rcx, src);
10365 }
10366 
10367 void Assembler::fmula(int i) {
10368   emit_farith(0xDC, 0xC8, i);
10369 }
10370 
10371 void Assembler::fmulp(int i) {
10372   emit_farith(0xDE, 0xC8, i);
10373 }
10374 
10375 void Assembler::fnsave(Address dst) {
10376   InstructionMark im(this);
10377   emit_int8((unsigned char)0xDD);
10378   emit_operand32(rsi, dst);
10379 }
10380 
10381 void Assembler::fnstcw(Address src) {
10382   InstructionMark im(this);
10383   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10384   emit_operand32(rdi, src);
10385 }
10386 
10387 void Assembler::fnstsw_ax() {
10388   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10389 }
10390 
10391 void Assembler::fprem() {
10392   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10393 }
10394 
10395 void Assembler::fprem1() {
10396   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10397 }
10398 
10399 void Assembler::frstor(Address src) {
10400   InstructionMark im(this);
10401   emit_int8((unsigned char)0xDD);
10402   emit_operand32(rsp, src);
10403 }
10404 
10405 void Assembler::fsin() {
10406   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10407 }
10408 
10409 void Assembler::fsqrt() {
10410   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
10411 }
10412 
10413 void Assembler::fst_d(Address adr) {
10414   InstructionMark im(this);
10415   emit_int8((unsigned char)0xDD);
10416   emit_operand32(rdx, adr);
10417 }
10418 
10419 void Assembler::fst_s(Address adr) {
10420   InstructionMark im(this);
10421   emit_int8((unsigned char)0xD9);
10422   emit_operand32(rdx, adr);
10423 }
10424 
10425 void Assembler::fstp_d(Address adr) {
10426   InstructionMark im(this);
10427   emit_int8((unsigned char)0xDD);
10428   emit_operand32(rbx, adr);
10429 }
10430 
10431 void Assembler::fstp_d(int index) {
10432   emit_farith(0xDD, 0xD8, index);
10433 }
10434 
10435 void Assembler::fstp_s(Address adr) {
10436   InstructionMark im(this);
10437   emit_int8((unsigned char)0xD9);
10438   emit_operand32(rbx, adr);
10439 }
10440 
10441 void Assembler::fsub(int i) {
10442   emit_farith(0xD8, 0xE0, i);
10443 }
10444 
10445 void Assembler::fsub_d(Address src) {
10446   InstructionMark im(this);
10447   emit_int8((unsigned char)0xDC);
10448   emit_operand32(rsp, src);
10449 }
10450 
10451 void Assembler::fsub_s(Address src) {
10452   InstructionMark im(this);
10453   emit_int8((unsigned char)0xD8);
10454   emit_operand32(rsp, src);
10455 }
10456 
10457 void Assembler::fsuba(int i) {
10458   emit_farith(0xDC, 0xE8, i);
10459 }
10460 
10461 void Assembler::fsubp(int i) {
10462   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
10463 }
10464 
10465 void Assembler::fsubr(int i) {
10466   emit_farith(0xD8, 0xE8, i);
10467 }
10468 
10469 void Assembler::fsubr_d(Address src) {
10470   InstructionMark im(this);
10471   emit_int8((unsigned char)0xDC);
10472   emit_operand32(rbp, src);
10473 }
10474 
10475 void Assembler::fsubr_s(Address src) {
10476   InstructionMark im(this);
10477   emit_int8((unsigned char)0xD8);
10478   emit_operand32(rbp, src);
10479 }
10480 
10481 void Assembler::fsubra(int i) {
10482   emit_farith(0xDC, 0xE0, i);
10483 }
10484 
10485 void Assembler::fsubrp(int i) {
10486   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
10487 }
10488 
10489 void Assembler::ftan() {
10490   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
10491 }
10492 
10493 void Assembler::ftst() {
10494   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
10495 }
10496 
10497 void Assembler::fucomi(int i) {
10498   // make sure the instruction is supported (introduced for P6, together with cmov)
10499   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10500   emit_farith(0xDB, 0xE8, i);
10501 }
10502 
10503 void Assembler::fucomip(int i) {
10504   // make sure the instruction is supported (introduced for P6, together with cmov)
10505   guarantee(VM_Version::supports_cmov(), "illegal instruction");
10506   emit_farith(0xDF, 0xE8, i);
10507 }
10508 
10509 void Assembler::fwait() {
10510   emit_int8((unsigned char)0x9B);
10511 }
10512 
10513 void Assembler::fxch(int i) {
10514   emit_farith(0xD9, 0xC8, i);
10515 }
10516 
10517 void Assembler::fyl2x() {
10518   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
10519 }
10520 
10521 void Assembler::frndint() {
10522   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
10523 }
10524 
10525 void Assembler::f2xm1() {
10526   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
10527 }
10528 
10529 void Assembler::fldl2e() {
10530   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
10531 }
10532 #endif // !_LP64
10533 
10534 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
10535 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
10536 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
10537 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
10538 
10539 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
10540 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10541   if (pre > 0) {
10542     emit_int8(simd_pre[pre]);
10543   }
10544   if (rex_w) {
10545     prefixq(adr, xreg);
10546   } else {
10547     prefix(adr, xreg);
10548   }
10549   if (opc > 0) {
10550     emit_int8(0x0F);
10551     int opc2 = simd_opc[opc];
10552     if (opc2 > 0) {
10553       emit_int8(opc2);
10554     }
10555   }
10556 }
10557 
10558 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
10559   if (pre > 0) {
10560     emit_int8(simd_pre[pre]);
10561   }
10562   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
10563   if (opc > 0) {
10564     emit_int8(0x0F);
10565     int opc2 = simd_opc[opc];
10566     if (opc2 > 0) {
10567       emit_int8(opc2);
10568     }
10569   }
10570   return encode;
10571 }
10572 
10573 
10574 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
10575   int vector_len = _attributes->get_vector_len();
10576   bool vex_w = _attributes->is_rex_vex_w();
10577   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
10578     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
10579     byte1 = (~byte1) & 0xE0;
10580     byte1 |= opc;
10581 
10582     int byte2 = ((~nds_enc) & 0xf) << 3;
10583     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
10584 
10585     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
10586   } else {
10587     int byte1 = vex_r ? VEX_R : 0;
10588     byte1 = (~byte1) & 0x80;
10589     byte1 |= ((~nds_enc) & 0xf) << 3;
10590     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
10591     emit_int16((unsigned char)VEX_2bytes, byte1);
10592   }
10593 }
10594 
10595 // This is a 4 byte encoding
10596 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
10597   // EVEX 0x62 prefix
10598   // byte1 = EVEX_4bytes;
10599 
10600   bool vex_w = _attributes->is_rex_vex_w();
10601   int evex_encoding = (vex_w ? VEX_W : 0);
10602   // EVEX.b is not currently used for broadcast of single element or data rounding modes
10603   _attributes->set_evex_encoding(evex_encoding);
10604 
10605   // P0: byte 2, initialized to RXBR`00mm
10606   // instead of not'd
10607   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
10608   byte2 = (~byte2) & 0xF0;
10609   // confine opc opcode extensions in mm bits to lower two bits
10610   // of form {0F, 0F_38, 0F_3A}
10611   byte2 |= opc;
10612 
10613   // P1: byte 3 as Wvvvv1pp
10614   int byte3 = ((~nds_enc) & 0xf) << 3;
10615   // p[10] is always 1
10616   byte3 |= EVEX_F;
10617   byte3 |= (vex_w & 1) << 7;
10618   // confine pre opcode extensions in pp bits to lower two bits
10619   // of form {66, F3, F2}
10620   byte3 |= pre;
10621 
10622   // P2: byte 4 as zL'Lbv'aaa
10623   // kregs are implemented in the low 3 bits as aaa
10624   int byte4 = (_attributes->is_no_reg_mask()) ?
10625               0 :
10626               _attributes->get_embedded_opmask_register_specifier();
10627   // EVEX.v` for extending EVEX.vvvv or VIDX
10628   byte4 |= (evex_v ? 0: EVEX_V);
10629   // third EXEC.b for broadcast actions
10630   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
10631   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
10632   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
10633   // last is EVEX.z for zero/merge actions
10634   if (_attributes->is_no_reg_mask() == false &&
10635       _attributes->get_embedded_opmask_register_specifier() != 0) {
10636     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
10637   }
10638 
10639   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
10640 }
10641 
10642 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
10643   bool vex_r = (xreg_enc & 8) == 8;
10644   bool vex_b = adr.base_needs_rex();
10645   bool vex_x;
10646   if (adr.isxmmindex()) {
10647     vex_x = adr.xmmindex_needs_rex();
10648   } else {
10649     vex_x = adr.index_needs_rex();
10650   }
10651   set_attributes(attributes);
10652   attributes->set_current_assembler(this);
10653 
10654   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
10655   // is allowed in legacy mode and has resources which will fit in it.
10656   // Pure EVEX instructions will have is_evex_instruction set in their definition.
10657   if (!attributes->is_legacy_mode()) {
10658     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
10659       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
10660           attributes->set_is_legacy_mode();
10661       }
10662     }
10663   }
10664 
10665   if (UseAVX > 2) {
10666     assert(((!attributes->uses_vl()) ||
10667             (attributes->get_vector_len() == AVX_512bit) ||
10668             (!_legacy_mode_vl) ||
10669             (attributes->is_legacy_mode())),"XMM register should be 0-15");
10670     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
10671   }
10672 
10673   clear_managed();
10674   if (UseAVX > 2 && !attributes->is_legacy_mode())
10675   {
10676     bool evex_r = (xreg_enc >= 16);
10677     bool evex_v;
10678     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
10679     if (adr.isxmmindex())  {
10680       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
10681     } else {
10682       evex_v = (nds_enc >= 16);
10683     }
10684     attributes->set_is_evex_instruction();
10685     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
10686   } else {
10687     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
10688       attributes->set_rex_vex_w(false);
10689     }
10690     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
10691   }
10692 }
10693 
10694 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
10695   bool vex_r = (dst_enc & 8) == 8;
10696   bool vex_b = (src_enc & 8) == 8;
10697   bool vex_x = false;
10698   set_attributes(attributes);
10699   attributes->set_current_assembler(this);
10700 
10701   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
10702   // is allowed in legacy mode and has resources which will fit in it.
10703   // Pure EVEX instructions will have is_evex_instruction set in their definition.
10704   if (!attributes->is_legacy_mode()) {
10705     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
10706       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
10707           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
10708           attributes->set_is_legacy_mode();
10709       }
10710     }
10711   }
10712 
10713   if (UseAVX > 2) {
10714     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
10715     // Instruction with uses_vl true are vector instructions
10716     // All the vector instructions with AVX_512bit length can have legacy_mode as false
10717     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
10718     // Rest all should have legacy_mode set as true
10719     assert(((!attributes->uses_vl()) ||
10720             (attributes->get_vector_len() == AVX_512bit) ||
10721             (!_legacy_mode_vl) ||
10722             (attributes->is_legacy_mode())),"XMM register should be 0-15");
10723     // Instruction with legacy_mode true should have dst, nds and src < 15
10724     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
10725   }
10726 
10727   clear_managed();
10728   if (UseAVX > 2 && !attributes->is_legacy_mode())
10729   {
10730     bool evex_r = (dst_enc >= 16);
10731     bool evex_v = (nds_enc >= 16);
10732     // can use vex_x as bank extender on rm encoding
10733     vex_x = (src_enc >= 16);
10734     attributes->set_is_evex_instruction();
10735     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
10736   } else {
10737     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
10738       attributes->set_rex_vex_w(false);
10739     }
10740     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
10741   }
10742 
10743   // return modrm byte components for operands
10744   return (((dst_enc & 7) << 3) | (src_enc & 7));
10745 }
10746 
10747 
10748 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
10749                             VexOpcode opc, InstructionAttr *attributes) {
10750   if (UseAVX > 0) {
10751     int xreg_enc = xreg->encoding();
10752     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
10753     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
10754   } else {
10755     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
10756     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
10757   }
10758 }
10759 
10760 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
10761                                       VexOpcode opc, InstructionAttr *attributes) {
10762   int dst_enc = dst->encoding();
10763   int src_enc = src->encoding();
10764   if (UseAVX > 0) {
10765     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
10766     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
10767   } else {
10768     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
10769     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
10770   }
10771 }
10772 
10773 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
10774   assert(VM_Version::supports_avx(), "");
10775   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10776   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10777   emit_int16(0x5F, (0xC0 | encode));
10778 }
10779 
10780 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
10781   assert(VM_Version::supports_avx(), "");
10782   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10783   attributes.set_rex_vex_w_reverted();
10784   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10785   emit_int16(0x5F, (0xC0 | encode));
10786 }
10787 
10788 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
10789   assert(VM_Version::supports_avx(), "");
10790   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10792   emit_int16(0x5D, (0xC0 | encode));
10793 }
10794 
10795 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
10796   assert(VM_Version::supports_avx(), "");
10797   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10798   attributes.set_rex_vex_w_reverted();
10799   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10800   emit_int16(0x5D, (0xC0 | encode));
10801 }
10802 
10803 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
10804   assert(VM_Version::supports_avx(), "");
10805   assert(vector_len <= AVX_256bit, "");
10806   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10807   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10808   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
10809 }
10810 
10811 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
10812   assert(VM_Version::supports_avx(), "");
10813   assert(vector_len <= AVX_256bit, "");
10814   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10815   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10816   int src2_enc = src2->encoding();
10817   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
10818 }
10819 
10820 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
10821   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
10822   assert(vector_len <= AVX_256bit, "");
10823   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10824   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10825   int src2_enc = src2->encoding();
10826   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
10827 }
10828 
10829 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
10830   assert(VM_Version::supports_avx2(), "");
10831   assert(vector_len <= AVX_256bit, "");
10832   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10833   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10834   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
10835 }
10836 
10837 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
10838   assert(VM_Version::supports_avx(), "");
10839   assert(vector_len <= AVX_256bit, "");
10840   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10841   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10842   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
10843 }
10844 
10845 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
10846                         ComparisonPredicateFP comparison, int vector_len) {
10847   assert(VM_Version::supports_evex(), "");
10848   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
10849   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10850   attributes.set_is_evex_instruction();
10851   attributes.set_embedded_opmask_register_specifier(mask);
10852   attributes.reset_is_clear_context();
10853   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10854   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
10855 }
10856 
10857 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
10858                         ComparisonPredicateFP comparison, int vector_len) {
10859   assert(VM_Version::supports_evex(), "");
10860   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
10861   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10862   attributes.set_is_evex_instruction();
10863   attributes.set_embedded_opmask_register_specifier(mask);
10864   attributes.reset_is_clear_context();
10865   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10866   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
10867 }
10868 
10869 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
10870   assert(VM_Version::supports_sse4_1(), "");
10871   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
10872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10873   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10874   emit_int16(0x14, (0xC0 | encode));
10875 }
10876 
10877 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
10878   assert(VM_Version::supports_sse4_1(), "");
10879   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
10880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10882   emit_int16(0x15, (0xC0 | encode));
10883 }
10884 
10885 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
10886   assert(VM_Version::supports_sse4_1(), "");
10887   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
10888   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10889   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10890   emit_int16(0x10, (0xC0 | encode));
10891 }
10892 
10893 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
10894   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
10895   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10896   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10897   int src2_enc = src2->encoding();
10898   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
10899 }
10900 
10901 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
10902   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10904   emit_int24(0x0C, (0xC0 | encode), imm8);
10905 }
10906 
10907 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10908   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
10909   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
10910   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10911   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10912   emit_int16(0x64, (0xC0 | encode));
10913 }
10914 
10915 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10916   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
10917   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
10918   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10919   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10920   emit_int16(0x65, (0xC0 | encode));
10921 }
10922 
10923 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10924   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
10925   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
10926   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10927   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10928   emit_int16(0x66, (0xC0 | encode));
10929 }
10930 
10931 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10932   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
10933   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
10934   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10935   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10936   emit_int16(0x37, (0xC0 | encode));
10937 }
10938 
10939 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
10940                         int comparison, bool is_signed, int vector_len) {
10941   assert(VM_Version::supports_evex(), "");
10942   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
10943   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
10944   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10945   attributes.set_is_evex_instruction();
10946   attributes.set_embedded_opmask_register_specifier(mask);
10947   attributes.reset_is_clear_context();
10948   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10949   int opcode = is_signed ? 0x1F : 0x1E;
10950   emit_int24(opcode, (0xC0 | encode), comparison);
10951 }
10952 
10953 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
10954                         int comparison, bool is_signed, int vector_len) {
10955   assert(VM_Version::supports_evex(), "");
10956   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
10957   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
10958   InstructionMark im(this);
10959   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10960   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
10961   attributes.set_is_evex_instruction();
10962   attributes.set_embedded_opmask_register_specifier(mask);
10963   attributes.reset_is_clear_context();
10964   int dst_enc = kdst->encoding();
10965   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10966   int opcode = is_signed ? 0x1F : 0x1E;
10967   emit_int8((unsigned char)opcode);
10968   emit_operand(as_Register(dst_enc), src);
10969   emit_int8((unsigned char)comparison);
10970 }
10971 
10972 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
10973                         int comparison, bool is_signed, int vector_len) {
10974   assert(VM_Version::supports_evex(), "");
10975   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
10976   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
10977   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10978   attributes.set_is_evex_instruction();
10979   attributes.set_embedded_opmask_register_specifier(mask);
10980   attributes.reset_is_clear_context();
10981   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10982   int opcode = is_signed ? 0x1F : 0x1E;
10983   emit_int24(opcode, (0xC0 | encode), comparison);
10984 }
10985 
10986 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
10987                         int comparison, bool is_signed, int vector_len) {
10988   assert(VM_Version::supports_evex(), "");
10989   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
10990   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
10991   InstructionMark im(this);
10992   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10993   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
10994   attributes.set_is_evex_instruction();
10995   attributes.set_embedded_opmask_register_specifier(mask);
10996   attributes.reset_is_clear_context();
10997   int dst_enc = kdst->encoding();
10998   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10999   int opcode = is_signed ? 0x1F : 0x1E;
11000   emit_int8((unsigned char)opcode);
11001   emit_operand(as_Register(dst_enc), src);
11002   emit_int8((unsigned char)comparison);
11003 }
11004 
11005 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11006                         int comparison, bool is_signed, int vector_len) {
11007   assert(VM_Version::supports_evex(), "");
11008   assert(VM_Version::supports_avx512bw(), "");
11009   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11010   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11011   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11012   attributes.set_is_evex_instruction();
11013   attributes.set_embedded_opmask_register_specifier(mask);
11014   attributes.reset_is_clear_context();
11015   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11016   int opcode = is_signed ? 0x3F : 0x3E;
11017   emit_int24(opcode, (0xC0 | encode), comparison);
11018 }
11019 
11020 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11021                         int comparison, bool is_signed, int vector_len) {
11022   assert(VM_Version::supports_evex(), "");
11023   assert(VM_Version::supports_avx512bw(), "");
11024   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11025   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11026   InstructionMark im(this);
11027   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11028   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11029   attributes.set_is_evex_instruction();
11030   attributes.set_embedded_opmask_register_specifier(mask);
11031   attributes.reset_is_clear_context();
11032   int dst_enc = kdst->encoding();
11033   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11034   int opcode = is_signed ? 0x3F : 0x3E;
11035   emit_int8((unsigned char)opcode);
11036   emit_operand(as_Register(dst_enc), src);
11037   emit_int8((unsigned char)comparison);
11038 }
11039 
11040 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11041                         int comparison, bool is_signed, int vector_len) {
11042   assert(VM_Version::supports_evex(), "");
11043   assert(VM_Version::supports_avx512bw(), "");
11044   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11045   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11046   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11047   attributes.set_is_evex_instruction();
11048   attributes.set_embedded_opmask_register_specifier(mask);
11049   attributes.reset_is_clear_context();
11050   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11051   int opcode = is_signed ? 0x3F : 0x3E;
11052   emit_int24(opcode, (0xC0 | encode), comparison);
11053 }
11054 
11055 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11056                         int comparison, bool is_signed, int vector_len) {
11057   assert(VM_Version::supports_evex(), "");
11058   assert(VM_Version::supports_avx512bw(), "");
11059   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11060   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11061   InstructionMark im(this);
11062   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11063   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11064   attributes.set_is_evex_instruction();
11065   attributes.set_embedded_opmask_register_specifier(mask);
11066   attributes.reset_is_clear_context();
11067   int dst_enc = kdst->encoding();
11068   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11069   int opcode = is_signed ? 0x3F : 0x3E;
11070   emit_int8((unsigned char)opcode);
11071   emit_operand(as_Register(dst_enc), src);
11072   emit_int8((unsigned char)comparison);
11073 }
11074 
11075 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11076   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11077   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11078   attributes.set_is_evex_instruction();
11079   attributes.set_embedded_opmask_register_specifier(mask);
11080   if (merge) {
11081     attributes.reset_is_clear_context();
11082   }
11083   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11084   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11085 }
11086 
11087 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11088   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11089   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11090   attributes.set_is_evex_instruction();
11091   attributes.set_embedded_opmask_register_specifier(mask);
11092   if (merge) {
11093     attributes.reset_is_clear_context();
11094   }
11095   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11096   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11097 }
11098 
11099 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11100   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11102   attributes.set_is_evex_instruction();
11103   attributes.set_embedded_opmask_register_specifier(mask);
11104   if (merge) {
11105     attributes.reset_is_clear_context();
11106   }
11107   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11108   emit_int16(0x14, (0xC0 | encode));
11109 }
11110 
11111 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11112   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11113   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11114   attributes.set_is_evex_instruction();
11115   attributes.set_embedded_opmask_register_specifier(mask);
11116   if (merge) {
11117     attributes.reset_is_clear_context();
11118   }
11119   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11120   emit_int16(0x14, (0xC0 | encode));
11121 }
11122 
11123 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11124   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11126   attributes.set_is_evex_instruction();
11127   attributes.set_embedded_opmask_register_specifier(mask);
11128   if (merge) {
11129     attributes.reset_is_clear_context();
11130   }
11131   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11132   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11133 }
11134 
11135 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11136   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11137   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11138   attributes.set_is_evex_instruction();
11139   attributes.set_embedded_opmask_register_specifier(mask);
11140   if (merge) {
11141     attributes.reset_is_clear_context();
11142   }
11143   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11144   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11145 }
11146 
11147 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11148   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11149   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11150   attributes.set_is_evex_instruction();
11151   attributes.set_embedded_opmask_register_specifier(mask);
11152   if (merge) {
11153     attributes.reset_is_clear_context();
11154   }
11155   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11156   emit_int16(0x15, (0xC0 | encode));
11157 }
11158 
11159 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11160   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11161   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11162   attributes.set_is_evex_instruction();
11163   attributes.set_embedded_opmask_register_specifier(mask);
11164   if (merge) {
11165     attributes.reset_is_clear_context();
11166   }
11167   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11168   emit_int16(0x15, (0xC0 | encode));
11169 }
11170 
11171 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11172   assert(VM_Version::supports_avx(), "");
11173   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11174   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11175   int mask_enc = mask->encoding();
11176   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11177 }
11178 
11179 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11180   assert(VM_Version::supports_evex(), "");
11181   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11182   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11183   attributes.set_is_evex_instruction();
11184   attributes.set_embedded_opmask_register_specifier(mask);
11185   if (merge) {
11186     attributes.reset_is_clear_context();
11187   }
11188   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11189   emit_int16(0x65, (0xC0 | encode));
11190 }
11191 
11192 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11193   assert(VM_Version::supports_evex(), "");
11194   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11195   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11196   attributes.set_is_evex_instruction();
11197   attributes.set_embedded_opmask_register_specifier(mask);
11198   if (merge) {
11199     attributes.reset_is_clear_context();
11200   }
11201   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11202   emit_int16(0x65, (0xC0 | encode));
11203 }
11204 
11205 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11206   assert(VM_Version::supports_evex(), "");
11207   assert(VM_Version::supports_avx512bw(), "");
11208   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11209   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11210   attributes.set_is_evex_instruction();
11211   attributes.set_embedded_opmask_register_specifier(mask);
11212   if (merge) {
11213     attributes.reset_is_clear_context();
11214   }
11215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11216   emit_int16(0x66, (0xC0 | encode));
11217 }
11218 
11219 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11220   assert(VM_Version::supports_evex(), "");
11221   assert(VM_Version::supports_avx512bw(), "");
11222   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11223   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11224   attributes.set_is_evex_instruction();
11225   attributes.set_embedded_opmask_register_specifier(mask);
11226   if (merge) {
11227     attributes.reset_is_clear_context();
11228   }
11229   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11230   emit_int16(0x66, (0xC0 | encode));
11231 }
11232 
11233 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11234   assert(VM_Version::supports_evex(), "");
11235   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11236   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11237   attributes.set_is_evex_instruction();
11238   attributes.set_embedded_opmask_register_specifier(mask);
11239   if (merge) {
11240     attributes.reset_is_clear_context();
11241   }
11242   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11243   emit_int16(0x64, (0xC0 | encode));
11244 }
11245 
11246 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11247   assert(VM_Version::supports_evex(), "");
11248   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11249   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11250   attributes.set_is_evex_instruction();
11251   attributes.set_embedded_opmask_register_specifier(mask);
11252   if (merge) {
11253     attributes.reset_is_clear_context();
11254   }
11255   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11256   emit_int16(0x64, (0xC0 | encode));
11257 }
11258 
11259 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11260   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11261   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11262   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11263   emit_int16((unsigned char)0xF5, (0xC0 | encode));
11264 }
11265 
11266 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11267   assert(VM_Version::supports_bmi2(), "");
11268   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11269   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11270   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11271 }
11272 
11273 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11274   assert(VM_Version::supports_bmi2(), "");
11275   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11276   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11277   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11278 }
11279 
11280 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11281   assert(VM_Version::supports_bmi2(), "");
11282   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11283   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11284   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11285 }
11286 
11287 void Assembler::shrxq(Register dst, Register src1, Register src2) {
11288   assert(VM_Version::supports_bmi2(), "");
11289   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11290   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11291   emit_int16((unsigned char)0xF7, (0xC0 | encode));
11292 }
11293 
11294 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
11295   assert(VM_Version::supports_avx512vldq(), "");
11296   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11297   attributes.set_is_evex_instruction();
11298   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11299   emit_int16(0x39, (0xC0 | encode));
11300 }
11301 
11302 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
11303   assert(VM_Version::supports_avx512vldq(), "");
11304   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11305   attributes.set_is_evex_instruction();
11306   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11307   emit_int16(0x39, (0xC0 | encode));
11308 }
11309 
11310 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
11311   assert(VM_Version::supports_avx512vlbw(), "");
11312   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11313   attributes.set_is_evex_instruction();
11314   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11315   emit_int16(0x29, (0xC0 | encode));
11316 }
11317 
11318 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
11319   assert(VM_Version::supports_avx512vlbw(), "");
11320   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11321   attributes.set_is_evex_instruction();
11322   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11323   emit_int16(0x29, (0xC0 | encode));
11324 }
11325 
11326 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
11327   assert(VM_Version::supports_avx512vldq(), "");
11328   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11329   attributes.set_is_evex_instruction();
11330   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11331   emit_int16(0x38, (0xC0 | encode));
11332 }
11333 
11334 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
11335   assert(VM_Version::supports_avx512vldq(), "");
11336   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11337   attributes.set_is_evex_instruction();
11338   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11339   emit_int16(0x38, (0xC0 | encode));
11340 }
11341 
11342 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
11343   assert(VM_Version::supports_avx512vlbw(), "");
11344   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11345   attributes.set_is_evex_instruction();
11346   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11347   emit_int16(0x28, (0xC0 | encode));
11348 }
11349 
11350 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
11351   assert(VM_Version::supports_avx512vlbw(), "");
11352   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11353   attributes.set_is_evex_instruction();
11354   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11355   emit_int16(0x28, (0xC0 | encode));
11356 }
11357 #ifndef _LP64
11358 
11359 void Assembler::incl(Register dst) {
11360   // Don't use it directly. Use MacroAssembler::incrementl() instead.
11361   emit_int8(0x40 | dst->encoding());
11362 }
11363 
11364 void Assembler::lea(Register dst, Address src) {
11365   leal(dst, src);
11366 }
11367 
11368 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
11369   InstructionMark im(this);
11370   emit_int8((unsigned char)0xC7);
11371   emit_operand(rax, dst);
11372   emit_data((int)imm32, rspec, 0);
11373 }
11374 
11375 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
11376   InstructionMark im(this);
11377   int encode = prefix_and_encode(dst->encoding());
11378   emit_int8((0xB8 | encode));
11379   emit_data((int)imm32, rspec, 0);
11380 }
11381 
11382 void Assembler::popa() { // 32bit
11383   emit_int8(0x61);
11384 }
11385 
11386 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
11387   InstructionMark im(this);
11388   emit_int8(0x68);
11389   emit_data(imm32, rspec, 0);
11390 }
11391 
11392 void Assembler::pusha() { // 32bit
11393   emit_int8(0x60);
11394 }
11395 
11396 void Assembler::set_byte_if_not_zero(Register dst) {
11397   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
11398 }
11399 
11400 #else // LP64
11401 
11402 void Assembler::set_byte_if_not_zero(Register dst) {
11403   int enc = prefix_and_encode(dst->encoding(), true);
11404   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
11405 }
11406 
11407 // 64bit only pieces of the assembler
11408 // This should only be used by 64bit instructions that can use rip-relative
11409 // it cannot be used by instructions that want an immediate value.
11410 
11411 bool Assembler::reachable(AddressLiteral adr) {
11412   int64_t disp;
11413   relocInfo::relocType relocType = adr.reloc();
11414 
11415   // None will force a 64bit literal to the code stream. Likely a placeholder
11416   // for something that will be patched later and we need to certain it will
11417   // always be reachable.
11418   if (relocType == relocInfo::none) {
11419     return false;
11420   }
11421   if (relocType == relocInfo::internal_word_type) {
11422     // This should be rip relative and easily reachable.
11423     return true;
11424   }
11425   if (relocType == relocInfo::virtual_call_type ||
11426       relocType == relocInfo::opt_virtual_call_type ||
11427       relocType == relocInfo::static_call_type ||
11428       relocType == relocInfo::static_stub_type ) {
11429     // This should be rip relative within the code cache and easily
11430     // reachable until we get huge code caches. (At which point
11431     // ic code is going to have issues).
11432     return true;
11433   }
11434   if (relocType != relocInfo::external_word_type &&
11435       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
11436       relocType != relocInfo::poll_type &&         // relocs to identify them
11437       relocType != relocInfo::runtime_call_type ) {
11438     return false;
11439   }
11440 
11441   // Stress the correction code
11442   if (ForceUnreachable) {
11443     // Must be runtimecall reloc, see if it is in the codecache
11444     // Flipping stuff in the codecache to be unreachable causes issues
11445     // with things like inline caches where the additional instructions
11446     // are not handled.
11447     if (CodeCache::find_blob(adr._target) == NULL) {
11448       return false;
11449     }
11450   }
11451   // For external_word_type/runtime_call_type if it is reachable from where we
11452   // are now (possibly a temp buffer) and where we might end up
11453   // anywhere in the codeCache then we are always reachable.
11454   // This would have to change if we ever save/restore shared code
11455   // to be more pessimistic.
11456   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
11457   if (!is_simm32(disp)) return false;
11458   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
11459   if (!is_simm32(disp)) return false;
11460 
11461   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
11462 
11463   // Because rip relative is a disp + address_of_next_instruction and we
11464   // don't know the value of address_of_next_instruction we apply a fudge factor
11465   // to make sure we will be ok no matter the size of the instruction we get placed into.
11466   // We don't have to fudge the checks above here because they are already worst case.
11467 
11468   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
11469   // + 4 because better safe than sorry.
11470   const int fudge = 12 + 4;
11471   if (disp < 0) {
11472     disp -= fudge;
11473   } else {
11474     disp += fudge;
11475   }
11476   return is_simm32(disp);
11477 }
11478 
11479 void Assembler::emit_data64(jlong data,
11480                             relocInfo::relocType rtype,
11481                             int format) {
11482   if (rtype == relocInfo::none) {
11483     emit_int64(data);
11484   } else {
11485     emit_data64(data, Relocation::spec_simple(rtype), format);
11486   }
11487 }
11488 
11489 void Assembler::emit_data64(jlong data,
11490                             RelocationHolder const& rspec,
11491                             int format) {
11492   assert(imm_operand == 0, "default format must be immediate in this file");
11493   assert(imm_operand == format, "must be immediate");
11494   assert(inst_mark() != NULL, "must be inside InstructionMark");
11495   // Do not use AbstractAssembler::relocate, which is not intended for
11496   // embedded words.  Instead, relocate to the enclosing instruction.
11497   code_section()->relocate(inst_mark(), rspec, format);
11498 #ifdef ASSERT
11499   check_relocation(rspec, format);
11500 #endif
11501   emit_int64(data);
11502 }
11503 
11504 void Assembler::prefix(Register reg) {
11505   if (reg->encoding() >= 8) {
11506     prefix(REX_B);
11507   }
11508 }
11509 
11510 void Assembler::prefix(Register dst, Register src, Prefix p) {
11511   if (src->encoding() >= 8) {
11512     p = (Prefix)(p | REX_B);
11513   }
11514   if (dst->encoding() >= 8) {
11515     p = (Prefix)(p | REX_R);
11516   }
11517   if (p != Prefix_EMPTY) {
11518     // do not generate an empty prefix
11519     prefix(p);
11520   }
11521 }
11522 
11523 void Assembler::prefix(Register dst, Address adr, Prefix p) {
11524   if (adr.base_needs_rex()) {
11525     if (adr.index_needs_rex()) {
11526       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
11527     } else {
11528       prefix(REX_B);
11529     }
11530   } else {
11531     if (adr.index_needs_rex()) {
11532       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
11533     }
11534   }
11535   if (dst->encoding() >= 8) {
11536     p = (Prefix)(p | REX_R);
11537   }
11538   if (p != Prefix_EMPTY) {
11539     // do not generate an empty prefix
11540     prefix(p);
11541   }
11542 }
11543 
11544 void Assembler::prefix(Address adr) {
11545   if (adr.base_needs_rex()) {
11546     if (adr.index_needs_rex()) {
11547       prefix(REX_XB);
11548     } else {
11549       prefix(REX_B);
11550     }
11551   } else {
11552     if (adr.index_needs_rex()) {
11553       prefix(REX_X);
11554     }
11555   }
11556 }
11557 
11558 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
11559   if (reg->encoding() < 8) {
11560     if (adr.base_needs_rex()) {
11561       if (adr.index_needs_rex()) {
11562         prefix(REX_XB);
11563       } else {
11564         prefix(REX_B);
11565       }
11566     } else {
11567       if (adr.index_needs_rex()) {
11568         prefix(REX_X);
11569       } else if (byteinst && reg->encoding() >= 4) {
11570         prefix(REX);
11571       }
11572     }
11573   } else {
11574     if (adr.base_needs_rex()) {
11575       if (adr.index_needs_rex()) {
11576         prefix(REX_RXB);
11577       } else {
11578         prefix(REX_RB);
11579       }
11580     } else {
11581       if (adr.index_needs_rex()) {
11582         prefix(REX_RX);
11583       } else {
11584         prefix(REX_R);
11585       }
11586     }
11587   }
11588 }
11589 
11590 void Assembler::prefix(Address adr, XMMRegister reg) {
11591   if (reg->encoding() < 8) {
11592     if (adr.base_needs_rex()) {
11593       if (adr.index_needs_rex()) {
11594         prefix(REX_XB);
11595       } else {
11596         prefix(REX_B);
11597       }
11598     } else {
11599       if (adr.index_needs_rex()) {
11600         prefix(REX_X);
11601       }
11602     }
11603   } else {
11604     if (adr.base_needs_rex()) {
11605       if (adr.index_needs_rex()) {
11606         prefix(REX_RXB);
11607       } else {
11608         prefix(REX_RB);
11609       }
11610     } else {
11611       if (adr.index_needs_rex()) {
11612         prefix(REX_RX);
11613       } else {
11614         prefix(REX_R);
11615       }
11616     }
11617   }
11618 }
11619 
11620 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
11621   if (reg_enc >= 8) {
11622     prefix(REX_B);
11623     reg_enc -= 8;
11624   } else if (byteinst && reg_enc >= 4) {
11625     prefix(REX);
11626   }
11627   return reg_enc;
11628 }
11629 
11630 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
11631   if (dst_enc < 8) {
11632     if (src_enc >= 8) {
11633       prefix(REX_B);
11634       src_enc -= 8;
11635     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
11636       prefix(REX);
11637     }
11638   } else {
11639     if (src_enc < 8) {
11640       prefix(REX_R);
11641     } else {
11642       prefix(REX_RB);
11643       src_enc -= 8;
11644     }
11645     dst_enc -= 8;
11646   }
11647   return dst_enc << 3 | src_enc;
11648 }
11649 
11650 int8_t Assembler::get_prefixq(Address adr) {
11651   int8_t prfx = get_prefixq(adr, rax);
11652   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
11653   return prfx;
11654 }
11655 
11656 int8_t Assembler::get_prefixq(Address adr, Register src) {
11657   int8_t prfx = (int8_t)(REX_W +
11658                          ((int)adr.base_needs_rex()) +
11659                          ((int)adr.index_needs_rex() << 1) +
11660                          ((int)(src->encoding() >= 8) << 2));
11661 #ifdef ASSERT
11662   if (src->encoding() < 8) {
11663     if (adr.base_needs_rex()) {
11664       if (adr.index_needs_rex()) {
11665         assert(prfx == REX_WXB, "must be");
11666       } else {
11667         assert(prfx == REX_WB, "must be");
11668       }
11669     } else {
11670       if (adr.index_needs_rex()) {
11671         assert(prfx == REX_WX, "must be");
11672       } else {
11673         assert(prfx == REX_W, "must be");
11674       }
11675     }
11676   } else {
11677     if (adr.base_needs_rex()) {
11678       if (adr.index_needs_rex()) {
11679         assert(prfx == REX_WRXB, "must be");
11680       } else {
11681         assert(prfx == REX_WRB, "must be");
11682       }
11683     } else {
11684       if (adr.index_needs_rex()) {
11685         assert(prfx == REX_WRX, "must be");
11686       } else {
11687         assert(prfx == REX_WR, "must be");
11688       }
11689     }
11690   }
11691 #endif
11692   return prfx;
11693 }
11694 
11695 void Assembler::prefixq(Address adr) {
11696   emit_int8(get_prefixq(adr));
11697 }
11698 
11699 void Assembler::prefixq(Address adr, Register src) {
11700   emit_int8(get_prefixq(adr, src));
11701 }
11702 
11703 void Assembler::prefixq(Address adr, XMMRegister src) {
11704   if (src->encoding() < 8) {
11705     if (adr.base_needs_rex()) {
11706       if (adr.index_needs_rex()) {
11707         prefix(REX_WXB);
11708       } else {
11709         prefix(REX_WB);
11710       }
11711     } else {
11712       if (adr.index_needs_rex()) {
11713         prefix(REX_WX);
11714       } else {
11715         prefix(REX_W);
11716       }
11717     }
11718   } else {
11719     if (adr.base_needs_rex()) {
11720       if (adr.index_needs_rex()) {
11721         prefix(REX_WRXB);
11722       } else {
11723         prefix(REX_WRB);
11724       }
11725     } else {
11726       if (adr.index_needs_rex()) {
11727         prefix(REX_WRX);
11728       } else {
11729         prefix(REX_WR);
11730       }
11731     }
11732   }
11733 }
11734 
11735 int Assembler::prefixq_and_encode(int reg_enc) {
11736   if (reg_enc < 8) {
11737     prefix(REX_W);
11738   } else {
11739     prefix(REX_WB);
11740     reg_enc -= 8;
11741   }
11742   return reg_enc;
11743 }
11744 
11745 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
11746   if (dst_enc < 8) {
11747     if (src_enc < 8) {
11748       prefix(REX_W);
11749     } else {
11750       prefix(REX_WB);
11751       src_enc -= 8;
11752     }
11753   } else {
11754     if (src_enc < 8) {
11755       prefix(REX_WR);
11756     } else {
11757       prefix(REX_WRB);
11758       src_enc -= 8;
11759     }
11760     dst_enc -= 8;
11761   }
11762   return dst_enc << 3 | src_enc;
11763 }
11764 
11765 void Assembler::adcq(Register dst, int32_t imm32) {
11766   (void) prefixq_and_encode(dst->encoding());
11767   emit_arith(0x81, 0xD0, dst, imm32);
11768 }
11769 
11770 void Assembler::adcq(Register dst, Address src) {
11771   InstructionMark im(this);
11772   emit_int16(get_prefixq(src, dst), 0x13);
11773   emit_operand(dst, src);
11774 }
11775 
11776 void Assembler::adcq(Register dst, Register src) {
11777   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11778   emit_arith(0x13, 0xC0, dst, src);
11779 }
11780 
11781 void Assembler::addq(Address dst, int32_t imm32) {
11782   InstructionMark im(this);
11783   prefixq(dst);
11784   emit_arith_operand(0x81, rax, dst, imm32);
11785 }
11786 
11787 void Assembler::addq(Address dst, Register src) {
11788   InstructionMark im(this);
11789   emit_int16(get_prefixq(dst, src), 0x01);
11790   emit_operand(src, dst);
11791 }
11792 
11793 void Assembler::addq(Register dst, int32_t imm32) {
11794   (void) prefixq_and_encode(dst->encoding());
11795   emit_arith(0x81, 0xC0, dst, imm32);
11796 }
11797 
11798 void Assembler::addq(Register dst, Address src) {
11799   InstructionMark im(this);
11800   emit_int16(get_prefixq(src, dst), 0x03);
11801   emit_operand(dst, src);
11802 }
11803 
11804 void Assembler::addq(Register dst, Register src) {
11805   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11806   emit_arith(0x03, 0xC0, dst, src);
11807 }
11808 
11809 void Assembler::adcxq(Register dst, Register src) {
11810   //assert(VM_Version::supports_adx(), "adx instructions not supported");
11811   emit_int8(0x66);
11812   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11813   emit_int32(0x0F,
11814              0x38,
11815              (unsigned char)0xF6,
11816              (0xC0 | encode));
11817 }
11818 
11819 void Assembler::adoxq(Register dst, Register src) {
11820   //assert(VM_Version::supports_adx(), "adx instructions not supported");
11821   emit_int8((unsigned char)0xF3);
11822   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11823   emit_int32(0x0F,
11824              0x38,
11825              (unsigned char)0xF6,
11826              (0xC0 | encode));
11827 }
11828 
11829 void Assembler::andq(Address dst, int32_t imm32) {
11830   InstructionMark im(this);
11831   prefixq(dst);
11832   emit_arith_operand(0x81, as_Register(4), dst, imm32);
11833 }
11834 
11835 void Assembler::andq(Register dst, int32_t imm32) {
11836   (void) prefixq_and_encode(dst->encoding());
11837   emit_arith(0x81, 0xE0, dst, imm32);
11838 }
11839 
11840 void Assembler::andq(Register dst, Address src) {
11841   InstructionMark im(this);
11842   emit_int16(get_prefixq(src, dst), 0x23);
11843   emit_operand(dst, src);
11844 }
11845 
11846 void Assembler::andq(Register dst, Register src) {
11847   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11848   emit_arith(0x23, 0xC0, dst, src);
11849 }
11850 
11851 void Assembler::andq(Address dst, Register src) {
11852   InstructionMark im(this);
11853   emit_int16(get_prefixq(dst, src), 0x21);
11854   emit_operand(src, dst);
11855 }
11856 
11857 void Assembler::andnq(Register dst, Register src1, Register src2) {
11858   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11859   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11860   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11861   emit_int16((unsigned char)0xF2, (0xC0 | encode));
11862 }
11863 
11864 void Assembler::andnq(Register dst, Register src1, Address src2) {
11865   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11866   InstructionMark im(this);
11867   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11868   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11869   emit_int8((unsigned char)0xF2);
11870   emit_operand(dst, src2);
11871 }
11872 
11873 void Assembler::bsfq(Register dst, Register src) {
11874   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11875   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
11876 }
11877 
11878 void Assembler::bsrq(Register dst, Register src) {
11879   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11880   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
11881 }
11882 
11883 void Assembler::bswapq(Register reg) {
11884   int encode = prefixq_and_encode(reg->encoding());
11885   emit_int16(0x0F, (0xC8 | encode));
11886 }
11887 
11888 void Assembler::blsiq(Register dst, Register src) {
11889   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11890   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11891   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11892   emit_int16((unsigned char)0xF3, (0xC0 | encode));
11893 }
11894 
11895 void Assembler::blsiq(Register dst, Address src) {
11896   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11897   InstructionMark im(this);
11898   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11899   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11900   emit_int8((unsigned char)0xF3);
11901   emit_operand(rbx, src);
11902 }
11903 
11904 void Assembler::blsmskq(Register dst, Register src) {
11905   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11906   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11907   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11908   emit_int16((unsigned char)0xF3, (0xC0 | encode));
11909 }
11910 
11911 void Assembler::blsmskq(Register dst, Address src) {
11912   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11913   InstructionMark im(this);
11914   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11915   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11916   emit_int8((unsigned char)0xF3);
11917   emit_operand(rdx, src);
11918 }
11919 
11920 void Assembler::blsrq(Register dst, Register src) {
11921   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11922   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11923   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11924   emit_int16((unsigned char)0xF3, (0xC0 | encode));
11925 }
11926 
11927 void Assembler::blsrq(Register dst, Address src) {
11928   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
11929   InstructionMark im(this);
11930   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11931   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11932   emit_int8((unsigned char)0xF3);
11933   emit_operand(rcx, src);
11934 }
11935 
11936 void Assembler::cdqq() {
11937   emit_int16(REX_W, (unsigned char)0x99);
11938 }
11939 
11940 void Assembler::clflush(Address adr) {
11941   assert(VM_Version::supports_clflush(), "should do");
11942   prefix(adr);
11943   emit_int16(0x0F, (unsigned char)0xAE);
11944   emit_operand(rdi, adr);
11945 }
11946 
11947 void Assembler::clflushopt(Address adr) {
11948   assert(VM_Version::supports_clflushopt(), "should do!");
11949   // adr should be base reg only with no index or offset
11950   assert(adr.index() == noreg, "index should be noreg");
11951   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
11952   assert(adr.disp() == 0, "displacement should be 0");
11953   // instruction prefix is 0x66
11954   emit_int8(0x66);
11955   prefix(adr);
11956   // opcode family is 0x0F 0xAE
11957   emit_int16(0x0F, (unsigned char)0xAE);
11958   // extended opcode byte is 7 == rdi
11959   emit_operand(rdi, adr);
11960 }
11961 
11962 void Assembler::clwb(Address adr) {
11963   assert(VM_Version::supports_clwb(), "should do!");
11964   // adr should be base reg only with no index or offset
11965   assert(adr.index() == noreg, "index should be noreg");
11966   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
11967   assert(adr.disp() == 0, "displacement should be 0");
11968   // instruction prefix is 0x66
11969   emit_int8(0x66);
11970   prefix(adr);
11971   // opcode family is 0x0f 0xAE
11972   emit_int16(0x0F, (unsigned char)0xAE);
11973   // extended opcode byte is 6 == rsi
11974   emit_operand(rsi, adr);
11975 }
11976 
11977 void Assembler::cmovq(Condition cc, Register dst, Register src) {
11978   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11979   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
11980 }
11981 
11982 void Assembler::cmovq(Condition cc, Register dst, Address src) {
11983   InstructionMark im(this);
11984   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
11985   emit_operand(dst, src);
11986 }
11987 
11988 void Assembler::cmpq(Address dst, int32_t imm32) {
11989   InstructionMark im(this);
11990   emit_int16(get_prefixq(dst), (unsigned char)0x81);
11991   emit_operand(rdi, dst, 4);
11992   emit_int32(imm32);
11993 }
11994 
11995 void Assembler::cmpq(Register dst, int32_t imm32) {
11996   (void) prefixq_and_encode(dst->encoding());
11997   emit_arith(0x81, 0xF8, dst, imm32);
11998 }
11999 
12000 void Assembler::cmpq(Address dst, Register src) {
12001   InstructionMark im(this);
12002   emit_int16(get_prefixq(dst, src), 0x39);
12003   emit_operand(src, dst);
12004 }
12005 
12006 void Assembler::cmpq(Register dst, Register src) {
12007   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12008   emit_arith(0x3B, 0xC0, dst, src);
12009 }
12010 
12011 void Assembler::cmpq(Register dst, Address src) {
12012   InstructionMark im(this);
12013   emit_int16(get_prefixq(src, dst), 0x3B);
12014   emit_operand(dst, src);
12015 }
12016 
12017 void Assembler::cmpxchgq(Register reg, Address adr) {
12018   InstructionMark im(this);
12019   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12020   emit_operand(reg, adr);
12021 }
12022 
12023 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12024   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12025   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12026   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12027   emit_int16(0x2A, (0xC0 | encode));
12028 }
12029 
12030 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12031   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12032   InstructionMark im(this);
12033   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12034   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12035   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12036   emit_int8(0x2A);
12037   emit_operand(dst, src);
12038 }
12039 
12040 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12041   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12042   InstructionMark im(this);
12043   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12044   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12045   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12046   emit_int8(0x2A);
12047   emit_operand(dst, src);
12048 }
12049 
12050 void Assembler::cvttsd2siq(Register dst, Address src) {
12051   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12052   // F2 REX.W 0F 2C /r
12053   // CVTTSD2SI r64, xmm1/m64
12054   InstructionMark im(this);
12055   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12056   emit_operand(dst, src);
12057 }
12058 
12059 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12060   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12061   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12062   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12063   emit_int16(0x2C, (0xC0 | encode));
12064 }
12065 
12066 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12067   NOT_LP64(assert(VM_Version::supports_sse(), ""));
12068   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12069   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12070   emit_int16(0x2C, (0xC0 | encode));
12071 }
12072 
12073 void Assembler::decl(Register dst) {
12074   // Don't use it directly. Use MacroAssembler::decrementl() instead.
12075   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12076   int encode = prefix_and_encode(dst->encoding());
12077   emit_int16((unsigned char)0xFF, (0xC8 | encode));
12078 }
12079 
12080 void Assembler::decq(Register dst) {
12081   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12082   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12083   int encode = prefixq_and_encode(dst->encoding());
12084   emit_int16((unsigned char)0xFF, 0xC8 | encode);
12085 }
12086 
12087 void Assembler::decq(Address dst) {
12088   // Don't use it directly. Use MacroAssembler::decrementq() instead.
12089   InstructionMark im(this);
12090   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12091   emit_operand(rcx, dst);
12092 }
12093 
12094 void Assembler::fxrstor(Address src) {
12095   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12096   emit_operand(as_Register(1), src);
12097 }
12098 
12099 void Assembler::xrstor(Address src) {
12100   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12101   emit_operand(as_Register(5), src);
12102 }
12103 
12104 void Assembler::fxsave(Address dst) {
12105   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12106   emit_operand(as_Register(0), dst);
12107 }
12108 
12109 void Assembler::xsave(Address dst) {
12110   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12111   emit_operand(as_Register(4), dst);
12112 }
12113 
12114 void Assembler::idivq(Register src) {
12115   int encode = prefixq_and_encode(src->encoding());
12116   emit_int16((unsigned char)0xF7, (0xF8 | encode));
12117 }
12118 
12119 void Assembler::imulq(Register dst, Register src) {
12120   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12121   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12122 }
12123 
12124 void Assembler::imulq(Register src) {
12125   int encode = prefixq_and_encode(src->encoding());
12126   emit_int16((unsigned char)0xF7, (0xE8 | encode));
12127 }
12128 
12129 void Assembler::imulq(Register dst, Address src, int32_t value) {
12130   InstructionMark im(this);
12131   prefixq(src, dst);
12132   if (is8bit(value)) {
12133     emit_int8((unsigned char)0x6B);
12134     emit_operand(dst, src);
12135     emit_int8(value);
12136   } else {
12137     emit_int8((unsigned char)0x69);
12138     emit_operand(dst, src);
12139     emit_int32(value);
12140   }
12141 }
12142 
12143 void Assembler::imulq(Register dst, Register src, int value) {
12144   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12145   if (is8bit(value)) {
12146     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12147   } else {
12148     emit_int16(0x69, (0xC0 | encode));
12149     emit_int32(value);
12150   }
12151 }
12152 
12153 void Assembler::imulq(Register dst, Address src) {
12154   InstructionMark im(this);
12155   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12156   emit_operand(dst, src);
12157 }
12158 
12159 void Assembler::incl(Register dst) {
12160   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12161   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12162   int encode = prefix_and_encode(dst->encoding());
12163   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12164 }
12165 
12166 void Assembler::incq(Register dst) {
12167   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12168   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12169   int encode = prefixq_and_encode(dst->encoding());
12170   emit_int16((unsigned char)0xFF, (0xC0 | encode));
12171 }
12172 
12173 void Assembler::incq(Address dst) {
12174   // Don't use it directly. Use MacroAssembler::incrementq() instead.
12175   InstructionMark im(this);
12176   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12177   emit_operand(rax, dst);
12178 }
12179 
12180 void Assembler::lea(Register dst, Address src) {
12181   leaq(dst, src);
12182 }
12183 
12184 void Assembler::leaq(Register dst, Address src) {
12185   InstructionMark im(this);
12186   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
12187   emit_operand(dst, src);
12188 }
12189 
12190 void Assembler::mov64(Register dst, int64_t imm64) {
12191   InstructionMark im(this);
12192   int encode = prefixq_and_encode(dst->encoding());
12193   emit_int8(0xB8 | encode);
12194   emit_int64(imm64);
12195 }
12196 
12197 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
12198   InstructionMark im(this);
12199   int encode = prefixq_and_encode(dst->encoding());
12200   emit_int8(0xB8 | encode);
12201   emit_data64(imm64, rtype, format);
12202 }
12203 
12204 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
12205   InstructionMark im(this);
12206   int encode = prefixq_and_encode(dst->encoding());
12207   emit_int8(0xB8 | encode);
12208   emit_data64(imm64, rspec);
12209 }
12210 
12211 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12212   InstructionMark im(this);
12213   int encode = prefix_and_encode(dst->encoding());
12214   emit_int8(0xB8 | encode);
12215   emit_data((int)imm32, rspec, narrow_oop_operand);
12216 }
12217 
12218 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
12219   InstructionMark im(this);
12220   prefix(dst);
12221   emit_int8((unsigned char)0xC7);
12222   emit_operand(rax, dst, 4);
12223   emit_data((int)imm32, rspec, narrow_oop_operand);
12224 }
12225 
12226 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
12227   InstructionMark im(this);
12228   int encode = prefix_and_encode(src1->encoding());
12229   emit_int16((unsigned char)0x81, (0xF8 | encode));
12230   emit_data((int)imm32, rspec, narrow_oop_operand);
12231 }
12232 
12233 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
12234   InstructionMark im(this);
12235   prefix(src1);
12236   emit_int8((unsigned char)0x81);
12237   emit_operand(rax, src1, 4);
12238   emit_data((int)imm32, rspec, narrow_oop_operand);
12239 }
12240 
12241 void Assembler::lzcntq(Register dst, Register src) {
12242   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12243   emit_int8((unsigned char)0xF3);
12244   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12245   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12246 }
12247 
12248 void Assembler::movdq(XMMRegister dst, Register src) {
12249   // table D-1 says MMX/SSE2
12250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12251   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12252   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12253   emit_int16(0x6E, (0xC0 | encode));
12254 }
12255 
12256 void Assembler::movdq(Register dst, XMMRegister src) {
12257   // table D-1 says MMX/SSE2
12258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12259   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12260   // swap src/dst to get correct prefix
12261   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12262   emit_int16(0x7E,
12263              (0xC0 | encode));
12264 }
12265 
12266 void Assembler::movq(Register dst, Register src) {
12267   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12268   emit_int16((unsigned char)0x8B,
12269              (0xC0 | encode));
12270 }
12271 
12272 void Assembler::movq(Register dst, Address src) {
12273   InstructionMark im(this);
12274   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
12275   emit_operand(dst, src);
12276 }
12277 
12278 void Assembler::movq(Address dst, Register src) {
12279   InstructionMark im(this);
12280   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
12281   emit_operand(src, dst);
12282 }
12283 
12284 void Assembler::movq(Address dst, int32_t imm32) {
12285   InstructionMark im(this);
12286   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12287   emit_operand(as_Register(0), dst);
12288   emit_int32(imm32);
12289 }
12290 
12291 void Assembler::movq(Register dst, int32_t imm32) {
12292   int encode = prefixq_and_encode(dst->encoding());
12293   emit_int16((unsigned char)0xC7, (0xC0 | encode));
12294   emit_int32(imm32);
12295 }
12296 
12297 void Assembler::movsbq(Register dst, Address src) {
12298   InstructionMark im(this);
12299   emit_int24(get_prefixq(src, dst),
12300              0x0F,
12301              (unsigned char)0xBE);
12302   emit_operand(dst, src);
12303 }
12304 
12305 void Assembler::movsbq(Register dst, Register src) {
12306   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12307   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
12308 }
12309 
12310 void Assembler::movslq(Register dst, int32_t imm32) {
12311   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
12312   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
12313   // as a result we shouldn't use until tested at runtime...
12314   ShouldNotReachHere();
12315   InstructionMark im(this);
12316   int encode = prefixq_and_encode(dst->encoding());
12317   emit_int8(0xC7 | encode);
12318   emit_int32(imm32);
12319 }
12320 
12321 void Assembler::movslq(Address dst, int32_t imm32) {
12322   assert(is_simm32(imm32), "lost bits");
12323   InstructionMark im(this);
12324   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12325   emit_operand(rax, dst, 4);
12326   emit_int32(imm32);
12327 }
12328 
12329 void Assembler::movslq(Register dst, Address src) {
12330   InstructionMark im(this);
12331   emit_int16(get_prefixq(src, dst), 0x63);
12332   emit_operand(dst, src);
12333 }
12334 
12335 void Assembler::movslq(Register dst, Register src) {
12336   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12337   emit_int16(0x63, (0xC0 | encode));
12338 }
12339 
12340 void Assembler::movswq(Register dst, Address src) {
12341   InstructionMark im(this);
12342   emit_int24(get_prefixq(src, dst),
12343              0x0F,
12344              (unsigned char)0xBF);
12345   emit_operand(dst, src);
12346 }
12347 
12348 void Assembler::movswq(Register dst, Register src) {
12349   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12350   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
12351 }
12352 
12353 void Assembler::movzbq(Register dst, Address src) {
12354   InstructionMark im(this);
12355   emit_int24(get_prefixq(src, dst),
12356              0x0F,
12357              (unsigned char)0xB6);
12358   emit_operand(dst, src);
12359 }
12360 
12361 void Assembler::movzbq(Register dst, Register src) {
12362   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12363   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
12364 }
12365 
12366 void Assembler::movzwq(Register dst, Address src) {
12367   InstructionMark im(this);
12368   emit_int24(get_prefixq(src, dst),
12369              0x0F,
12370              (unsigned char)0xB7);
12371   emit_operand(dst, src);
12372 }
12373 
12374 void Assembler::movzwq(Register dst, Register src) {
12375   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12376   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
12377 }
12378 
12379 void Assembler::mulq(Address src) {
12380   InstructionMark im(this);
12381   emit_int16(get_prefixq(src), (unsigned char)0xF7);
12382   emit_operand(rsp, src);
12383 }
12384 
12385 void Assembler::mulq(Register src) {
12386   int encode = prefixq_and_encode(src->encoding());
12387   emit_int16((unsigned char)0xF7, (0xE0 | encode));
12388 }
12389 
12390 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
12391   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12392   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12393   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12394   emit_int16((unsigned char)0xF6, (0xC0 | encode));
12395 }
12396 
12397 void Assembler::negq(Register dst) {
12398   int encode = prefixq_and_encode(dst->encoding());
12399   emit_int16((unsigned char)0xF7, (0xD8 | encode));
12400 }
12401 
12402 void Assembler::negq(Address dst) {
12403   InstructionMark im(this);
12404   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
12405   emit_operand(as_Register(3), dst);
12406 }
12407 
12408 void Assembler::notq(Register dst) {
12409   int encode = prefixq_and_encode(dst->encoding());
12410   emit_int16((unsigned char)0xF7, (0xD0 | encode));
12411 }
12412 
12413 void Assembler::btsq(Address dst, int imm8) {
12414   assert(isByte(imm8), "not a byte");
12415   InstructionMark im(this);
12416   emit_int24(get_prefixq(dst),
12417              0x0F,
12418              (unsigned char)0xBA);
12419   emit_operand(rbp /* 5 */, dst, 1);
12420   emit_int8(imm8);
12421 }
12422 
12423 void Assembler::btrq(Address dst, int imm8) {
12424   assert(isByte(imm8), "not a byte");
12425   InstructionMark im(this);
12426   emit_int24(get_prefixq(dst),
12427              0x0F,
12428              (unsigned char)0xBA);
12429   emit_operand(rsi /* 6 */, dst, 1);
12430   emit_int8(imm8);
12431 }
12432 
12433 void Assembler::orq(Address dst, int32_t imm32) {
12434   InstructionMark im(this);
12435   prefixq(dst);
12436   emit_arith_operand(0x81, as_Register(1), dst, imm32);
12437 }
12438 
12439 void Assembler::orq(Address dst, Register src) {
12440   InstructionMark im(this);
12441   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
12442   emit_operand(src, dst);
12443 }
12444 
12445 void Assembler::orq(Register dst, int32_t imm32) {
12446   (void) prefixq_and_encode(dst->encoding());
12447   emit_arith(0x81, 0xC8, dst, imm32);
12448 }
12449 
12450 void Assembler::orq(Register dst, Address src) {
12451   InstructionMark im(this);
12452   emit_int16(get_prefixq(src, dst), 0x0B);
12453   emit_operand(dst, src);
12454 }
12455 
12456 void Assembler::orq(Register dst, Register src) {
12457   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12458   emit_arith(0x0B, 0xC0, dst, src);
12459 }
12460 
12461 void Assembler::popcntq(Register dst, Address src) {
12462   assert(VM_Version::supports_popcnt(), "must support");
12463   InstructionMark im(this);
12464   emit_int32((unsigned char)0xF3,
12465              get_prefixq(src, dst),
12466              0x0F,
12467              (unsigned char)0xB8);
12468   emit_operand(dst, src);
12469 }
12470 
12471 void Assembler::popcntq(Register dst, Register src) {
12472   assert(VM_Version::supports_popcnt(), "must support");
12473   emit_int8((unsigned char)0xF3);
12474   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12475   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
12476 }
12477 
12478 void Assembler::popq(Address dst) {
12479   InstructionMark im(this);
12480   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
12481   emit_operand(rax, dst);
12482 }
12483 
12484 void Assembler::popq(Register dst) {
12485   emit_int8((unsigned char)0x58 | dst->encoding());
12486 }
12487 
12488 // Precomputable: popa, pusha, vzeroupper
12489 
12490 // The result of these routines are invariant from one invocation to another
12491 // invocation for the duration of a run. Caching the result on bootstrap
12492 // and copying it out on subsequent invocations can thus be beneficial
12493 static bool     precomputed = false;
12494 
12495 static u_char* popa_code  = NULL;
12496 static int     popa_len   = 0;
12497 
12498 static u_char* pusha_code = NULL;
12499 static int     pusha_len  = 0;
12500 
12501 static u_char* vzup_code  = NULL;
12502 static int     vzup_len   = 0;
12503 
12504 void Assembler::precompute_instructions() {
12505   assert(!Universe::is_fully_initialized(), "must still be single threaded");
12506   guarantee(!precomputed, "only once");
12507   precomputed = true;
12508   ResourceMark rm;
12509 
12510   // Make a temporary buffer big enough for the routines we're capturing
12511   int size = 256;
12512   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
12513   CodeBuffer buffer((address)tmp_code, size);
12514   MacroAssembler masm(&buffer);
12515 
12516   address begin_popa  = masm.code_section()->end();
12517   masm.popa_uncached();
12518   address end_popa    = masm.code_section()->end();
12519   masm.pusha_uncached();
12520   address end_pusha   = masm.code_section()->end();
12521   masm.vzeroupper_uncached();
12522   address end_vzup    = masm.code_section()->end();
12523 
12524   // Save the instructions to permanent buffers.
12525   popa_len = (int)(end_popa - begin_popa);
12526   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
12527   memcpy(popa_code, begin_popa, popa_len);
12528 
12529   pusha_len = (int)(end_pusha - end_popa);
12530   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
12531   memcpy(pusha_code, end_popa, pusha_len);
12532 
12533   vzup_len = (int)(end_vzup - end_pusha);
12534   if (vzup_len > 0) {
12535     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
12536     memcpy(vzup_code, end_pusha, vzup_len);
12537   } else {
12538     vzup_code = pusha_code; // dummy
12539   }
12540 
12541   assert(masm.code()->total_oop_size() == 0 &&
12542          masm.code()->total_metadata_size() == 0 &&
12543          masm.code()->total_relocation_size() == 0,
12544          "pre-computed code can't reference oops, metadata or contain relocations");
12545 }
12546 
12547 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
12548   assert(src != NULL, "code to copy must have been pre-computed");
12549   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
12550   address end = code_section->end();
12551   memcpy(end, src, src_len);
12552   code_section->set_end(end + src_len);
12553 }
12554 
12555 void Assembler::popa() { // 64bit
12556   emit_copy(code_section(), popa_code, popa_len);
12557 }
12558 
12559 void Assembler::popa_uncached() { // 64bit
12560   movq(r15, Address(rsp, 0));
12561   movq(r14, Address(rsp, wordSize));
12562   movq(r13, Address(rsp, 2 * wordSize));
12563   movq(r12, Address(rsp, 3 * wordSize));
12564   movq(r11, Address(rsp, 4 * wordSize));
12565   movq(r10, Address(rsp, 5 * wordSize));
12566   movq(r9,  Address(rsp, 6 * wordSize));
12567   movq(r8,  Address(rsp, 7 * wordSize));
12568   movq(rdi, Address(rsp, 8 * wordSize));
12569   movq(rsi, Address(rsp, 9 * wordSize));
12570   movq(rbp, Address(rsp, 10 * wordSize));
12571   // Skip rsp as it is restored automatically to the value
12572   // before the corresponding pusha when popa is done.
12573   movq(rbx, Address(rsp, 12 * wordSize));
12574   movq(rdx, Address(rsp, 13 * wordSize));
12575   movq(rcx, Address(rsp, 14 * wordSize));
12576   movq(rax, Address(rsp, 15 * wordSize));
12577 
12578   addq(rsp, 16 * wordSize);
12579 }
12580 
12581 // Does not actually store the value of rsp on the stack.
12582 // The slot for rsp just contains an arbitrary value.
12583 void Assembler::pusha() { // 64bit
12584   emit_copy(code_section(), pusha_code, pusha_len);
12585 }
12586 
12587 // Does not actually store the value of rsp on the stack.
12588 // The slot for rsp just contains an arbitrary value.
12589 void Assembler::pusha_uncached() { // 64bit
12590   subq(rsp, 16 * wordSize);
12591 
12592   movq(Address(rsp, 15 * wordSize), rax);
12593   movq(Address(rsp, 14 * wordSize), rcx);
12594   movq(Address(rsp, 13 * wordSize), rdx);
12595   movq(Address(rsp, 12 * wordSize), rbx);
12596   // Skip rsp as the value is normally not used. There are a few places where
12597   // the original value of rsp needs to be known but that can be computed
12598   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
12599   movq(Address(rsp, 10 * wordSize), rbp);
12600   movq(Address(rsp, 9 * wordSize), rsi);
12601   movq(Address(rsp, 8 * wordSize), rdi);
12602   movq(Address(rsp, 7 * wordSize), r8);
12603   movq(Address(rsp, 6 * wordSize), r9);
12604   movq(Address(rsp, 5 * wordSize), r10);
12605   movq(Address(rsp, 4 * wordSize), r11);
12606   movq(Address(rsp, 3 * wordSize), r12);
12607   movq(Address(rsp, 2 * wordSize), r13);
12608   movq(Address(rsp, wordSize), r14);
12609   movq(Address(rsp, 0), r15);
12610 }
12611 
12612 void Assembler::vzeroupper() {
12613   emit_copy(code_section(), vzup_code, vzup_len);
12614 }
12615 
12616 void Assembler::pushq(Address src) {
12617   InstructionMark im(this);
12618   emit_int16(get_prefixq(src), (unsigned char)0xFF);
12619   emit_operand(rsi, src);
12620 }
12621 
12622 void Assembler::rclq(Register dst, int imm8) {
12623   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12624   int encode = prefixq_and_encode(dst->encoding());
12625   if (imm8 == 1) {
12626     emit_int16((unsigned char)0xD1, (0xD0 | encode));
12627   } else {
12628     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
12629   }
12630 }
12631 
12632 void Assembler::rcrq(Register dst, int imm8) {
12633   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12634   int encode = prefixq_and_encode(dst->encoding());
12635   if (imm8 == 1) {
12636     emit_int16((unsigned char)0xD1, (0xD8 | encode));
12637   } else {
12638     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
12639   }
12640 }
12641 
12642 
12643 void Assembler::rorxq(Register dst, Register src, int imm8) {
12644   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12645   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12646   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
12647   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
12648 }
12649 
12650 void Assembler::rorxd(Register dst, Register src, int imm8) {
12651   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12652   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12653   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
12654   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
12655 }
12656 
12657 #ifdef _LP64
12658 void Assembler::salq(Address dst, int imm8) {
12659   InstructionMark im(this);
12660   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12661   if (imm8 == 1) {
12662     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
12663     emit_operand(as_Register(4), dst);
12664   }
12665   else {
12666     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
12667     emit_operand(as_Register(4), dst);
12668     emit_int8(imm8);
12669   }
12670 }
12671 
12672 void Assembler::salq(Address dst) {
12673   InstructionMark im(this);
12674   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
12675   emit_operand(as_Register(4), dst);
12676 }
12677 
12678 void Assembler::salq(Register dst, int imm8) {
12679   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12680   int encode = prefixq_and_encode(dst->encoding());
12681   if (imm8 == 1) {
12682     emit_int16((unsigned char)0xD1, (0xE0 | encode));
12683   } else {
12684     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
12685   }
12686 }
12687 
12688 void Assembler::salq(Register dst) {
12689   int encode = prefixq_and_encode(dst->encoding());
12690   emit_int16((unsigned char)0xD3, (0xE0 | encode));
12691 }
12692 
12693 void Assembler::sarq(Address dst, int imm8) {
12694   InstructionMark im(this);
12695   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12696   if (imm8 == 1) {
12697     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
12698     emit_operand(as_Register(7), dst);
12699   }
12700   else {
12701     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
12702     emit_operand(as_Register(7), dst);
12703     emit_int8(imm8);
12704   }
12705 }
12706 
12707 void Assembler::sarq(Address dst) {
12708   InstructionMark im(this);
12709   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
12710   emit_operand(as_Register(7), dst);
12711 }
12712 
12713 void Assembler::sarq(Register dst, int imm8) {
12714   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12715   int encode = prefixq_and_encode(dst->encoding());
12716   if (imm8 == 1) {
12717     emit_int16((unsigned char)0xD1, (0xF8 | encode));
12718   } else {
12719     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
12720   }
12721 }
12722 
12723 void Assembler::sarq(Register dst) {
12724   int encode = prefixq_and_encode(dst->encoding());
12725   emit_int16((unsigned char)0xD3, (0xF8 | encode));
12726 }
12727 #endif
12728 
12729 void Assembler::sbbq(Address dst, int32_t imm32) {
12730   InstructionMark im(this);
12731   prefixq(dst);
12732   emit_arith_operand(0x81, rbx, dst, imm32);
12733 }
12734 
12735 void Assembler::sbbq(Register dst, int32_t imm32) {
12736   (void) prefixq_and_encode(dst->encoding());
12737   emit_arith(0x81, 0xD8, dst, imm32);
12738 }
12739 
12740 void Assembler::sbbq(Register dst, Address src) {
12741   InstructionMark im(this);
12742   emit_int16(get_prefixq(src, dst), 0x1B);
12743   emit_operand(dst, src);
12744 }
12745 
12746 void Assembler::sbbq(Register dst, Register src) {
12747   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12748   emit_arith(0x1B, 0xC0, dst, src);
12749 }
12750 
12751 void Assembler::shlq(Register dst, int imm8) {
12752   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12753   int encode = prefixq_and_encode(dst->encoding());
12754   if (imm8 == 1) {
12755     emit_int16((unsigned char)0xD1, (0xE0 | encode));
12756   } else {
12757     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
12758   }
12759 }
12760 
12761 void Assembler::shlq(Register dst) {
12762   int encode = prefixq_and_encode(dst->encoding());
12763   emit_int16((unsigned char)0xD3, (0xE0 | encode));
12764 }
12765 
12766 void Assembler::shrq(Register dst, int imm8) {
12767   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12768   int encode = prefixq_and_encode(dst->encoding());
12769   if (imm8 == 1) {
12770     emit_int16((unsigned char)0xD1, (0xE8 | encode));
12771   }
12772   else {
12773     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
12774   }
12775 }
12776 
12777 void Assembler::shrq(Register dst) {
12778   int encode = prefixq_and_encode(dst->encoding());
12779   emit_int16((unsigned char)0xD3, 0xE8 | encode);
12780 }
12781 
12782 void Assembler::shrq(Address dst) {
12783   InstructionMark im(this);
12784   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
12785   emit_operand(as_Register(5), dst);
12786 }
12787 
12788 void Assembler::shrq(Address dst, int imm8) {
12789   InstructionMark im(this);
12790   assert(isShiftCount(imm8 >> 1), "illegal shift count");
12791   if (imm8 == 1) {
12792     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
12793     emit_operand(as_Register(5), dst);
12794   }
12795   else {
12796     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
12797     emit_operand(as_Register(5), dst);
12798     emit_int8(imm8);
12799   }
12800 }
12801 
12802 void Assembler::subq(Address dst, int32_t imm32) {
12803   InstructionMark im(this);
12804   prefixq(dst);
12805   emit_arith_operand(0x81, rbp, dst, imm32);
12806 }
12807 
12808 void Assembler::subq(Address dst, Register src) {
12809   InstructionMark im(this);
12810   emit_int16(get_prefixq(dst, src), 0x29);
12811   emit_operand(src, dst);
12812 }
12813 
12814 void Assembler::subq(Register dst, int32_t imm32) {
12815   (void) prefixq_and_encode(dst->encoding());
12816   emit_arith(0x81, 0xE8, dst, imm32);
12817 }
12818 
12819 // Force generation of a 4 byte immediate value even if it fits into 8bit
12820 void Assembler::subq_imm32(Register dst, int32_t imm32) {
12821   (void) prefixq_and_encode(dst->encoding());
12822   emit_arith_imm32(0x81, 0xE8, dst, imm32);
12823 }
12824 
12825 void Assembler::subq(Register dst, Address src) {
12826   InstructionMark im(this);
12827   emit_int16(get_prefixq(src, dst), 0x2B);
12828   emit_operand(dst, src);
12829 }
12830 
12831 void Assembler::subq(Register dst, Register src) {
12832   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12833   emit_arith(0x2B, 0xC0, dst, src);
12834 }
12835 
12836 void Assembler::testq(Address dst, int32_t imm32) {
12837   InstructionMark im(this);
12838   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
12839   emit_operand(as_Register(0), dst);
12840   emit_int32(imm32);
12841 }
12842 
12843 void Assembler::testq(Register dst, int32_t imm32) {
12844   // not using emit_arith because test
12845   // doesn't support sign-extension of
12846   // 8bit operands
12847   int encode = dst->encoding();
12848   encode = prefixq_and_encode(encode);
12849   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12850   emit_int32(imm32);
12851 }
12852 
12853 void Assembler::testq(Register dst, Register src) {
12854   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12855   emit_arith(0x85, 0xC0, dst, src);
12856 }
12857 
12858 void Assembler::testq(Register dst, Address src) {
12859   InstructionMark im(this);
12860   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
12861   emit_operand(dst, src);
12862 }
12863 
12864 void Assembler::xaddq(Address dst, Register src) {
12865   InstructionMark im(this);
12866   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
12867   emit_operand(src, dst);
12868 }
12869 
12870 void Assembler::xchgq(Register dst, Address src) {
12871   InstructionMark im(this);
12872   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
12873   emit_operand(dst, src);
12874 }
12875 
12876 void Assembler::xchgq(Register dst, Register src) {
12877   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12878   emit_int16((unsigned char)0x87, (0xc0 | encode));
12879 }
12880 
12881 void Assembler::xorq(Register dst, Register src) {
12882   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12883   emit_arith(0x33, 0xC0, dst, src);
12884 }
12885 
12886 void Assembler::xorq(Register dst, Address src) {
12887   InstructionMark im(this);
12888   emit_int16(get_prefixq(src, dst), 0x33);
12889   emit_operand(dst, src);
12890 }
12891 
12892 void Assembler::xorq(Register dst, int32_t imm32) {
12893   (void) prefixq_and_encode(dst->encoding());
12894   emit_arith(0x81, 0xF0, dst, imm32);
12895 }
12896 
12897 void Assembler::xorq(Address dst, int32_t imm32) {
12898   InstructionMark im(this);
12899   prefixq(dst);
12900   emit_arith_operand(0x81, as_Register(6), dst, imm32);
12901 }
12902 
12903 void Assembler::xorq(Address dst, Register src) {
12904   InstructionMark im(this);
12905   emit_int16(get_prefixq(dst, src), 0x31);
12906   emit_operand(src, dst);
12907 }
12908 
12909 #endif // !LP64
12910 
12911 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
12912   if (VM_Version::supports_evex()) {
12913     _tuple_type = tuple_type;
12914     _input_size_in_bits = input_size_in_bits;
12915   }
12916 }
--- EOF ---